24    inline bool preloadEnabled()
 
   27      if ( indeterminate(envstate) ) {
 
   28#if APIConfig(LIBZYPP_CONFIG_USE_SERIAL_PACKAGE_DOWNLOAD_BY_DEFAULT) 
   34      return bool(envstate);
 
   37    zypp::Pathname pckCachedLocation ( 
const PoolItem &pck ) {
 
   79      if ( 
_parent._requiredDls.empty() ) {
 
   87        MIL << 
"No more jobs pending, exiting worker" << std::endl;
 
   95      _parent._requiredDls.pop_front();
 
   97      auto loc = 
_job.lookupLocation();
 
  107        ERR << 
"Failed to create target dir for file: " << 
_targetPath << std::endl;
 
  122          if ( !pathInfo.
isFile() ) {
 
  123            if ( pathInfo.
isDir () )
 
  128          } 
else if ( is_checksum( 
_targetPath, loc.checksum() ) ) {
 
  150        _req->resetRequestRanges();
 
  158      _req->transferSettings() = settings;
 
 
  171      const auto &pi = 
_job;
 
 
  218      auto &repoDlInfo = 
_parent._dlRepoInfo.at( 
_job.repository().id() );
 
  220      std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
 
  221      int currentSmallestRef = INT_MAX;
 
  223      for ( 
auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
 
  224        auto mirrorPtr = &(*i);
 
  226        if ( skip == mirrorPtr )
 
  234        if ( ( i->refs + i->miss ) < currentSmallestRef ) {
 
  235          currentSmallestRef = ( i->refs + i->miss );
 
  240      if ( curr == repoDlInfo._baseUrls.end() )
 
 
  246      MIL << 
"Request for " << req.
url() << 
" started" << std::endl;
 
 
  254        userData.
set( 
"Url", 
_req->url() );
 
  265      _parent.reportBytesDownloaded( downloaded );
 
 
  269      MIL << 
"Request for " << req.
url() << 
" finished. (" << err.
toString() << 
")" << std::endl;
 
  281        const auto &error = req.
error();
 
  282        switch ( error.type() ) {
 
  299            MIL << 
"Download from mirror failed for file " << req.
url () << 
" trying to taint mirror and move on" << std::endl;
 
  311              MIL << 
"Found new mirror: " << 
url << 
" recovering, retry count: " << 
_notFoundRetry << std::endl;
 
  314              _req->transferSettings () = settings;
 
  327            std::string authHint = error.extraInfoValue(
"authHint", std::string());
 
  346            DBG << 
"BUG: Download error flag is set , but Error code is NoError" << std::endl;
 
 
  357        userData.
set( 
"Url", *
url );
 
  359        userData.
set( 
"description", *errorMessage );
 
  362        _parent._missedDownloads = 
true;
 
  364      _parent._report->fileDone( localPath, e, userData );
 
 
  375      const auto &loc = 
_job.lookupLocation();
 
  378      if ( loc.medianr() > 1 )
 
  382      url.appendPathName( loc.filename() );
 
  385      for ( 
const auto & el : 
_myMirror->headers ) {
 
  386        std::string header { el.first };
 
  389        MIL << 
"Added custom header -> " << header << std::endl;
 
  394      resultSet = settings;
 
 
  400    zyppng::NetworkRequestRef 
_req;
 
 
  424    if ( !preloadEnabled() ) {
 
  425      MIL << 
"CommitPackagePreloader disabled" << std::endl;
 
  431      MIL << 
"No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
 
  436    _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
 
  443    _pTracker = std::make_shared<internal::ProgressTracker>();
 
  454    for ( 
const auto &step : steps ) {
 
  455      switch ( step.stepType() )
 
  474      if ( pi->lookupLocation().checksum().empty() )
 
  478      if( !pckCachedLocation(pi).empty() )
 
  486          ERR << 
"Failed to create predownload cache for repo " << pi.
repoInfo().
alias() << std::endl;
 
  491        std::vector<RepoUrl> repoUrls;
 
  493        for ( 
const auto &origin: origins ) {
 
  495            media::UrlResolverPlugin::HeaderList custom_headers;
 
  496            Url url = media::UrlResolverPlugin::resolveUrl(u.url(), custom_headers);
 
  498            if ( media::MediaHandlerFactory::handlerType(url) != media::MediaHandlerFactory::MediaCURLType )
 
  503              const auto rewriteUrl = media::MediaNetworkCommonHandler::findGeoIPRedirect( url );
 
  504              if ( rewriteUrl.isValid () )
 
  508            MIL << 
"Adding Url: " << 
url << 
" to the mirror set" << std::endl;
 
  511                                  .baseUrl = std::move(url),
 
  512                                  .headers = std::move(custom_headers)
 
  518        if( repoUrls.empty() ) {
 
  519          MIL << 
"Skipping predownload for " << step.satSolvable() << 
" no downloading URL" << std::endl;
 
  525        if ( repoUrls.begin()->baseUrl.getHost() == 
"download.opensuse.org" ){
 
  532                                ._baseUrls = std::move(repoUrls)
 
  542    if ( _requiredDls.empty() )
 
  546    std::sort( _requiredDls.begin(), _requiredDls.end(), []( 
const PoolItem &
a , 
const PoolItem &
b ) { return a.repository() < b.repository(); });
 
  548    const auto &workerDone = [&, 
this](){
 
  549      if ( std::all_of( _workers.begin(), _workers.end(), []( 
const auto &w ) { return w->finished();} ) )
 
  563      if (_requiredDls.empty())
 
  565      auto worker = std::make_shared<PreloadWorker>(*
this);
 
  566      worker->sigWorkerFinished().connect(workerDone);
 
  568      _workers.push_back( std::move(worker) );
 
  571    if( std::any_of( _workers.begin(), _workers.end(), []( 
const auto &w ) { return !w->finished(); } ) ) {
 
  572      MIL << 
"Running preload event loop!" << std::endl;
 
  576    MIL << 
"Preloading done, mirror stats: " << std::endl;
 
  577    for ( 
const auto &elem : _dlRepoInfo ) {
 
  578      std::for_each ( elem.second._baseUrls.begin (), elem.second._baseUrls.end(), []( 
const RepoUrl &repoUrl ){
 
  579        MIL << 
"url: " << repoUrl.baseUrl << 
" misses: " << repoUrl.miss << std::endl;
 
  582    MIL << 
"Preloading done, mirror stats end" << std::endl;
 
 
  587    if ( !preloadEnabled() ) {
 
  588      MIL << 
"CommitPackagePreloader disabled" << std::endl;
 
  592      filesystem::clean_dir ( Repository(elem.first).info().predownloadPath() );
 
 
  604    const auto now = clock::now();
 
  605    bool canUpdate = 
false;
 
  607      const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - *
_lastProgressUpdate);
 
  608      canUpdate = (duration >= std::chrono::milliseconds(500));
 
  620      userData.
set( 
"dbps_avg"    ,   
static_cast<double>( 
_pTracker->_drateTotal ) );
 
  621      userData.
set( 
"dbps_current",   
static_cast<double>( 
_pTracker->_drateLast ) );
 
  622      userData.
set( 
"bytesReceived",  
static_cast<double>( 
_pTracker->_dnlNow ) );
 
  623      userData.
set( 
"bytesRequired",  
static_cast<double>( 
_pTracker->_dnlTotal ) );
 
 
Store and operate with byte count.
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror.
zyppng::NetworkRequestRef _req
Repository::IdType _currentRepoId
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
void onRequestStarted(zyppng::NetworkRequest &req)
zyppng::SignalProxy< void()> sigWorkerFinished()
PreloadWorker(CommitPackagePreloader &parent)
zyppng::Signal< void()> _sigFinished
zypp::Pathname _targetPath
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found.
std::set< RepoUrl * > _taintedMirrors
CommitPackagePreloader & _parent
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
void finishCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage, bool fatal)
callback::SendReport< media::CommitPreloadReport > _report
ByteCount _downloadedBytes
std::optional< clock::time_point > _lastProgressUpdate
zyppng::Ref< internal::ProgressTracker > _pTracker
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
void reportBytesDownloaded(ByteCount newBytes)
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
zyppng::NetworkRequestDispatcherRef _dispatcher
std::deque< PoolItem > _requiredDls
Represents a single, configurable network endpoint, combining a URL with specific access settings.
Combining sat::Solvable and ResStatus.
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
MirroredOriginSet repoOrigins() const
The repodata origins.
IdType id() const
Expert backdoor.
sat::detail::RepoIdType IdType
static ZConfig & instance()
Singleton ctor.
Typesafe passing of user data via callbacks.
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
Wrapper class for stat/lstat.
bool isExist() const
Return whether valid stat info exists.
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile.
std::string alias() const
unique identifier for this source.
@ TRANSACTION_MULTIINSTALL
[M] Install(multiversion) item (
@ TRANSACTION_INSTALL
[+] Install(update) item
The NetworkRequestError class Represents a error that occured in.
std::string toString() const
toString Returns a string representation of the error
bool hasError() const
Checks if there was a error with the request.
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
NetworkRequestError error() const
Returns the last set Error.
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
TransferSettings & transferSettings()
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
String related utilities and Regular expression matching.
TriBool getenvBool(const C_Str &var_r)
If the environment variable var_r is set to a legal true or false string return bool,...
int rmdir(const Pathname &path)
Like 'rmdir'.
int unlink(const Pathname &path)
Like 'unlink'.
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
int chmodApplyUmask(const Pathname &path, mode_t mode)
Similar to 'chmod', but mode is modified by the process's umask in the usual way.
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
boost::logic::tribool TriBool
3-state boolean logic (true, false and indeterminate).
std::string asString(const Patch::Category &obj)
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
media::UrlResolverPlugin::HeaderList headers
RepoInfo repoInfo() const
Repository repository() const