19#include <zypp-curl/parser/MetaLinkParser> 
   56      struct RepoMirrorListTempProvider
 
   58        RepoMirrorListTempProvider()
 
   61        RepoMirrorListTempProvider( Pathname localfile_r )
 
   62        : _localfile(std::move( localfile_r ))
 
   65        RepoMirrorListTempProvider( 
const Url & url_r )
 
   67          if ( url_r.schemeIsDownloading()
 
   69               && url_r.getQueryStringMap().count(
"mirrorlist") > 0 ) {
 
   72            const auto &authCb = [&]( 
const zypp::Url &, media::TransferSettings &settings, 
const std::string & availAuthTypes, 
bool firstTry, 
bool &canContinue ) {
 
   73              media::CredentialManager cm(media::CredManagerOptions(
ZConfig::instance().repoManagerRoot()));
 
   81            internal::MediaNetworkRequestExecutor executor;
 
   82            executor.sigAuthRequired ().connect(authCb);
 
   84            _tmpfile = filesystem::TmpFile();
 
   85            _localfile = _tmpfile->path();
 
   89            auto tSettings = media::TransferSettings();
 
   92            auto req = std::make_shared<zyppng::NetworkRequest>( url_r, _localfile );
 
   93            req->transferSettings () = tSettings;
 
   94            executor.executeRequest ( req, 
nullptr );
 
   99              ERR << 
"Failed to chmod file " << _localfile << endl;
 
  106          Url abs_url( url_r );
 
  107          abs_url.setPathName( 
"/" );
 
  108          _access.reset( 
new MediaSetAccess( zypp::MirroredOrigin{abs_url} ) );
 
  109          _localfile = _access->provideFile( url_r.getPathName() );
 
  113        const Pathname & localfile()
 const 
  114        { 
return _localfile; }
 
  116        shared_ptr<MediaSetAccess> _access;
 
  118        std::optional<filesystem::TmpFile> _tmpfile;
 
  121      enum class RepoMirrorListFormat {
 
  129      static RepoMirrorListFormat detectRepoMirrorListFormat( 
const Pathname &localfile ) {
 
  133        MIL << 
"Detecting RepoMirrorlist Format based on file content" << std::endl;
 
  135        if ( localfile.empty () )
 
  136          return RepoMirrorListFormat::Empty;
 
  138        InputStream tmpfstream (localfile);
 
  139        auto &str = tmpfstream.stream();
 
  143        while ( !str.eof () && !str.bad() && ( c == 
' ' || c == 
'\t' || c == 
'\n' || c == 
'\r') )
 
  147          ERR << 
"Failed to read RepoMirrorList file, stream hit EOF early." << std::endl;
 
  148          return RepoMirrorListFormat::Empty;
 
  152          ERR << 
"Failed to read RepoMirrorList file, stream became bad." << std::endl;
 
  153          return RepoMirrorListFormat::Error;
 
  158            MIL << 
"Detected Metalink, file starts with <" << std::endl;
 
  159            return RepoMirrorListFormat::MetaLink;
 
  162            MIL << 
"Detected JSON, file starts with [" << std::endl;
 
  163            return RepoMirrorListFormat::MirrorListJson;
 
  166            MIL << 
"Detected TXT, file starts with " << c << std::endl;
 
  167            return RepoMirrorListFormat::MirrorListTxt;
 
  172      inline std::vector<Url> RepoMirrorListParseXML( 
const Pathname &tmpfile )
 
  175          media::MetaLinkParser metalink;
 
  176          metalink.parse(tmpfile);
 
  177          return metalink.getUrls();
 
  180          zypp::parser::ParseException ex(
"Invalid repo metalink format.");
 
  181          ex.remember ( std::current_exception () );
 
  186      inline std::vector<Url> RepoMirrorListParseJSON( 
const Pathname &tmpfile )
 
  188        InputStream tmpfstream (tmpfile);
 
  191          using namespace zyppng::operators;
 
  192          using zyppng::operators::operator|;
 
  195          auto res = parser.
parse ( tmpfstream )
 
  196          | 
and_then([&]( json::Value data ) {
 
  198            std::vector<Url> urls;
 
  200              MIL << 
"Empty mirrorlist received, no mirrors available." << std::endl;
 
  205              MIL << 
"Unexpected JSON format, top level element must be an array." << std::endl;
 
  206              return zyppng::expected<std::vector<Url>>::error( 
ZYPP_EXCPT_PTR( zypp::Exception(
"Unexpected JSON format, top level element must be an array.") ));
 
  208            const auto &topArray = data.
asArray ();
 
  209            for ( 
const auto &val : topArray ) {
 
  211                MIL << 
"Unexpected JSON element, array must contain only objects. Ignoring current element" << std::endl;
 
  215              const auto &obj = val.asObject();
 
  216              for ( 
const auto &key : obj ) {
 
  217                if ( key.first == 
"url" ) {
 
  218                  const auto &elemValue = key.second;
 
  220                    MIL << 
"Unexpected JSON element, element \"url\" must contain a string. Ignoring current element" << std::endl;
 
  224                    MIL << 
"Trying to parse URL: " << std::string(elemValue.asString()) << std::endl;
 
  225                    urls.push_back ( 
Url( elemValue.asString() ) );
 
  226                  } 
catch ( 
const url::UrlException &e ) {
 
  228                    MIL << 
"Invalid URL in mirrors file: "<< elemValue.asString() << 
", ignoring" << std::endl;
 
  237            using zypp::operator<<;
 
  238            MIL << 
"Error while parsing mirrorlist: (" << res.error() << 
"), no mirrors available" << std::endl;
 
  246          MIL << 
"Caught exception while parsing json" << std::endl;
 
  248          zypp::parser::ParseException ex(
"Invalid repo mirror list format, valid JSON was expected.");
 
  249          ex.remember ( std::current_exception () );
 
  255      inline std::vector<Url> RepoMirrorListParseTXT( 
const Pathname &tmpfile )
 
  257        InputStream tmpfstream (tmpfile);
 
  258        std::vector<Url> my_urls;
 
  260        while (
getline(tmpfstream.stream(), tmpurl))
 
  262          if ( tmpurl[0] == 
'#' )
 
  265            Url mirrUrl( tmpurl );
 
  266            if ( !mirrUrl.schemeIsDownloading( ) ) {
 
  267              MIL << 
"Ignoring non downloading URL " << tmpurl << std::endl;
 
  269            my_urls.push_back(
Url(tmpurl));
 
  276            ERR << 
"Invalid URL in mirrorlist file." << std::endl;
 
  278            zypp::parser::ParseException ex(
"Invalid repo mirror list format, all Urls must be valid in a mirrorlist txt file.");
 
  279            ex.remember ( std::current_exception () );
 
  287      inline std::vector<Url> RepoMirrorListParse( 
const Url & url_r, 
const Pathname & listfile_r )
 
  289        MIL << 
"Parsing mirrorlist file: " << listfile_r << 
" originally received from " << url_r << endl;
 
  291        std::vector<Url> mirrorurls;
 
  292        switch( detectRepoMirrorListFormat (listfile_r) ) {
 
  293          case RepoMirrorListFormat::Error:
 
  295            ZYPP_THROW( zypp::parser::ParseException( str::Format(
"Unable to detect metalink file format for: %1%") % listfile_r ));
 
  296          case RepoMirrorListFormat::Empty:
 
  299          case RepoMirrorListFormat::MetaLink:
 
  300            mirrorurls = RepoMirrorListParseXML( listfile_r );
 
  302          case RepoMirrorListFormat::MirrorListJson:
 
  303            mirrorurls = RepoMirrorListParseJSON( listfile_r );
 
  305          case RepoMirrorListFormat::MirrorListTxt:
 
  306            mirrorurls = RepoMirrorListParseTXT( listfile_r );
 
  310        std::vector<Url> ret;
 
  311        for ( 
auto & murl : mirrorurls )
 
  313          if ( murl.getScheme() != 
"rsync" )
 
  315            std::string pName = murl.getPathName();
 
  316            size_t delpos = pName.find(
"repodata/repomd.xml");
 
  317            if( delpos != std::string::npos )
 
  319              murl.setPathName( pName.erase(delpos)  );
 
  321            ret.push_back( murl );
 
  332      PathInfo metaPathInfo( metadatapath_r);
 
  333      std::exception_ptr errors; 
 
  340        else if ( !metaPathInfo.
isDir() )
 
  343          RepoMirrorListTempProvider provider( url_r ); 
 
  344          _urls = RepoMirrorListParse( url_r, provider.localfile() );
 
  353          bool needRefresh = ( !cacheinfo.
isFile()
 
  361          if ( !needRefresh ) {
 
  362            MIL << 
"Mirror cachefile cookie valid and cache is not too old, skipping download (" << cachefile << 
")" << std::endl;
 
  364              _urls = RepoMirrorListParse( url_r, cachefile );
 
  365              if( 
_urls.empty() ) {
 
  366                DBG << 
"Removing Cachefile as it contains no URLs" << endl;
 
  376              errors = std::make_exception_ptr(ex);
 
  377              MIL << 
"Invalid mirrorlist cachefile, deleting it and trying to fetch a new one" << std::endl;
 
  382          if( cacheinfo.
isFile() ) {
 
  390          MIL << 
"Getting MirrorList from URL: " << url_r << endl;
 
  391          RepoMirrorListTempProvider provider( url_r ); 
 
  392          _urls = RepoMirrorListParse( url_r, provider.localfile() );
 
  396            DBG << 
"Copy MirrorList file to " << cachefile << endl;
 
 
  419      static const std::vector<std::string> hosts{
 
  420        "download.opensuse.org",
 
  423      return ( std::find( hosts.begin(), hosts.end(), 
str::toLower( 
url.getHost() )) != hosts.end() );
 
 
  428      std::ifstream file( path_r.
c_str() );
 
  430        WAR << 
"No cookie file " << path_r << endl;
 
 
  447      std::ofstream file(path_r.
c_str());
 
  452      MIL << 
"Saving mirrorlist cookie file " << path_r << std::endl;
 
 
std::string checksum() const
static CheckSum sha256FromString(const std::string &input_r)
Base class for Exception.
void remember(const Exception &old_r)
Store an other Exception as history.
std::string getScheme() const
Returns the scheme name of the URL.
std::string asCompleteString() const
Returns a complete string representation of the Url object.
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
const char * c_str() const
String representation.
const std::string & asString() const
String representation.
zyppng::expected< Value > parse(const InputStream &input_r)
Parse the stream.
const Array & asArray() const
static std::string makeCookie(const zypp::Url &url_r)
Generates the cookie value, currently this is only derived from the Url.
static std::string readCookieFile(const Pathname &path_r)
RepoMirrorList(const Url &url_r, const Pathname &metadatapath_r)
static void saveToCookieFile(const Pathname &path_r, const zypp::Url &url_r)
static bool urlSupportsMirrorLink(const zypp::Url &url)
static constexpr const char * cookieFileName()
static constexpr const char * cacheFileName()
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
std::string getline(std::istream &str)
Read one line from stream.
std::string toLower(const std::string &s)
Return lowercase version of s.
std::string getline(std::istream &str, const Trim trim_r)
Return stream content up to (but not returning) the next newline.
Easy-to use interface to the ZYPP dependency resolver.
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
ResultType and_then(const expected< T, E > &exp, Function &&f)
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_EXCPT_PTR(EXCPT)
Drops a logline and returns Exception as a std::exception_ptr.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.