libzypp  17.31.20
request.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
13 #include <zypp-core/zyppng/base/EventDispatcher>
14 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15 #include <zypp-core/zyppng/core/String>
16 #include <zypp-core/fs/PathInfo.h>
18 #include <zypp-curl/CurlConfig>
19 #include <zypp-curl/auth/CurlAuthData>
20 #include <zypp-media/MediaConfig>
21 #include <zypp-core/base/String.h>
22 #include <zypp-core/base/StringV.h>
23 #include <zypp-core/Pathname.h>
24 #include <curl/curl.h>
25 #include <stdio.h>
26 #include <fcntl.h>
27 #include <sstream>
28 #include <utility>
29 
30 #include <iostream>
31 #include <boost/variant.hpp>
32 #include <boost/variant/polymorphic_get.hpp>
33 
34 
35 namespace zyppng {
36 
37  namespace {
38  static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
39  if ( !userdata )
40  return 0;
41 
42  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
43  return that->headerCallback( ptr, size, nmemb );
44  }
45  static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
46  if ( !userdata )
47  return 0;
48 
49  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
50  return that->writeCallback( ptr, size, nmemb );
51  }
52 
53  //helper for std::visit
54  template<class T> struct always_false : std::false_type {};
55  }
56 
57  std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
58  {
59  if ( !fd )
60  return {};
61 
62  fflush( fd );
63 
64  std::vector<char> data( count + 1 , '\0' );
65 
66  ssize_t l = -1;
67  while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
68  ;
69  if (l == -1)
70  return {};
71 
72  return data;
73  }
74 
75  NetworkRequest::Range NetworkRequest::Range::make(size_t start, size_t len, zyppng::NetworkRequest::DigestPtr &&digest, zyppng::NetworkRequest::CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding )
76  {
77  return NetworkRequest::Range {
78  .start = start,
79  .len = len,
80  .bytesWritten = 0,
81  ._digest = std::move( digest ),
82  ._checksum = std::move( expectedChkSum ),
83  ._relevantDigestLen = std::move( digestCompareLen ),
84  ._chksumPad = std::move( dataBlockPadding ),
85  .userData = std::move( userData ),
86  ._rangeState = State::Pending
87  };
88  }
89 
91  : _outFile( std::move(prevState._outFile) )
92  , _downloaded( prevState._downloaded )
93  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
94  { }
95 
97  : _requireStatusPartial( prevState._requireStatusPartial )
98  { }
99 
101  : _outFile( std::move(prevState._outFile) )
102  , _requireStatusPartial( true )
103  , _downloaded( prevState._downloaded )
104  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
105  { }
106 
108  : BasePrivate(p)
109  , _url ( std::move(url) )
110  , _targetFile ( std::move( targetFile) )
111  , _fMode ( std::move(fMode) )
112  , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
113  { }
114 
116  {
117  if ( _easyHandle ) {
118  //clean up for now, later we might reuse handles
119  curl_easy_cleanup( _easyHandle );
120  //reset in request but make sure the request was not enqueued again and got a new handle
121  _easyHandle = nullptr;
122  }
123  }
124 
125  bool NetworkRequestPrivate::initialize( std::string &errBuf )
126  {
127  reset();
128 
129  if ( _easyHandle )
130  //will reset to defaults but keep live connections, session ID and DNS caches
131  curl_easy_reset( _easyHandle );
132  else
133  _easyHandle = curl_easy_init();
134  return setupHandle ( errBuf );
135  }
136 
137  bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
138  {
140  curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
141 
142  const std::string urlScheme = _url.getScheme();
143  if ( urlScheme == "http" || urlScheme == "https" )
145 
146  try {
147 
148  setCurlOption( CURLOPT_PRIVATE, this );
149  setCurlOption( CURLOPT_XFERINFOFUNCTION, NetworkRequestPrivate::curlProgressCallback );
150  setCurlOption( CURLOPT_XFERINFODATA, this );
151  setCurlOption( CURLOPT_NOPROGRESS, 0L);
152  setCurlOption( CURLOPT_FAILONERROR, 1L);
153  setCurlOption( CURLOPT_NOSIGNAL, 1L);
154 
155  std::string urlBuffer( _url.asString() );
156  setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
157 
158  setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
159  setCurlOption( CURLOPT_WRITEDATA, this );
160 
162  setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
163  setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
164  }
166  // instead of returning no data with NOBODY, we return
167  // little data, that works with broken servers, and
168  // works for ftp as well, because retrieving only headers
169  // ftp will return always OK code ?
170  // See http://curl.haxx.se/docs/knownbugs.html #58
172  setCurlOption( CURLOPT_NOBODY, 1L );
173  else
174  setCurlOption( CURLOPT_RANGE, "0-1" );
175  }
176 
178  if ( _requestedRanges.size() ) {
179  if ( ! prepareNextRangeBatch ( errBuf ))
180  return false;
181  } else {
182  std::visit( [&]( auto &arg ){
183  using T = std::decay_t<decltype(arg)>;
184  if constexpr ( std::is_same_v<T, pending_t> ) {
185  arg._requireStatusPartial = false;
186  } else {
187  DBG << _easyHandle << " " << "NetworkRequestPrivate::setupHandle called in unexpected state" << std::endl;
188  }
189  }, _runningMode );
191  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
192  }
193  }
194 
195  //make a local copy of the settings, so headers are not added multiple times
196  TransferSettings locSet = _settings;
197 
198  if ( _dispatcher ) {
199  locSet.setUserAgentString( _dispatcher->agentString().c_str() );
200 
201  // add custom headers as configured (bsc#955801)
202  const auto &cHeaders = _dispatcher->hostSpecificHeaders();
203  if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
204  for ( const auto &[key, value] : i->second ) {
206  "%s: %s", key.c_str(), value.c_str() )
207  ));
208  }
209  }
210  }
211 
212  locSet.addHeader("Pragma:");
213 
216  {
217  case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
218  case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
219  default: break;
220  }
221 
222  setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
223  setCurlOption( CURLOPT_HEADERDATA, this );
224 
228  setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
229  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
230  // just in case curl does not trigger its progress callback frequently
231  // enough.
232  if ( locSet.timeout() )
233  {
234  setCurlOption( CURLOPT_TIMEOUT, 3600L );
235  }
236 
237  if ( urlScheme == "https" )
238  {
239 #if CURLVERSION_AT_LEAST(7,19,4)
240  // restrict following of redirections from https to https only
241  if ( _url.getHost() == "download.opensuse.org" )
242  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
243  else
244  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
245 #endif
246 
247  if( locSet.verifyPeerEnabled() ||
248  locSet.verifyHostEnabled() )
249  {
250  setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
251  }
252 
253  if( ! locSet.clientCertificatePath().empty() )
254  {
255  setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
256  }
257  if( ! locSet.clientKeyPath().empty() )
258  {
259  setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
260  }
261 
262 #ifdef CURLSSLOPT_ALLOW_BEAST
263  // see bnc#779177
264  setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
265 #endif
266  setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
267  setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
268  // bnc#903405 - POODLE: libzypp should only talk TLS
269  setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
270  }
271 
272  // follow any Location: header that the server sends as part of
273  // an HTTP header (#113275)
274  setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
275  // 3 redirects seem to be too few in some cases (bnc #465532)
276  setCurlOption( CURLOPT_MAXREDIRS, 6L );
277 
278  //set the user agent
279  setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
280 
281 
282  /*---------------------------------------------------------------*
283  CURLOPT_USERPWD: [user name]:[password]
284  Url::username/password -> CURLOPT_USERPWD
285  If not provided, anonymous FTP identification
286  *---------------------------------------------------------------*/
287  if ( locSet.userPassword().size() )
288  {
289  setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
290  std::string use_auth = _settings.authType();
291  if (use_auth.empty())
292  use_auth = "digest,basic"; // our default
293  long auth = zypp::media::CurlAuthData::auth_type_str2long(use_auth);
294  if( auth != CURLAUTH_NONE)
295  {
296  DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
297  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
298  setCurlOption(CURLOPT_HTTPAUTH, auth);
299  }
300  }
301 
302  if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
303  {
304  DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
305  setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
306  setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
307 
308  /*---------------------------------------------------------------*
309  * CURLOPT_PROXYUSERPWD: [user name]:[password]
310  *
311  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
312  * If not provided, $HOME/.curlrc is evaluated
313  *---------------------------------------------------------------*/
314 
315  std::string proxyuserpwd = locSet.proxyUserPassword();
316 
317  if ( proxyuserpwd.empty() )
318  {
319  zypp::media::CurlConfig curlconf;
320  zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
321  if ( curlconf.proxyuserpwd.empty() )
322  DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
323  else
324  {
325  proxyuserpwd = curlconf.proxyuserpwd;
326  DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
327  }
328  }
329  else
330  {
331  DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
332  }
333 
334  if ( ! proxyuserpwd.empty() )
335  {
336  setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
337  }
338  }
339 #if CURLVERSION_AT_LEAST(7,19,4)
340  else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
341  {
342  // Explicitly disabled in URL (see fillSettingsFromUrl()).
343  // This should also prevent libcurl from looking into the environment.
344  DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
345  setCurlOption(CURLOPT_NOPROXY, "*");
346  }
347 
348 #endif
349  // else: Proxy: not explicitly set; libcurl may look into the environment
350 
352  if ( locSet.minDownloadSpeed() != 0 )
353  {
354  setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
355  // default to 10 seconds at low speed
356  setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
357  }
358 
359 #if CURLVERSION_AT_LEAST(7,15,5)
360  if ( locSet.maxDownloadSpeed() != 0 )
361  setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
362 #endif
363 
365  if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
366  setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
367  else
368  MIL << _easyHandle << " " << "No cookies requested" << std::endl;
369  setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
370 
371 #if CURLVERSION_AT_LEAST(7,18,0)
372  // bnc #306272
373  setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
374 #endif
375 
376  // Append settings custom headers to curl.
377  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
378  for ( const auto &header : locSet.headers() ) {
379  if ( !z_func()->addRequestHeader( header.c_str() ) )
381  }
382 
383  if ( _headers )
384  setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
385 
386  return true;
387 
388  } catch ( const zypp::Exception &excp ) {
389  ZYPP_CAUGHT(excp);
390  errBuf = excp.asString();
391  }
392  return false;
393  }
394 
396  {
397  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
398  if ( !rmode ) {
399  DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
400  return false;
401  }
402  // if we have no open file create or open it
403  if ( !rmode->_outFile ) {
404  std::string openMode = "w+b";
406  openMode = "r+b";
407 
408  rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
409 
410  //if the file does not exist create a new one
411  if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
412  rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
413  }
414 
415  if ( !rmode->_outFile ) {
417  ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
418  return false;
419  }
420  }
421 
422  return true;
423  }
424 
426  {
427  // We can recover from RangeFail errors if we have more batch sizes to try
428  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
429  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail )
430  return ( rmode->_rangeAttemptIdx + 1 < sizeof( _rangeAttempt ) ) && hasMoreWork();
431  return false;
432  }
433 
434  bool NetworkRequestPrivate::prepareToContinue( std::string &errBuf )
435  {
436  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
437 
438  if ( hasMoreWork() ) {
439  // go to the next range batch level if we are restarted due to a failed range request
440  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail ) {
441  if ( rmode->_rangeAttemptIdx + 1 >= sizeof( _rangeAttempt ) ) {
442  errBuf = "No more range batch sizes available";
443  return false;
444  }
445  rmode->_rangeAttemptIdx++;
446  }
447 
448  _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
449 
450  // we reset the handle to default values. We do this to not run into
451  // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
452  // we cancel a connection because of a range error to request a smaller batch.
453  // The error will still happen but much less frequently than without resetting the handle.
454  //
455  // Note: Even creating a new handle will NOT fix the issue
456  curl_easy_reset( _easyHandle );
457  if ( !setupHandle (errBuf) )
458  return false;
459  return true;
460  }
461  errBuf = "Request has no more work";
462  return false;
463 
464  }
465 
467  {
468  if ( _requestedRanges.size() == 0 ) {
469  errBuf = "Calling the prepareNextRangeBatch function without a range to download is not supported.";
470  return false;
471  }
472 
473  std::string rangeDesc;
474  uint rangesAdded = 0;
475  if ( _requestedRanges.size() > 1 && _protocolMode != ProtocolMode::HTTP ) {
476  errBuf = "Using more than one range is not supported with protocols other than HTTP/HTTPS";
477  return false;
478  }
479 
480  // check if we have one big range convering the whole file
481  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 ) {
482  if ( !std::holds_alternative<pending_t>( _runningMode ) ) {
483  errBuf = zypp::str::Str() << "Unexpected state when calling prepareNextRangeBatch " << _runningMode.index ();
484  return false;
485  }
486 
487  _requestedRanges[0]._rangeState = NetworkRequest::Running;
488  std::get<pending_t>( _runningMode )._requireStatusPartial = false;
489 
490  } else {
491  std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
492  return ( elem1.start < elem2.start );
493  });
494 
495  if ( std::holds_alternative<pending_t>( _runningMode ) )
496  std::get<pending_t>( _runningMode )._requireStatusPartial = true;
497 
498  auto maxRanges = _rangeAttempt[0];
499  if ( std::holds_alternative<prepareNextRangeBatch_t>( _runningMode ) )
500  maxRanges = _rangeAttempt[std::get<prepareNextRangeBatch_t>( _runningMode )._rangeAttemptIdx];
501 
502  // helper function to build up the request string for the range
503  auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
504  std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
505  if( range.second > 0 )
506  rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
507 
508  if ( rangeDesc.size() )
509  rangeDesc.append(",").append( rangeD );
510  else
511  rangeDesc = std::move( rangeD );
512 
513  rangesAdded++;
514  };
515 
516  std::optional<std::pair<size_t, size_t>> currentZippedRange;
517  bool closedRange = true;
518  for ( auto &range : _requestedRanges ) {
519 
520  if ( range._rangeState != NetworkRequest::Pending )
521  continue;
522 
523  //reset the download results
524  range.bytesWritten = 0;
525 
526  //when we have a open range in the list of ranges we will get from start of range to end of file,
527  //all following ranges would never be marked as valid, so we have to fail early
528  if ( !closedRange ) {
529  errBuf = "It is not supported to request more ranges after a open range.";
530  return false;
531  }
532 
533  const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
534  closedRange = (rangeEnd > 0);
535 
536  // remember this range was already requested
537  range._rangeState = NetworkRequest::Running;
538  range.bytesWritten = 0;
539  if ( range._digest )
540  range._digest->reset();
541 
542  // we try to compress the requested ranges into as big chunks as possible for the request,
543  // when receiving we still track the original ranges so we can collect and test their checksums
544  if ( !currentZippedRange ) {
545  currentZippedRange = std::make_pair( range.start, rangeEnd );
546  } else {
547  //range is directly consecutive to the previous range
548  if ( currentZippedRange->second + 1 == range.start ) {
549  currentZippedRange->second = rangeEnd;
550  } else {
551  //this range does not directly follow the previous one, we build the string and start a new one
552  addRangeString( *currentZippedRange );
553  currentZippedRange = std::make_pair( range.start, rangeEnd );
554  }
555  }
556 
557  if ( rangesAdded >= maxRanges ) {
558  MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
559  break;
560  }
561  }
562 
563  // add the last range too
564  if ( currentZippedRange )
565  addRangeString( *currentZippedRange );
566 
567  MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
568 
569  setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
570  }
571 
572  return true;
573  }
574 
576  {
577  // check if we have ranges that have never been requested
578  return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == NetworkRequest::Pending; });
579  }
580 
582  {
583  bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
584  if ( isRangeContinuation ) {
585  MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
586  _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
587  } else {
588  auto mode = running_t( std::move(std::get<pending_t>( _runningMode )) );
589  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 )
590  mode._currentRange = 0;
591 
592  _runningMode = std::move(mode);
593  }
594 
595  auto &m = std::get<running_t>( _runningMode );
596 
597  if ( m._activityTimer ) {
598  DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
599  m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
600  m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
601  }
602 
603  if ( !isRangeContinuation )
604  _sigStarted.emit( *z_func() );
605  }
606 
608  {
609  if ( std::holds_alternative<running_t>(_runningMode) ) {
610  auto &rmode = std::get<running_t>( _runningMode );
611  // if we still have a current range set it valid by checking the checksum
612  if ( rmode._currentRange >= 0 ) {
613  auto &currR = _requestedRanges[rmode._currentRange];
614  rmode._currentRange = -1;
615  validateRange( currR );
616  }
617  }
618  }
619 
621  {
622 
623  finished_t resState;
624  resState._result = std::move(err);
625 
626  if ( std::holds_alternative<running_t>(_runningMode) ) {
627 
628  auto &rmode = std::get<running_t>( _runningMode );
629  rmode._outFile.reset();
630  resState._downloaded = rmode._downloaded;
631  resState._contentLenght = rmode._contentLenght;
632 
634  //we have a successful download lets see if we got everything we needed
635  for ( const auto &r : _requestedRanges ) {
636  if ( r._rangeState != NetworkRequest::Finished ) {
637  if ( r.len > 0 && r.bytesWritten != r.len )
638  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
639  else if ( r._digest && r._checksum.size() && ! checkIfRangeChkSumIsValid(r) ) {
640  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
641  } else {
643  }
644  //we only report the first error
645  break;
646  }
647  }
648  }
649  }
650 
651  _runningMode = std::move( resState );
652  _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
653  }
654 
656  {
658  _headers.reset( nullptr );
659  _errorBuf.fill( 0 );
661  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
662  range._rangeState = NetworkRequest::Pending;
663  });
664  }
665 
667  {
668  auto &m = std::get<running_t>( _runningMode );
669 
670  MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
671  std::map<std::string, boost::any> extraInfo;
672  extraInfo.insert( {"requestUrl", _url } );
673  extraInfo.insert( {"filepath", _targetFile } );
674  _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
675  }
676 
678  {
679  if ( rng._digest && rng._checksum.size() ) {
680  auto bytesHashed = rng._digest->bytesHashed ();
681  if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
682  MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
683  zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
684  rng._digest->update( padding.data(), padding.size() );
685  }
686  auto digVec = rng._digest->digestVector();
687  if ( rng._relevantDigestLen ) {
688  digVec.resize( *rng._relevantDigestLen );
689  }
690  return ( digVec == rng._checksum );
691  }
692 
693  // no checksum required
694  return true;
695  }
696 
698  {
699  if ( rng._digest && rng._checksum.size() ) {
700  if ( ( rng.len == 0 || rng.bytesWritten == rng.len ) && checkIfRangeChkSumIsValid(rng) )
702  else
704  } else {
705  if ( rng.len == 0 ? true : rng.bytesWritten == rng.len )
707  else
709  }
710  }
711 
712  bool NetworkRequestPrivate::parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len )
713  { //content-range: bytes 10485760-19147879/19147880
714  static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
715 
716  zypp::str::smatch what;
717  if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
718  DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
719  return false;
720  }
721 
722  size_t s = zypp::str::strtonum<size_t>( what[1]);
723  size_t e = zypp::str::strtonum<size_t>( what[2]);
724  start = std::move(s);
725  len = ( e - s ) + 1;
726  return true;
727  }
728 
729  bool NetworkRequestPrivate::parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
730  {
731  static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
732 
733  zypp::str::smatch what;
734  if( zypp::str::regex_match( std::string(line), what, regex ) ) {
735  if ( what.size() >= 2 ) {
736  boundary = what[1];
737  return true;
738  }
739  }
740  return false;
741  }
742 
744  {
745  return std::string( _errorBuf.data() );
746  }
747 
749  {
750  if ( std::holds_alternative<running_t>( _runningMode ) ){
751  auto &rmode = std::get<running_t>( _runningMode );
752  if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
753  rmode._activityTimer->start();
754  }
755  }
756 
757  int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
758  {
759  if ( !clientp )
760  return CURLE_OK;
761  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
762 
763  if ( !std::holds_alternative<running_t>(that->_runningMode) ){
764  DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
765  return -1;
766  }
767 
768  auto &rmode = std::get<running_t>( that->_runningMode );
769 
770  //reset the timer
771  that->resetActivityTimer();
772 
773  rmode._isInCallback = true;
774  if ( rmode._lastProgressNow != dlnow ) {
775  rmode._lastProgressNow = dlnow;
776  that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
777  }
778  rmode._isInCallback = false;
779 
780  return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
781  }
782 
783  size_t NetworkRequestPrivate::headerCallback(char *ptr, size_t size, size_t nmemb)
784  {
785  //it is valid to call this function with no data to write, just return OK
786  if ( size * nmemb == 0)
787  return 0;
788 
790 
792 
793  std::string_view hdr( ptr, size*nmemb );
794 
795  hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
796  const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
797  if ( lastNonWhitespace != hdr.npos )
798  hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
799  else
800  hdr = std::string_view();
801 
802  auto &rmode = std::get<running_t>( _runningMode );
803  if ( !hdr.size() ) {
804  return ( size * nmemb );
805  }
806  if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
807 
808  long statuscode = 0;
809  (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
810 
811  const auto &doRangeFail = [&](){
812  WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
813  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::RangeFail, "Expected range status code 206, but got none." );
814 
815  // reset all ranges we requested to pending, we never got the data for them
816  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
817  if ( range._rangeState == NetworkRequest::Running )
818  range._rangeState = NetworkRequest::Pending;
819  });
820  return 0;
821  };
822 
823  // if we have a status 204 we need to create a empty file
824  if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
826 
827  if ( rmode._requireStatusPartial ) {
828  // ignore other status codes, maybe we are redirected etc.
829  if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
830  || statuscode == 416 ) {
831  return doRangeFail();
832  }
833  }
834 
835  } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
836  _lastRedirect = hdr.substr( 9 );
837  DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
838 
839  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
840  std::string sep;
841  if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
842  rmode._gotMultiRangeHeader = true;
843  rmode._seperatorString = "--"+sep;
844  }
845  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
847  if ( !parseContentRangeHeader( hdr, r.start, r.len) ) {
848  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
849  return 0;
850  }
851  DBG << _easyHandle << " " << "Got content range :" << r.start << " len " << r.len << std::endl;
852  rmode._gotContentRangeHeader = true;
853  rmode._currentSrvRange = r;
854 
855  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
856  auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
857  auto str = std::string ( lenStr.data(), lenStr.length() );
858  auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
859  if ( len > 0 ) {
860  DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
861  rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
862  }
863  }
864  }
865 
866  return ( size * nmemb );
867  }
868 
869  size_t NetworkRequestPrivate::writeCallback(char *ptr, size_t size, size_t nmemb)
870  {
871  const auto max = ( size * nmemb );
872 
874 
875  //it is valid to call this function with no data to write, just return OK
876  if ( max == 0)
877  return 0;
878 
879  //in case of a HEAD request, we do not write anything
881  return ( size * nmemb );
882  }
883 
884  auto &rmode = std::get<running_t>( _runningMode );
885 
886  auto writeDataToFile = [ this, &rmode ]( off_t offset, const char *data, size_t len ) -> off_t {
887 
888  if ( rmode._currentRange < 0 ) {
889  DBG << _easyHandle << " " << "Current range is zero in write request" << std::endl;
890  return 0;
891  }
892 
893  // if we have no open file create or open it
894  if ( !assertOutputFile() )
895  return 0;
896 
897  // seek to the given offset
898  if ( offset >= 0 ) {
899  if ( fseek( rmode._outFile, offset, SEEK_SET ) != 0 ) {
901  "Unable to set output file pointer." );
902  return 0;
903  }
904  }
905 
906  auto &rng = _requestedRanges[ rmode._currentRange ];
907  const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, len ) : len;
908 
909  //make sure we do not write after the expected file size
910  if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>(rng.start + rng.bytesWritten + bytesToWrite) ) {
911  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
912  return 0;
913  }
914 
915  auto written = fwrite( data, 1, bytesToWrite, rmode._outFile );
916  if ( written == 0 )
917  return 0;
918 
919  if ( rng._digest && rng._checksum.size() ) {
920  if ( !rng._digest->update( data, written ) )
921  return 0;
922  }
923 
924  rng.bytesWritten += written;
925  if ( rmode._currentSrvRange ) rmode._currentSrvRange->bytesWritten += written;
926 
927  if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
928  rmode._currentRange = -1;
929  validateRange( rng );
930  }
931 
932  if ( rmode._currentSrvRange && rmode._currentSrvRange->len > 0 && rmode._currentSrvRange->bytesWritten >= rmode._currentSrvRange->len ) {
933  rmode._currentSrvRange.reset();
934  // we ran out of data in the current chunk, reset the target range as well because next data will be
935  // a chunk header again
936  rmode._currentRange = -1;
937  }
938 
939  // count the number of real bytes we have downloaded so far
940  rmode._downloaded += written;
941  _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
942 
943  return written;
944  };
945 
946  // we are currenty writing a range, continue until we hit the end of the requested chunk, or if we hit end of data
947  size_t bytesWrittenSoFar = 0;
948 
949  while ( bytesWrittenSoFar != max ) {
950 
951  off_t seekTo = -1;
952 
953  // this is called after all headers have been processed
954  if ( !rmode._allHeadersReceived ) {
955  rmode._allHeadersReceived = true;
956 
957  // no ranges at all, must be a normal download
958  if ( !rmode._gotMultiRangeHeader && !rmode._gotContentRangeHeader ) {
959 
960  if ( rmode._requireStatusPartial ) {
961  //we got a invalid response, the status code pointed to being partial but we got no range definition
963  "Invalid data from server, range respone was announced but there was no range definiton." );
964  return 0;
965  }
966 
967  //we always download a range even if it is not explicitly requested
968  if ( _requestedRanges.empty() ) {
970  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
971  }
972 
973  rmode._currentRange = 0;
974  seekTo = _requestedRanges[0].start;
975  }
976  }
977 
978  if ( rmode._currentSrvRange && rmode._currentRange == -1 ) {
979  //if we enter this branch, we just have finished writing a requested chunk but
980  //are still inside a chunk that was sent by the server, due to the std the server can coalesce requested ranges
981  //to optimize downloads we need to find the best match ( because the current offset might not even be in our requested ranges )
982  //Or we just parsed a Content-Lenght header and start a new block
983 
984  std::optional<uint> foundRange;
985  const size_t beginRange = rmode._currentSrvRange->start + rmode._currentSrvRange->bytesWritten;
986  const size_t endRange = beginRange + (rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten);
987  auto currDist = ULONG_MAX;
988  for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
989  const auto &currR = _requestedRanges[i];
990 
991  // do not allow double ranges
992  if ( currR._rangeState == NetworkRequest::Finished || currR._rangeState == NetworkRequest::Error )
993  continue;
994 
995  // check if the range was already written
996  if ( currR.len == currR.bytesWritten )
997  continue;
998 
999  const auto currRBegin = currR.start + currR.bytesWritten;
1000  if ( !( beginRange <= currRBegin && endRange >= currRBegin ) )
1001  continue;
1002 
1003  // calculate the distance of the current ranges offset+data written to the range we got back from the server
1004  const auto newDist = currRBegin - beginRange;
1005 
1006  if ( !foundRange ) {
1007  foundRange = i;
1008  currDist = newDist;
1009  } else {
1010  //pick the range with the closest distance
1011  if ( newDist < currDist ) {
1012  foundRange = i;
1013  currDist = newDist;
1014  }
1015  }
1016  }
1017  if ( !foundRange ) {
1019  , "Unable to find a matching range for data returned by the server." );
1020  return 0;
1021  }
1022 
1023  //set the found range as the current one
1024  rmode._currentRange = *foundRange;
1025 
1026  //continue writing where we stopped
1027  seekTo = _requestedRanges[*foundRange].start + _requestedRanges[*foundRange].bytesWritten;
1028 
1029  //if we skip bytes we need to advance our written bytecount
1030  const auto skipBytes = seekTo - beginRange;
1031  bytesWrittenSoFar += skipBytes;
1032  rmode._currentSrvRange->bytesWritten += skipBytes;
1033  }
1034 
1035  if ( rmode._currentRange >= 0 ) {
1036  auto availableData = max - bytesWrittenSoFar;
1037  if ( rmode._currentSrvRange ) {
1038  availableData = std::min( availableData, rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten );
1039  }
1040  auto bw = writeDataToFile( seekTo, ptr + bytesWrittenSoFar, availableData );
1041  if ( bw <= 0 )
1042  return 0;
1043 
1044  bytesWrittenSoFar += bw;
1045  }
1046 
1047  if ( bytesWrittenSoFar == max )
1048  return max;
1049 
1050  if ( rmode._currentRange == -1 ) {
1051 
1052  // we still are inside the current range from the server
1053  if ( rmode._currentSrvRange )
1054  continue;
1055 
1056  std::string_view incoming( ptr + bytesWrittenSoFar, max - bytesWrittenSoFar );
1057  auto hdrEnd = incoming.find("\r\n\r\n");
1058  if ( hdrEnd == incoming.npos ) {
1059  //no header end in the data yet, push to buffer and return
1060  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
1061  return max;
1062  }
1063 
1064  //append the data of the current header to the buffer and parse it
1065  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
1066  bytesWrittenSoFar += ( hdrEnd + 4 ); //header data plus header end
1067 
1068  std::string_view data( rmode._rangePrefaceBuffer.data(), rmode._rangePrefaceBuffer.size() );
1069  auto sepStrIndex = data.find( rmode._seperatorString );
1070  if ( sepStrIndex == data.npos ) {
1072  "Invalid multirange header format, seperator string missing." );
1073  return 0;
1074  }
1075 
1076  auto startOfHeader = sepStrIndex + rmode._seperatorString.length();
1077  std::vector<std::string_view> lines;
1078  zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
1079  for ( const auto &hdrLine : lines ) {
1080  if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
1082  //if we can not parse the header the message must be broken
1083  if(! parseContentRangeHeader( hdrLine, r.start, r.len ) ) {
1084  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
1085  return 0;
1086  }
1087  rmode._currentSrvRange = r;
1088  break;
1089  }
1090  }
1091  //clear the buffer again
1092  rmode._rangePrefaceBuffer.clear();
1093  }
1094  }
1095  return bytesWrittenSoFar;
1096  }
1097 
1099 
1100  NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
1101  : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
1102  {
1103  }
1104 
1106  {
1107  Z_D();
1108 
1109  if ( d->_dispatcher )
1110  d->_dispatcher->cancel( *this, "Request destroyed while still running" );
1111  }
1112 
1114  {
1115  d_func()->_expectedFileSize = std::move( expectedFileSize );
1116  }
1117 
1118  void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
1119  {
1120  Z_D();
1121  d->_priority = prio;
1122  if ( state() == Pending && triggerReschedule && d->_dispatcher )
1123  d->_dispatcher->reschedule();
1124  }
1125 
1127  {
1128  return d_func()->_priority;
1129  }
1130 
1131  void NetworkRequest::setOptions( Options opt )
1132  {
1133  d_func()->_options = opt;
1134  }
1135 
1136  NetworkRequest::Options NetworkRequest::options() const
1137  {
1138  return d_func()->_options;
1139  }
1140 
1141  void NetworkRequest::addRequestRange( size_t start, size_t len, DigestPtr digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
1142  {
1143  Z_D();
1144  if ( state() == Running )
1145  return;
1146 
1147  d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
1148  }
1149 
1151  {
1152  Z_D();
1153  if ( state() == Running )
1154  return;
1155 
1156  d->_requestedRanges.push_back( range );
1157  auto &rng = d->_requestedRanges.back();
1158  rng._rangeState = NetworkRequest::Pending;
1159  rng.bytesWritten = 0;
1160  if ( rng._digest )
1161  rng._digest->reset();
1162  }
1163 
1165  {
1166  Z_D();
1167  if ( state() == Running )
1168  return;
1169  d->_requestedRanges.clear();
1170  }
1171 
1172  std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
1173  {
1174  const auto mystate = state();
1175  if ( mystate != Finished && mystate != Error )
1176  return {};
1177 
1178  Z_D();
1179 
1180  std::vector<Range> failed;
1181  for ( const auto &r : d->_requestedRanges ) {
1182  if ( r._rangeState != NetworkRequest::Finished )
1183  failed.push_back( r );
1184  }
1185  return failed;
1186  }
1187 
1188  const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
1189  {
1190  return d_func()->_requestedRanges;
1191  }
1192 
1193  const std::string &NetworkRequest::lastRedirectInfo() const
1194  {
1195  return d_func()->_lastRedirect;
1196  }
1197 
1199  {
1200  return d_func()->_easyHandle;
1201  }
1202 
1203  std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
1204  {
1205  const auto myerr = error();
1206  const auto mystate = state();
1207  if ( mystate != Finished )
1208  return {};
1209 
1210  Timings t;
1211 
1212  auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
1213  using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
1214  double val = 0;
1215  const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
1216  if ( CURLE_OK == res ) {
1217  target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
1218  }
1219  };
1220 
1221  getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
1222  getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
1223  getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
1224  getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
1225  getMeasurement( CURLINFO_TOTAL_TIME, t.total);
1226  getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
1227 
1228  return t;
1229  }
1230 
1231  std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
1232  {
1233  Z_D();
1234 
1235  if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
1236  return {};
1237 
1238  const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
1239  return peek_data_fd( rmode._outFile, offset, count );
1240  }
1241 
1243  {
1244  return d_func()->_url;
1245  }
1246 
1247  void NetworkRequest::setUrl(const Url &url)
1248  {
1249  Z_D();
1250  if ( state() == NetworkRequest::Running )
1251  return;
1252 
1253  d->_url = url;
1254  }
1255 
1257  {
1258  return d_func()->_targetFile;
1259  }
1260 
1262  {
1263  Z_D();
1264  if ( state() == NetworkRequest::Running )
1265  return;
1266  d->_targetFile = path;
1267  }
1268 
1270  {
1271  return d_func()->_fMode;
1272  }
1273 
1275  {
1276  Z_D();
1277  if ( state() == NetworkRequest::Running )
1278  return;
1279  d->_fMode = std::move( mode );
1280  }
1281 
1282  std::string NetworkRequest::contentType() const
1283  {
1284  char *ptr = NULL;
1285  if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
1286  return std::string(ptr);
1287  return std::string();
1288  }
1289 
1291  {
1292  return std::visit([](auto& arg) -> zypp::ByteCount {
1293  using T = std::decay_t<decltype(arg)>;
1294  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1295  return zypp::ByteCount(0);
1296  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1297  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1298  return arg._contentLenght;
1299  else
1300  static_assert(always_false<T>::value, "Unhandled state type");
1301  }, d_func()->_runningMode);
1302  }
1303 
1305  {
1306  return std::visit([](auto& arg) -> zypp::ByteCount {
1307  using T = std::decay_t<decltype(arg)>;
1308  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1309  return zypp::ByteCount();
1310  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1311  || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
1312  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1313  return arg._downloaded;
1314  else
1315  static_assert(always_false<T>::value, "Unhandled state type");
1316  }, d_func()->_runningMode);
1317  }
1318 
1320  {
1321  return d_func()->_settings;
1322  }
1323 
1325  {
1326  return std::visit([this](auto& arg) {
1327  using T = std::decay_t<decltype(arg)>;
1328  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1329  return Pending;
1330  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1331  return Running;
1332  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
1333  if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
1334  return Error;
1335  else
1336  return Finished;
1337  }
1338  else
1339  static_assert(always_false<T>::value, "Unhandled state type");
1340  }, d_func()->_runningMode);
1341  }
1342 
1344  {
1345  const auto s = state();
1346  if ( s != Error && s != Finished )
1347  return NetworkRequestError();
1348  return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1349  }
1350 
1352  {
1353  if ( !hasError() )
1354  return std::string();
1355 
1356  return error().nativeErrorString();
1357  }
1358 
1360  {
1361  return error().isError();
1362  }
1363 
1364  bool NetworkRequest::addRequestHeader( const std::string &header )
1365  {
1366  Z_D();
1367 
1368  curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1369  if ( !res )
1370  return false;
1371 
1372  if ( !d->_headers )
1373  d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1374 
1375  return true;
1376  }
1377 
1378  SignalProxy<void (NetworkRequest &req)> NetworkRequest::sigStarted()
1379  {
1380  return d_func()->_sigStarted;
1381  }
1382 
1383  SignalProxy<void (NetworkRequest &req, zypp::ByteCount count)> NetworkRequest::sigBytesDownloaded()
1384  {
1385  return d_func()->_sigBytesDownloaded;
1386  }
1387 
1388  SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1389  {
1390  return d_func()->_sigProgress;
1391  }
1392 
1393  SignalProxy<void (zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)> NetworkRequest::sigFinished()
1394  {
1395  return d_func()->_sigFinished;
1396  }
1397 
1398 }
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:132
long timeout() const
transfer timeout
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
std::string errorMessage() const
Definition: request.cc:743
bool isError() const
isError Will return true if this is a actual error
#define MIL
Definition: Logger.h:96
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:1203
void * nativeHandle() const
Definition: request.cc:1198
std::optional< size_t > _chksumPad
Definition: request.h:88
#define DBG_MEDIA
Definition: mediadebug_p.h:28
unsigned size() const
Definition: Regex.cc:106
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size...
Definition: request.cc:1290
const std::vector< Range > & requestedRanges() const
Definition: request.cc:1188
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::chrono::microseconds connect
Definition: request.h:98
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
void addRequestRange(size_t start, size_t len=0, DigestPtr digest=nullptr, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:1141
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Regular expression.
Definition: Regex.h:94
ZYPP_IMPL_PRIVATE(Provide)
std::optional< size_t > _relevantDigestLen
Definition: request.h:87
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1383
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
Definition: String.h:1030
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:677
Store and operate with byte count.
Definition: ByteCount.h:30
const std::string & lastRedirectInfo() const
Definition: request.cc:1193
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
const std::string _currentCookieFile
Definition: request_p.h:126
std::chrono::microseconds pretransfer
Definition: request.h:100
Holds transfer setting.
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:1304
const std::string & authType() const
get the allowed authentication types
NetworkRequest::Options _options
Definition: request_p.h:118
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
const char * c_str() const
String representation.
Definition: Pathname.h:110
String related utilities and Regular expression matching.
Definition: Arch.h:360
std::chrono::microseconds appconnect
Definition: request.h:99
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:466
constexpr bool always_false
Definition: PathInfo.cc:544
running_t(pending_t &&prevState)
Definition: request.cc:96
std::string nativeErrorString() const
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:133
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
void setOptions(Options opt)
Definition: request.cc:1131
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
TransferSettings & transferSettings()
Definition: request.cc:1319
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:1113
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:1274
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1359
void onActivityTimeout(Timer &)
Definition: request.cc:666
const Headers & headers() const
returns a list of all added headers (trimmed)
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:184
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
zypp::Pathname _targetFile
Definition: request_p.h:116
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:1247
std::chrono::microseconds namelookup
Definition: request.h:97
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
Do not differentiate case.
Definition: Regex.h:99
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:783
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:211
bool addRequestHeader(const std::string &header)
Definition: request.cc:1364
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
const std::string & asString() const
String representation.
Definition: Pathname.h:91
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:134
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:729
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
long connectTimeout() const
connection timeout
bool initialize(std::string &errBuf)
Definition: request.cc:125
#define WAR
Definition: Logger.h:97
#define nullptr
Definition: Easy.h:55
The NetworkRequestError class Represents a error that occured in.
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:1231
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1343
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
static constexpr int _rangeAttempt[]
Definition: request_p.h:148
UByteArray CheckSumBytes
Definition: request.h:47
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1351
Priority priority() const
Definition: request.cc:1126
std::string proxyuserpwd
Definition: curlconfig.h:49
bool setupHandle(std::string &errBuf)
Definition: request.cc:137
const Pathname & clientKeyPath() const
SSL client key file.
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:1256
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:697
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:141
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
#define MIL_MEDIA
Definition: mediadebug_p.h:29
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:712
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:57
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
bool proxyEnabled() const
proxy is enabled
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:1261
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:757
Regular expression match result.
Definition: Regex.h:167
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:1282
static const Unit B
1 Byte
Definition: ByteCount.h:42
Base class for Exception.
Definition: Exception.h:145
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:125
std::chrono::microseconds total
Definition: request.h:101
bool any_of(const Container &c, Fnc &&cb)
Definition: Algorithm.h:76
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
Definition: request.h:86
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
static Range make(size_t start, size_t len=0, DigestPtr &&digest=nullptr, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition: request.cc:75
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:1118
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:1324
TransferSettings _settings
Definition: request_p.h:117
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:129
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
virtual ~NetworkRequest()
Definition: request.cc:1105
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Options options() const
Definition: request.cc:1136
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:869
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
std::chrono::microseconds redirect
Definition: request.h:102
std::shared_ptr< zypp::Digest > DigestPtr
Definition: request.h:46
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1393
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:135
Type type() const
type Returns the type of the error
These are enforced even if you don&#39;t pass them as flag argument.
Definition: Regex.h:103
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1378
std::string userPassword() const
returns the user and password as a user:pass string
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:1269
std::vector< Range > failedRanges() const
Definition: request.cc:1172
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:107
void setResult(NetworkRequestError &&err)
Definition: request.cc:620
const std::string & proxy() const
proxy host
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1388
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:434
const std::string & userAgentString() const
user agent string (trimmed)
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
ZYppCommitResult & _result
Definition: TargetImpl.cc:1597
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:211