libzypp  17.31.20
request_p.h
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------/
9 *
10 * This file contains private API, this might break at any time between releases.
11 * You have been warned!
12 *
13 */
14 #ifndef ZYPP_NG_MEDIA_CURL_PRIVATE_REQUEST_P_H_INCLUDED
15 #define ZYPP_NG_MEDIA_CURL_PRIVATE_REQUEST_P_H_INCLUDED
16 
17 #include <zypp-core/zyppng/base/private/base_p.h>
19 #include <zypp-media/MediaException>
20 #include <zypp-core/zyppng/base/Timer>
21 #include <zypp-core/base/Regex.h>
22 #include <curl/curl.h>
23 #include <array>
24 #include <memory>
25 #include <zypp-core/Digest.h>
26 #include <zypp-core/AutoDispose.h>
27 
28 #include <boost/optional.hpp>
29 #include <variant>
30 
31 namespace zyppng {
32 
33 
34 
36  {
37  ZYPP_DECLARE_PUBLIC(NetworkRequest)
38  public:
39  enum class ProtocolMode{
40  Default, //< use this mode if no special checks are required in header or write callbacks
41  HTTP //< this mode is used for HTTP and HTTPS downloads
43 
45  virtual ~NetworkRequestPrivate();
46 
47  bool initialize( std::string &errBuf );
48 
49  bool setupHandle ( std::string &errBuf );
50 
51  bool assertOutputFile ();
52 
58  bool canRecover () const;
59 
65  bool prepareToContinue ( std::string &errBuf );
66 
70  bool prepareNextRangeBatch( std::string &errBuf );
71 
77  bool hasMoreWork() const;
78 
83  void aboutToStart ( );
84 
90  void dequeueNotify();
91 
92  void setResult ( NetworkRequestError &&err );
93  void reset ();
94  void resetActivityTimer ();
95  void onActivityTimeout (Timer &);
98  bool parseContentRangeHeader (const std::string_view &line, size_t &start , size_t &len);
99  bool parseContentTypeMultiRangeHeader ( const std::string_view &line, std::string &boundary );
100 
101  std::string errorMessage () const;
102 
103 
104  std::array<char, CURL_ERROR_SIZE+1> _errorBuf; //provide a buffer for a nicely formatted error for CURL
105 
106  template<typename T>
107  void setCurlOption ( CURLoption opt, T data )
108  {
109  auto ret = curl_easy_setopt( _easyHandle, opt, data );
110  if ( ret != 0 ) {
112  }
113  }
114 
115  Url _url; //file URL
118  NetworkRequest::Options _options;
119  zypp::ByteCount _expectedFileSize; // the file size as expected by the user code
120  std::vector<NetworkRequest::Range> _requestedRanges;
121 
124 
125  std::string _lastRedirect;
126  const std::string _currentCookieFile = "/var/lib/YaST2/cookies";
127 
128  void *_easyHandle = nullptr; // the easy handle that controlling this request
129  NetworkRequestDispatcher *_dispatcher = nullptr; // the parent downloader owning this request
130 
131  //signals
132  Signal< void ( NetworkRequest &req )> _sigStarted;
133  Signal< void ( NetworkRequest &req, zypp::ByteCount count )> _sigBytesDownloaded;
134  Signal< void ( NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow )> _sigProgress;
135  Signal< void ( NetworkRequest &req, const NetworkRequestError &err )> _sigFinished;
136 
137  static int curlProgressCallback ( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow );
138  size_t headerCallback ( char *ptr, size_t size, size_t nmemb );
139  size_t writeCallback ( char *ptr, size_t size, size_t nmemb );
140 
141  std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) > _headers;
142 
143  // when requesting ranges from the server, we need to make sure not to request
144  // too many at the same time. Instead we batch our requests and reuse the open
145  // connection until we have the full file.
146  // However different server have different maximum nr of ranges, so we start with
147  // a high number and decrease until we find a rangecount that works
148  constexpr static int _rangeAttempt[] = {
149  255,
150  127,
151  63,
152  15,
153  5
154  };
155 
156  struct pending_t;
157  struct running_t;
158  struct prepareNextRangeBatch_t;
159 
160  struct pending_t {
162  bool _requireStatusPartial = false;
163  };
164 
166  {
167  prepareNextRangeBatch_t( running_t &&prevState );
168  zypp::AutoFILE _outFile; //the file we are writing to
169  off_t _downloaded = 0; //downloaded bytes
170  int _rangeAttemptIdx = 0; // which range attempt index are we currently using
171  };
172 
173  struct running_t {
174  running_t( pending_t &&prevState );
175  running_t( prepareNextRangeBatch_t &&prevState );
176 
177  Timer::Ptr _activityTimer = Timer::create();
178 
180  off_t _currentRange = -1;
181  std::optional<NetworkRequest::Range> _currentSrvRange;
182 
183  bool _allHeadersReceived = false;
185  bool _gotMultiRangeHeader = false;
186  bool _requireStatusPartial = false;
187 
188  // handle the case when cancel() is called from a slot to the progress signal
189  bool _isInCallback = false;
190 
191  // used to handle cancel() when emitting the progress signal, or when we explicitely trigger
192  // a error during callbacks.
193  std::optional<NetworkRequestError> _cachedResult;
194 
195  off_t _lastProgressNow = -1; // last value returned from CURL, lets only send signals if we get actual updates
196  off_t _downloaded = 0; //downloaded bytes
197  int _rangeAttemptIdx = 0; // which range attempt index are we currently using
198  zypp::ByteCount _contentLenght; // the content length as reported by the server
199 
200  //multirange support for HTTP requests (https://tools.ietf.org/html/rfc7233)
201  std::string _seperatorString;
202  std::vector<char> _rangePrefaceBuffer;
203  };
204 
205  struct finished_t {
206  off_t _downloaded = 0; //downloaded bytes
207  zypp::ByteCount _contentLenght = 0; // the content length as reported by the server
208  NetworkRequestError _result; // the overall result of the download
209  };
210 
211  std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode = pending_t();
212  };
213 
214  std::vector<char> peek_data_fd ( FILE *fd, off_t offset, size_t count );
215 }
216 
217 #endif
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:132
std::string errorMessage() const
Definition: request.cc:743
NetworkRequest::Priority _priority
Definition: request_p.h:123
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:677
Store and operate with byte count.
Definition: ByteCount.h:30
const std::string _currentCookieFile
Definition: request_p.h:126
Holds transfer setting.
NetworkRequest::Options _options
Definition: request_p.h:118
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:466
running_t(pending_t &&prevState)
Definition: request.cc:96
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:133
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void onActivityTimeout(Timer &)
Definition: request.cc:666
zypp::Pathname _targetFile
Definition: request_p.h:116
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:783
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:134
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:729
bool initialize(std::string &errBuf)
Definition: request.cc:125
The NetworkRequestError class Represents a error that occured in.
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
static constexpr int _rangeAttempt[]
Definition: request_p.h:148
bool setupHandle(std::string &errBuf)
Definition: request.cc:137
std::string _seperatorString
The seperator string for multipart responses as defined in RFC 7233 Section 4.1.
Definition: request_p.h:201
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:697
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:141
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:712
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:57
std::optional< NetworkRequest::Range > _currentSrvRange
Definition: request_p.h:181
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:757
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:125
std::optional< NetworkRequestError > _cachedResult
Definition: request_p.h:193
TransferSettings _settings
Definition: request_p.h:117
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:129
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:869
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:312
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:135
std::vector< char > _rangePrefaceBuffer
Here we buffer.
Definition: request_p.h:202
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:107
void setResult(NetworkRequestError &&err)
Definition: request.cc:620
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:434
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:211