httpfetch.cpp 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807
  1. /*
  2. Minetest
  3. Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
  4. This program is free software; you can redistribute it and/or modify
  5. it under the terms of the GNU Lesser General Public License as published by
  6. the Free Software Foundation; either version 2.1 of the License, or
  7. (at your option) any later version.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. GNU Lesser General Public License for more details.
  12. You should have received a copy of the GNU Lesser General Public License along
  13. with this program; if not, write to the Free Software Foundation, Inc.,
  14. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  15. */
  16. #include "httpfetch.h"
  17. #include "porting.h" // for sleep_ms(), get_sysinfo(), secure_rand_fill_buf()
  18. #include <iostream>
  19. #include <sstream>
  20. #include <list>
  21. #include <unordered_map>
  22. #include <cerrno>
  23. #include <mutex>
  24. #include "threading/event.h"
  25. #include "config.h"
  26. #include "exceptions.h"
  27. #include "debug.h"
  28. #include "log.h"
  29. #include "util/container.h"
  30. #include "util/thread.h"
  31. #include "version.h"
  32. #include "settings.h"
  33. #include "noise.h"
  34. static std::mutex g_httpfetch_mutex;
  35. static std::unordered_map<u64, std::queue<HTTPFetchResult>>
  36. g_httpfetch_results;
  37. static PcgRandom g_callerid_randomness;
  38. HTTPFetchRequest::HTTPFetchRequest() :
  39. timeout(g_settings->getS32("curl_timeout")),
  40. connect_timeout(10 * 1000),
  41. useragent(std::string(PROJECT_NAME_C "/") + g_version_hash + " (" + porting::get_sysinfo() + ")")
  42. {
  43. timeout = std::max(timeout, MIN_HTTPFETCH_TIMEOUT_INTERACTIVE);
  44. }
  45. static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result)
  46. {
  47. u64 caller = fetch_result.caller;
  48. if (caller != HTTPFETCH_DISCARD) {
  49. MutexAutoLock lock(g_httpfetch_mutex);
  50. g_httpfetch_results[caller].emplace(fetch_result);
  51. }
  52. }
  53. static void httpfetch_request_clear(u64 caller);
  54. u64 httpfetch_caller_alloc()
  55. {
  56. MutexAutoLock lock(g_httpfetch_mutex);
  57. // Check each caller ID except reserved ones
  58. for (u64 caller = HTTPFETCH_CID_START; caller != 0; ++caller) {
  59. auto it = g_httpfetch_results.find(caller);
  60. if (it == g_httpfetch_results.end()) {
  61. verbosestream << "httpfetch_caller_alloc: allocating "
  62. << caller << std::endl;
  63. // Access element to create it
  64. g_httpfetch_results[caller];
  65. return caller;
  66. }
  67. }
  68. FATAL_ERROR("httpfetch_caller_alloc: ran out of caller IDs");
  69. }
  70. u64 httpfetch_caller_alloc_secure()
  71. {
  72. MutexAutoLock lock(g_httpfetch_mutex);
  73. // Generate random caller IDs and make sure they're not
  74. // already used or reserved.
  75. // Give up after 100 tries to prevent infinite loop
  76. size_t tries = 100;
  77. u64 caller;
  78. do {
  79. caller = (((u64) g_callerid_randomness.next()) << 32) |
  80. g_callerid_randomness.next();
  81. if (--tries < 1) {
  82. FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs");
  83. return HTTPFETCH_DISCARD;
  84. }
  85. } while (caller >= HTTPFETCH_CID_START &&
  86. g_httpfetch_results.find(caller) != g_httpfetch_results.end());
  87. verbosestream << "httpfetch_caller_alloc_secure: allocating "
  88. << caller << std::endl;
  89. // Access element to create it
  90. g_httpfetch_results[caller];
  91. return caller;
  92. }
  93. void httpfetch_caller_free(u64 caller)
  94. {
  95. verbosestream<<"httpfetch_caller_free: freeing "
  96. <<caller<<std::endl;
  97. httpfetch_request_clear(caller);
  98. if (caller != HTTPFETCH_DISCARD) {
  99. MutexAutoLock lock(g_httpfetch_mutex);
  100. g_httpfetch_results.erase(caller);
  101. }
  102. }
  103. bool httpfetch_async_get(u64 caller, HTTPFetchResult &fetch_result)
  104. {
  105. MutexAutoLock lock(g_httpfetch_mutex);
  106. // Check that caller exists
  107. auto it = g_httpfetch_results.find(caller);
  108. if (it == g_httpfetch_results.end())
  109. return false;
  110. // Check that result queue is nonempty
  111. std::queue<HTTPFetchResult> &caller_results = it->second;
  112. if (caller_results.empty())
  113. return false;
  114. // Pop first result
  115. fetch_result = std::move(caller_results.front());
  116. caller_results.pop();
  117. return true;
  118. }
  119. #if USE_CURL
  120. #include <curl/curl.h>
  121. /*
  122. USE_CURL is on: use cURL based httpfetch implementation
  123. */
  124. static size_t httpfetch_writefunction(
  125. char *ptr, size_t size, size_t nmemb, void *userdata)
  126. {
  127. std::ostringstream *stream = (std::ostringstream*)userdata;
  128. size_t count = size * nmemb;
  129. stream->write(ptr, count);
  130. return count;
  131. }
  132. static size_t httpfetch_discardfunction(
  133. char *ptr, size_t size, size_t nmemb, void *userdata)
  134. {
  135. return size * nmemb;
  136. }
  137. class CurlHandlePool
  138. {
  139. std::vector<CURL*> handles;
  140. public:
  141. CurlHandlePool() = default;
  142. ~CurlHandlePool()
  143. {
  144. for (CURL *it : handles) {
  145. curl_easy_cleanup(it);
  146. }
  147. }
  148. CURL * alloc()
  149. {
  150. CURL *curl;
  151. if (handles.empty()) {
  152. curl = curl_easy_init();
  153. if (!curl)
  154. throw std::bad_alloc();
  155. } else {
  156. curl = handles.back();
  157. handles.pop_back();
  158. }
  159. return curl;
  160. }
  161. void free(CURL *handle)
  162. {
  163. if (handle)
  164. handles.push_back(handle);
  165. }
  166. };
  167. class HTTPFetchOngoing
  168. {
  169. public:
  170. HTTPFetchOngoing(const HTTPFetchRequest &request, CurlHandlePool *pool);
  171. ~HTTPFetchOngoing();
  172. CURLcode start(CURLM *multi);
  173. const HTTPFetchResult * complete(CURLcode res);
  174. const HTTPFetchRequest &getRequest() const { return request; };
  175. const CURL *getEasyHandle() const { return curl; };
  176. private:
  177. CurlHandlePool *pool;
  178. CURL *curl = nullptr;
  179. CURLM *multi = nullptr;
  180. HTTPFetchRequest request;
  181. HTTPFetchResult result;
  182. std::ostringstream oss;
  183. struct curl_slist *http_header = nullptr;
  184. curl_mime *multipart_mime = nullptr;
  185. };
  186. HTTPFetchOngoing::HTTPFetchOngoing(const HTTPFetchRequest &request_,
  187. CurlHandlePool *pool_):
  188. pool(pool_),
  189. request(request_),
  190. result(request_),
  191. oss(std::ios::binary)
  192. {
  193. curl = pool->alloc();
  194. if (curl == NULL) {
  195. return;
  196. }
  197. // Set static cURL options
  198. curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
  199. curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
  200. curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3);
  201. curl_easy_setopt(curl, CURLOPT_ENCODING, "gzip");
  202. std::string bind_address = g_settings->get("bind_address");
  203. if (!bind_address.empty()) {
  204. curl_easy_setopt(curl, CURLOPT_INTERFACE, bind_address.c_str());
  205. }
  206. if (!g_settings->getBool("enable_ipv6")) {
  207. curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
  208. }
  209. // Restrict protocols so that curl vulnerabilities in
  210. // other protocols don't affect us.
  211. #if LIBCURL_VERSION_NUM >= 0x075500
  212. // These settings were introduced in curl 7.85.0.
  213. const char *protocols = "HTTP,HTTPS,FTP,FTPS";
  214. curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, protocols);
  215. curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, protocols);
  216. #elif LIBCURL_VERSION_NUM >= 0x071304
  217. // These settings were introduced in curl 7.19.4, and later deprecated.
  218. long protocols =
  219. CURLPROTO_HTTP |
  220. CURLPROTO_HTTPS |
  221. CURLPROTO_FTP |
  222. CURLPROTO_FTPS;
  223. curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
  224. curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
  225. #endif
  226. // Set cURL options based on HTTPFetchRequest
  227. curl_easy_setopt(curl, CURLOPT_URL,
  228. request.url.c_str());
  229. curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
  230. request.timeout);
  231. curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
  232. request.connect_timeout);
  233. if (!request.useragent.empty())
  234. curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());
  235. // Set up a write callback that writes to the
  236. // ostringstream ongoing->oss, unless the data
  237. // is to be discarded
  238. if (request.caller == HTTPFETCH_DISCARD) {
  239. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
  240. httpfetch_discardfunction);
  241. curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
  242. } else {
  243. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
  244. httpfetch_writefunction);
  245. curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
  246. }
  247. // Set data from fields or raw_data
  248. if (request.multipart) {
  249. multipart_mime = curl_mime_init(curl);
  250. for (auto &it : request.fields) {
  251. curl_mimepart *part = curl_mime_addpart(multipart_mime);
  252. curl_mime_name(part, it.first.c_str());
  253. curl_mime_data(part, it.second.c_str(), it.second.size());
  254. }
  255. curl_easy_setopt(curl, CURLOPT_MIMEPOST, multipart_mime);
  256. } else {
  257. switch (request.method) {
  258. case HTTP_GET:
  259. curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
  260. break;
  261. case HTTP_POST:
  262. curl_easy_setopt(curl, CURLOPT_POST, 1);
  263. break;
  264. case HTTP_PUT:
  265. curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
  266. break;
  267. case HTTP_DELETE:
  268. curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
  269. break;
  270. }
  271. if (request.method != HTTP_GET) {
  272. if (!request.raw_data.empty()) {
  273. curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
  274. request.raw_data.size());
  275. curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
  276. request.raw_data.c_str());
  277. } else if (!request.fields.empty()) {
  278. std::string str;
  279. for (auto &field : request.fields) {
  280. if (!str.empty())
  281. str += "&";
  282. str += urlencode(field.first);
  283. str += "=";
  284. str += urlencode(field.second);
  285. }
  286. curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
  287. str.size());
  288. curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS,
  289. str.c_str());
  290. }
  291. }
  292. }
  293. // Set additional HTTP headers
  294. for (const std::string &extra_header : request.extra_headers) {
  295. http_header = curl_slist_append(http_header, extra_header.c_str());
  296. }
  297. curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_header);
  298. if (!g_settings->getBool("curl_verify_cert")) {
  299. curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
  300. }
  301. }
  302. CURLcode HTTPFetchOngoing::start(CURLM *multi_)
  303. {
  304. if (!curl)
  305. return CURLE_FAILED_INIT;
  306. if (!multi_) {
  307. // Easy interface (sync)
  308. return curl_easy_perform(curl);
  309. }
  310. // Multi interface (async)
  311. CURLMcode mres = curl_multi_add_handle(multi_, curl);
  312. if (mres != CURLM_OK) {
  313. errorstream << "curl_multi_add_handle"
  314. << " returned error code " << mres
  315. << std::endl;
  316. return CURLE_FAILED_INIT;
  317. }
  318. multi = multi_; // store for curl_multi_remove_handle
  319. return CURLE_OK;
  320. }
  321. const HTTPFetchResult * HTTPFetchOngoing::complete(CURLcode res)
  322. {
  323. result.succeeded = (res == CURLE_OK);
  324. result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
  325. result.data = oss.str();
  326. // Get HTTP/FTP response code
  327. result.response_code = 0;
  328. if (curl && (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
  329. &result.response_code) != CURLE_OK)) {
  330. // We failed to get a return code, make sure it is still 0
  331. result.response_code = 0;
  332. }
  333. if (res != CURLE_OK) {
  334. errorstream << "HTTPFetch for " << request.url << " failed: "
  335. << curl_easy_strerror(res);
  336. if (result.timeout)
  337. errorstream << " (timeout = " << request.timeout << "ms)" << std::endl;
  338. errorstream << std::endl;
  339. } else if (result.response_code >= 400) {
  340. errorstream << "HTTPFetch for " << request.url
  341. << " returned response code " << result.response_code
  342. << std::endl;
  343. if (result.caller == HTTPFETCH_PRINT_ERR && !result.data.empty()) {
  344. errorstream << "Response body:" << std::endl;
  345. safe_print_string(errorstream, result.data);
  346. errorstream << std::endl;
  347. }
  348. }
  349. return &result;
  350. }
  351. HTTPFetchOngoing::~HTTPFetchOngoing()
  352. {
  353. if (multi) {
  354. CURLMcode mres = curl_multi_remove_handle(multi, curl);
  355. if (mres != CURLM_OK) {
  356. errorstream << "curl_multi_remove_handle"
  357. << " returned error code " << mres
  358. << std::endl;
  359. }
  360. }
  361. // Set safe options for the reusable cURL handle
  362. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
  363. httpfetch_discardfunction);
  364. curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
  365. curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
  366. if (http_header) {
  367. curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
  368. curl_slist_free_all(http_header);
  369. }
  370. if (multipart_mime) {
  371. curl_easy_setopt(curl, CURLOPT_MIMEPOST, nullptr);
  372. curl_mime_free(multipart_mime);
  373. }
  374. // Store the cURL handle for reuse
  375. pool->free(curl);
  376. }
  377. #if LIBCURL_VERSION_NUM >= 0x074200
  378. #define HAVE_CURL_MULTI_POLL
  379. #else
  380. #undef HAVE_CURL_MULTI_POLL
  381. #endif
  382. class CurlFetchThread : public Thread
  383. {
  384. protected:
  385. enum RequestType {
  386. RT_FETCH,
  387. RT_CLEAR,
  388. RT_WAKEUP,
  389. };
  390. struct Request {
  391. RequestType type;
  392. HTTPFetchRequest fetch_request;
  393. Event *event = nullptr;
  394. };
  395. CURLM *m_multi;
  396. MutexedQueue<Request> m_requests;
  397. size_t m_parallel_limit;
  398. // Variables exclusively used within thread
  399. std::vector<std::unique_ptr<HTTPFetchOngoing>> m_all_ongoing;
  400. std::list<HTTPFetchRequest> m_queued_fetches;
  401. public:
  402. CurlFetchThread(int parallel_limit) :
  403. Thread("CurlFetch")
  404. {
  405. if (parallel_limit >= 1)
  406. m_parallel_limit = parallel_limit;
  407. else
  408. m_parallel_limit = 1;
  409. }
  410. void requestFetch(const HTTPFetchRequest &fetch_request)
  411. {
  412. Request req;
  413. req.type = RT_FETCH;
  414. req.fetch_request = fetch_request;
  415. m_requests.push_back(std::move(req));
  416. }
  417. void requestClear(u64 caller, Event *event)
  418. {
  419. Request req;
  420. req.type = RT_CLEAR;
  421. req.fetch_request.caller = caller;
  422. req.event = event;
  423. m_requests.push_back(std::move(req));
  424. }
  425. void requestWakeUp()
  426. {
  427. Request req;
  428. req.type = RT_WAKEUP;
  429. m_requests.push_back(std::move(req));
  430. }
  431. protected:
  432. // Handle a request from some other thread
  433. // E.g. new fetch; clear fetches for one caller; wake up
  434. void processRequest(Request &req)
  435. {
  436. if (req.type == RT_FETCH) {
  437. // New fetch, queue until there are less
  438. // than m_parallel_limit ongoing fetches
  439. m_queued_fetches.push_back(std::move(req.fetch_request));
  440. // see processQueued() for what happens next
  441. } else if (req.type == RT_CLEAR) {
  442. u64 caller = req.fetch_request.caller;
  443. // Abort all ongoing fetches for the caller
  444. for (auto it = m_all_ongoing.begin(); it != m_all_ongoing.end();) {
  445. if ((*it)->getRequest().caller == caller) {
  446. it = m_all_ongoing.erase(it);
  447. } else {
  448. ++it;
  449. }
  450. }
  451. // Also abort all queued fetches for the caller
  452. for (auto it = m_queued_fetches.begin();
  453. it != m_queued_fetches.end();) {
  454. if ((*it).caller == caller)
  455. it = m_queued_fetches.erase(it);
  456. else
  457. ++it;
  458. }
  459. } else if (req.type == RT_WAKEUP) {
  460. // Wakeup: Nothing to do, thread is awake at this point
  461. }
  462. if (req.event)
  463. req.event->signal();
  464. }
  465. // Start new ongoing fetches if m_parallel_limit allows
  466. void processQueued(CurlHandlePool *pool)
  467. {
  468. while (m_all_ongoing.size() < m_parallel_limit &&
  469. !m_queued_fetches.empty()) {
  470. HTTPFetchRequest request = std::move(m_queued_fetches.front());
  471. m_queued_fetches.pop_front();
  472. // Create ongoing fetch data and make a cURL handle
  473. // Set cURL options based on HTTPFetchRequest
  474. auto ongoing = std::make_unique<HTTPFetchOngoing>(request, pool);
  475. // Initiate the connection (curl_multi_add_handle)
  476. CURLcode res = ongoing->start(m_multi);
  477. if (res == CURLE_OK) {
  478. m_all_ongoing.push_back(std::move(ongoing));
  479. } else {
  480. httpfetch_deliver_result(*ongoing->complete(res));
  481. }
  482. }
  483. }
  484. // Process CURLMsg (indicates completion of a fetch)
  485. void processCurlMessage(CURLMsg *msg)
  486. {
  487. if (msg->msg != CURLMSG_DONE)
  488. return;
  489. // Determine which ongoing fetch the message pertains to
  490. for (auto it = m_all_ongoing.begin(); it != m_all_ongoing.end(); ++it) {
  491. auto &ongoing = **it;
  492. if (ongoing.getEasyHandle() != msg->easy_handle)
  493. continue;
  494. httpfetch_deliver_result(*ongoing.complete(msg->data.result));
  495. m_all_ongoing.erase(it);
  496. return;
  497. }
  498. }
  499. // Wait for a request from another thread, or timeout elapses
  500. void waitForRequest(long timeout)
  501. {
  502. if (m_queued_fetches.empty()) {
  503. try {
  504. Request req = m_requests.pop_front(timeout);
  505. processRequest(req);
  506. }
  507. catch (ItemNotFoundException &e) {}
  508. }
  509. }
  510. // Wait until some IO happens, or timeout elapses
  511. void waitForIO(long timeout)
  512. {
  513. CURLMcode mres;
  514. #ifdef HAVE_CURL_MULTI_POLL
  515. mres = curl_multi_poll(m_multi, nullptr, 0, timeout, nullptr);
  516. if (mres != CURLM_OK) {
  517. errorstream << "curl_multi_poll returned error code "
  518. << mres << std::endl;
  519. }
  520. #else
  521. // If there's nothing to do curl_multi_wait() will immediately return
  522. // so we have to emulate the sleeping.
  523. fd_set dummy;
  524. int max_fd;
  525. mres = curl_multi_fdset(m_multi, &dummy, &dummy, &dummy, &max_fd);
  526. if (mres != CURLM_OK) {
  527. errorstream << "curl_multi_fdset returned error code "
  528. << mres << std::endl;
  529. max_fd = -1;
  530. }
  531. if (max_fd == -1) { // curl has nothing to wait for
  532. if (timeout > 0)
  533. sleep_ms(timeout);
  534. } else {
  535. mres = curl_multi_wait(m_multi, nullptr, 0, timeout, nullptr);
  536. if (mres != CURLM_OK) {
  537. errorstream << "curl_multi_wait returned error code "
  538. << mres << std::endl;
  539. }
  540. }
  541. #endif
  542. }
  543. void *run()
  544. {
  545. CurlHandlePool pool;
  546. m_multi = curl_multi_init();
  547. FATAL_ERROR_IF(!m_multi, "curl_multi_init returned NULL");
  548. FATAL_ERROR_IF(!m_all_ongoing.empty(), "Expected empty");
  549. while (!stopRequested()) {
  550. BEGIN_DEBUG_EXCEPTION_HANDLER
  551. /*
  552. Handle new async requests
  553. */
  554. while (!m_requests.empty()) {
  555. Request req = m_requests.pop_frontNoEx();
  556. processRequest(req);
  557. }
  558. processQueued(&pool);
  559. /*
  560. Handle ongoing async requests
  561. */
  562. int still_ongoing = 0;
  563. while (curl_multi_perform(m_multi, &still_ongoing) ==
  564. CURLM_CALL_MULTI_PERFORM)
  565. /* noop */;
  566. /*
  567. Handle completed async requests
  568. */
  569. if (still_ongoing < (int) m_all_ongoing.size()) {
  570. CURLMsg *msg;
  571. int msgs_in_queue;
  572. msg = curl_multi_info_read(m_multi, &msgs_in_queue);
  573. while (msg != NULL) {
  574. processCurlMessage(msg);
  575. msg = curl_multi_info_read(m_multi, &msgs_in_queue);
  576. }
  577. }
  578. /*
  579. If there are ongoing requests, wait for data
  580. (with a timeout of 100ms so that new requests
  581. can be processed).
  582. If no ongoing requests, wait for a new request.
  583. (Possibly an empty request that signals
  584. that the thread should be stopped.)
  585. */
  586. if (m_all_ongoing.empty())
  587. waitForRequest(100000000);
  588. else
  589. waitForIO(100);
  590. END_DEBUG_EXCEPTION_HANDLER
  591. }
  592. // Call curl_multi_remove_handle and cleanup easy handles
  593. m_all_ongoing.clear();
  594. m_queued_fetches.clear();
  595. CURLMcode mres = curl_multi_cleanup(m_multi);
  596. if (mres != CURLM_OK) {
  597. errorstream<<"curl_multi_cleanup"
  598. <<" returned error code "<<mres
  599. <<std::endl;
  600. }
  601. return NULL;
  602. }
  603. };
  604. static std::unique_ptr<CurlFetchThread> g_httpfetch_thread;
  605. void httpfetch_init(int parallel_limit)
  606. {
  607. FATAL_ERROR_IF(g_httpfetch_thread, "httpfetch_init called twice");
  608. verbosestream<<"httpfetch_init: parallel_limit="<<parallel_limit
  609. <<std::endl;
  610. CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT);
  611. FATAL_ERROR_IF(res != CURLE_OK, "cURL init failed");
  612. g_httpfetch_thread = std::make_unique<CurlFetchThread>(parallel_limit);
  613. // Initialize g_callerid_randomness for httpfetch_caller_alloc_secure
  614. u64 randbuf[2];
  615. porting::secure_rand_fill_buf(randbuf, sizeof(u64) * 2);
  616. g_callerid_randomness = PcgRandom(randbuf[0], randbuf[1]);
  617. }
  618. void httpfetch_cleanup()
  619. {
  620. verbosestream<<"httpfetch_cleanup: cleaning up"<<std::endl;
  621. if (g_httpfetch_thread) {
  622. g_httpfetch_thread->stop();
  623. g_httpfetch_thread->requestWakeUp();
  624. g_httpfetch_thread->wait();
  625. g_httpfetch_thread.reset();
  626. }
  627. curl_global_cleanup();
  628. }
  629. void httpfetch_async(const HTTPFetchRequest &fetch_request)
  630. {
  631. g_httpfetch_thread->requestFetch(fetch_request);
  632. if (!g_httpfetch_thread->isRunning())
  633. g_httpfetch_thread->start();
  634. }
  635. static void httpfetch_request_clear(u64 caller)
  636. {
  637. if (g_httpfetch_thread->isRunning()) {
  638. Event event;
  639. g_httpfetch_thread->requestClear(caller, &event);
  640. event.wait();
  641. } else {
  642. g_httpfetch_thread->requestClear(caller, nullptr);
  643. }
  644. }
  645. void httpfetch_sync(const HTTPFetchRequest &fetch_request,
  646. HTTPFetchResult &fetch_result)
  647. {
  648. // Create ongoing fetch data and make a cURL handle
  649. // Set cURL options based on HTTPFetchRequest
  650. CurlHandlePool pool;
  651. HTTPFetchOngoing ongoing(fetch_request, &pool);
  652. // Do the fetch (curl_easy_perform)
  653. CURLcode res = ongoing.start(nullptr);
  654. // Update fetch result
  655. fetch_result = *ongoing.complete(res);
  656. }
  657. #else // USE_CURL
  658. /*
  659. USE_CURL is off:
  660. Dummy httpfetch implementation that always returns an error.
  661. */
  662. void httpfetch_init(int parallel_limit)
  663. {
  664. }
  665. void httpfetch_cleanup()
  666. {
  667. }
  668. void httpfetch_async(const HTTPFetchRequest &fetch_request)
  669. {
  670. errorstream << "httpfetch_async: unable to fetch " << fetch_request.url
  671. << " because USE_CURL=0" << std::endl;
  672. HTTPFetchResult fetch_result(fetch_request); // sets succeeded = false etc.
  673. httpfetch_deliver_result(fetch_result);
  674. }
  675. static void httpfetch_request_clear(u64 caller)
  676. {
  677. }
  678. void httpfetch_sync(const HTTPFetchRequest &fetch_request,
  679. HTTPFetchResult &fetch_result)
  680. {
  681. errorstream << "httpfetch_sync: unable to fetch " << fetch_request.url
  682. << " because USE_CURL=0" << std::endl;
  683. fetch_result = HTTPFetchResult(fetch_request); // sets succeeded = false etc.
  684. }
  685. #endif // USE_CURL