crawler.c 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. /***************************************************************************
  2. * _ _ ____ _
  3. * Project ___| | | | _ \| |
  4. * / __| | | | |_) | |
  5. * | (__| |_| | _ <| |___
  6. * \___|\___/|_| \_\_____|
  7. *
  8. * Copyright (C) Jeroen Ooms <jeroenooms@gmail.com>
  9. *
  10. * This software is licensed as described in the file COPYING, which
  11. * you should have received as part of this distribution. The terms
  12. * are also available at https://curl.se/docs/copyright.html.
  13. *
  14. * You may opt to use, copy, modify, merge, publish, distribute and/or sell
  15. * copies of the Software, and permit persons to whom the Software is
  16. * furnished to do so, under the terms of the COPYING file.
  17. *
  18. * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  19. * KIND, either express or implied.
  20. *
  21. * SPDX-License-Identifier: curl
  22. *
  23. * To compile:
  24. * gcc crawler.c $(pkg-config --cflags --libs libxml-2.0 libcurl)
  25. *
  26. */
  27. /* <DESC>
  28. * Web crawler based on curl and libxml2 to stress-test curl with
  29. * hundreds of concurrent connections to various servers.
  30. * </DESC>
  31. */
  32. /* Parameters */
  33. int max_con = 200;
  34. int max_total = 20000;
  35. int max_requests = 500;
  36. int max_link_per_page = 5;
  37. int follow_relative_links = 0;
  38. char *start_page = "https://www.reuters.com";
  39. #include <libxml/HTMLparser.h>
  40. #include <libxml/xpath.h>
  41. #include <libxml/uri.h>
  42. #include <curl/curl.h>
  43. #include <stdlib.h>
  44. #include <string.h>
  45. #include <math.h>
  46. #include <signal.h>
  47. int pending_interrupt = 0;
  48. void sighandler(int dummy)
  49. {
  50. pending_interrupt = 1;
  51. }
  52. /* resizable buffer */
  53. typedef struct {
  54. char *buf;
  55. size_t size;
  56. } memory;
  57. size_t grow_buffer(void *contents, size_t sz, size_t nmemb, void *ctx)
  58. {
  59. size_t realsize = sz * nmemb;
  60. memory *mem = (memory*) ctx;
  61. char *ptr = realloc(mem->buf, mem->size + realsize);
  62. if(!ptr) {
  63. /* out of memory */
  64. printf("not enough memory (realloc returned NULL)\n");
  65. return 0;
  66. }
  67. mem->buf = ptr;
  68. memcpy(&(mem->buf[mem->size]), contents, realsize);
  69. mem->size += realsize;
  70. return realsize;
  71. }
  72. CURL *make_handle(char *url)
  73. {
  74. CURL *handle = curl_easy_init();
  75. /* Important: use HTTP2 over HTTPS */
  76. curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
  77. curl_easy_setopt(handle, CURLOPT_URL, url);
  78. /* buffer body */
  79. memory *mem = malloc(sizeof(memory));
  80. mem->size = 0;
  81. mem->buf = malloc(1);
  82. curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, grow_buffer);
  83. curl_easy_setopt(handle, CURLOPT_WRITEDATA, mem);
  84. curl_easy_setopt(handle, CURLOPT_PRIVATE, mem);
  85. /* For completeness */
  86. curl_easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "");
  87. curl_easy_setopt(handle, CURLOPT_TIMEOUT, 5L);
  88. curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);
  89. curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L);
  90. curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L);
  91. curl_easy_setopt(handle, CURLOPT_COOKIEFILE, "");
  92. curl_easy_setopt(handle, CURLOPT_FILETIME, 1L);
  93. curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");
  94. curl_easy_setopt(handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
  95. curl_easy_setopt(handle, CURLOPT_UNRESTRICTED_AUTH, 1L);
  96. curl_easy_setopt(handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
  97. curl_easy_setopt(handle, CURLOPT_EXPECT_100_TIMEOUT_MS, 0L);
  98. return handle;
  99. }
  100. /* HREF finder implemented in libxml2 but could be any HTML parser */
  101. size_t follow_links(CURLM *multi_handle, memory *mem, char *url)
  102. {
  103. int opts = HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | \
  104. HTML_PARSE_NOWARNING | HTML_PARSE_NONET;
  105. htmlDocPtr doc = htmlReadMemory(mem->buf, mem->size, url, NULL, opts);
  106. if(!doc)
  107. return 0;
  108. xmlChar *xpath = (xmlChar*) "//a/@href";
  109. xmlXPathContextPtr context = xmlXPathNewContext(doc);
  110. xmlXPathObjectPtr result = xmlXPathEvalExpression(xpath, context);
  111. xmlXPathFreeContext(context);
  112. if(!result)
  113. return 0;
  114. xmlNodeSetPtr nodeset = result->nodesetval;
  115. if(xmlXPathNodeSetIsEmpty(nodeset)) {
  116. xmlXPathFreeObject(result);
  117. return 0;
  118. }
  119. size_t count = 0;
  120. int i;
  121. for(i = 0; i < nodeset->nodeNr; i++) {
  122. double r = rand();
  123. int x = r * nodeset->nodeNr / RAND_MAX;
  124. const xmlNode *node = nodeset->nodeTab[x]->xmlChildrenNode;
  125. xmlChar *href = xmlNodeListGetString(doc, node, 1);
  126. if(follow_relative_links) {
  127. xmlChar *orig = href;
  128. href = xmlBuildURI(href, (xmlChar *) url);
  129. xmlFree(orig);
  130. }
  131. char *link = (char *) href;
  132. if(!link || strlen(link) < 20)
  133. continue;
  134. if(!strncmp(link, "http://", 7) || !strncmp(link, "https://", 8)) {
  135. curl_multi_add_handle(multi_handle, make_handle(link));
  136. if(count++ == max_link_per_page)
  137. break;
  138. }
  139. xmlFree(link);
  140. }
  141. xmlXPathFreeObject(result);
  142. return count;
  143. }
  144. int is_html(char *ctype)
  145. {
  146. return ctype != NULL && strlen(ctype) > 10 && strstr(ctype, "text/html");
  147. }
  148. int main(void)
  149. {
  150. signal(SIGINT, sighandler);
  151. LIBXML_TEST_VERSION;
  152. curl_global_init(CURL_GLOBAL_DEFAULT);
  153. CURLM *multi_handle = curl_multi_init();
  154. curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con);
  155. curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 6L);
  156. /* enables http/2 if available */
  157. #ifdef CURLPIPE_MULTIPLEX
  158. curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
  159. #endif
  160. /* sets html start page */
  161. curl_multi_add_handle(multi_handle, make_handle(start_page));
  162. int msgs_left;
  163. int pending = 0;
  164. int complete = 0;
  165. int still_running = 1;
  166. while(still_running && !pending_interrupt) {
  167. int numfds;
  168. curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
  169. curl_multi_perform(multi_handle, &still_running);
  170. /* See how the transfers went */
  171. CURLMsg *m = NULL;
  172. while((m = curl_multi_info_read(multi_handle, &msgs_left))) {
  173. if(m->msg == CURLMSG_DONE) {
  174. CURL *handle = m->easy_handle;
  175. char *url;
  176. memory *mem;
  177. curl_easy_getinfo(handle, CURLINFO_PRIVATE, &mem);
  178. curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);
  179. if(m->data.result == CURLE_OK) {
  180. long res_status;
  181. curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &res_status);
  182. if(res_status == 200) {
  183. char *ctype;
  184. curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &ctype);
  185. printf("[%d] HTTP 200 (%s): %s\n", complete, ctype, url);
  186. if(is_html(ctype) && mem->size > 100) {
  187. if(pending < max_requests && (complete + pending) < max_total) {
  188. pending += follow_links(multi_handle, mem, url);
  189. still_running = 1;
  190. }
  191. }
  192. }
  193. else {
  194. printf("[%d] HTTP %d: %s\n", complete, (int) res_status, url);
  195. }
  196. }
  197. else {
  198. printf("[%d] Connection failure: %s\n", complete, url);
  199. }
  200. curl_multi_remove_handle(multi_handle, handle);
  201. curl_easy_cleanup(handle);
  202. free(mem->buf);
  203. free(mem);
  204. complete++;
  205. pending--;
  206. }
  207. }
  208. }
  209. curl_multi_cleanup(multi_handle);
  210. curl_global_cleanup();
  211. return 0;
  212. }