content_encoding.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086
  1. /***************************************************************************
  2. * _ _ ____ _
  3. * Project ___| | | | _ \| |
  4. * / __| | | | |_) | |
  5. * | (__| |_| | _ <| |___
  6. * \___|\___/|_| \_\_____|
  7. *
  8. * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  9. *
  10. * This software is licensed as described in the file COPYING, which
  11. * you should have received as part of this distribution. The terms
  12. * are also available at https://curl.se/docs/copyright.html.
  13. *
  14. * You may opt to use, copy, modify, merge, publish, distribute and/or sell
  15. * copies of the Software, and permit persons to whom the Software is
  16. * furnished to do so, under the terms of the COPYING file.
  17. *
  18. * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  19. * KIND, either express or implied.
  20. *
  21. * SPDX-License-Identifier: curl
  22. *
  23. ***************************************************************************/
  24. #include "curl_setup.h"
  25. #include "urldata.h"
  26. #include <curl/curl.h>
  27. #include <stddef.h>
  28. #ifdef HAVE_LIBZ
  29. #include <zlib.h>
  30. #endif
  31. #ifdef HAVE_BROTLI
  32. #if defined(__GNUC__)
  33. /* Ignore -Wvla warnings in brotli headers */
  34. #pragma GCC diagnostic push
  35. #pragma GCC diagnostic ignored "-Wvla"
  36. #endif
  37. #include <brotli/decode.h>
  38. #if defined(__GNUC__)
  39. #pragma GCC diagnostic pop
  40. #endif
  41. #endif
  42. #ifdef HAVE_ZSTD
  43. #include <zstd.h>
  44. #endif
  45. #include "sendf.h"
  46. #include "http.h"
  47. #include "content_encoding.h"
  48. #include "strdup.h"
  49. #include "strcase.h"
  50. /* The last 3 #include files should be in this order */
  51. #include "curl_printf.h"
  52. #include "curl_memory.h"
  53. #include "memdebug.h"
  54. #define CONTENT_ENCODING_DEFAULT "identity"
  55. #ifndef CURL_DISABLE_HTTP
  56. /* allow no more than 5 "chained" compression steps */
  57. #define MAX_ENCODE_STACK 5
  58. #define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
  59. #ifdef HAVE_LIBZ
  60. /* Comment this out if zlib is always going to be at least ver. 1.2.0.4
  61. (doing so will reduce code size slightly). */
  62. #define OLD_ZLIB_SUPPORT 1
  63. #define GZIP_MAGIC_0 0x1f
  64. #define GZIP_MAGIC_1 0x8b
  65. /* gzip flag byte */
  66. #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
  67. #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
  68. #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
  69. #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
  70. #define COMMENT 0x10 /* bit 4 set: file comment present */
  71. #define RESERVED 0xE0 /* bits 5..7: reserved */
  72. typedef enum {
  73. ZLIB_UNINIT, /* uninitialized */
  74. ZLIB_INIT, /* initialized */
  75. ZLIB_INFLATING, /* inflating started. */
  76. ZLIB_EXTERNAL_TRAILER, /* reading external trailer */
  77. ZLIB_GZIP_HEADER, /* reading gzip header */
  78. ZLIB_GZIP_INFLATING, /* inflating gzip stream */
  79. ZLIB_INIT_GZIP /* initialized in transparent gzip mode */
  80. } zlibInitState;
  81. /* Deflate and gzip writer. */
  82. struct zlib_writer {
  83. struct Curl_cwriter super;
  84. zlibInitState zlib_init; /* zlib init state */
  85. uInt trailerlen; /* Remaining trailer byte count. */
  86. z_stream z; /* State structure for zlib. */
  87. };
  88. static voidpf
  89. zalloc_cb(voidpf opaque, unsigned int items, unsigned int size)
  90. {
  91. (void) opaque;
  92. /* not a typo, keep it calloc() */
  93. return (voidpf) calloc(items, size);
  94. }
  95. static void
  96. zfree_cb(voidpf opaque, voidpf ptr)
  97. {
  98. (void) opaque;
  99. free(ptr);
  100. }
  101. static CURLcode
  102. process_zlib_error(struct Curl_easy *data, z_stream *z)
  103. {
  104. if(z->msg)
  105. failf(data, "Error while processing content unencoding: %s",
  106. z->msg);
  107. else
  108. failf(data, "Error while processing content unencoding: "
  109. "Unknown failure within decompression software.");
  110. return CURLE_BAD_CONTENT_ENCODING;
  111. }
  112. static CURLcode
  113. exit_zlib(struct Curl_easy *data,
  114. z_stream *z, zlibInitState *zlib_init, CURLcode result)
  115. {
  116. if(*zlib_init == ZLIB_GZIP_HEADER)
  117. Curl_safefree(z->next_in);
  118. if(*zlib_init != ZLIB_UNINIT) {
  119. if(inflateEnd(z) != Z_OK && result == CURLE_OK)
  120. result = process_zlib_error(data, z);
  121. *zlib_init = ZLIB_UNINIT;
  122. }
  123. return result;
  124. }
  125. static CURLcode process_trailer(struct Curl_easy *data,
  126. struct zlib_writer *zp)
  127. {
  128. z_stream *z = &zp->z;
  129. CURLcode result = CURLE_OK;
  130. uInt len = z->avail_in < zp->trailerlen? z->avail_in: zp->trailerlen;
  131. /* Consume expected trailer bytes. Terminate stream if exhausted.
  132. Issue an error if unexpected bytes follow. */
  133. zp->trailerlen -= len;
  134. z->avail_in -= len;
  135. z->next_in += len;
  136. if(z->avail_in)
  137. result = CURLE_WRITE_ERROR;
  138. if(result || !zp->trailerlen)
  139. result = exit_zlib(data, z, &zp->zlib_init, result);
  140. else {
  141. /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */
  142. zp->zlib_init = ZLIB_EXTERNAL_TRAILER;
  143. }
  144. return result;
  145. }
  146. static CURLcode inflate_stream(struct Curl_easy *data,
  147. struct Curl_cwriter *writer, int type,
  148. zlibInitState started)
  149. {
  150. struct zlib_writer *zp = (struct zlib_writer *) writer;
  151. z_stream *z = &zp->z; /* zlib state structure */
  152. uInt nread = z->avail_in;
  153. Bytef *orig_in = z->next_in;
  154. bool done = FALSE;
  155. CURLcode result = CURLE_OK; /* Curl_client_write status */
  156. char *decomp; /* Put the decompressed data here. */
  157. /* Check state. */
  158. if(zp->zlib_init != ZLIB_INIT &&
  159. zp->zlib_init != ZLIB_INFLATING &&
  160. zp->zlib_init != ZLIB_INIT_GZIP &&
  161. zp->zlib_init != ZLIB_GZIP_INFLATING)
  162. return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
  163. /* Dynamically allocate a buffer for decompression because it's uncommonly
  164. large to hold on the stack */
  165. decomp = malloc(DSIZ);
  166. if(!decomp)
  167. return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);
  168. /* because the buffer size is fixed, iteratively decompress and transfer to
  169. the client via next_write function. */
  170. while(!done) {
  171. int status; /* zlib status */
  172. done = TRUE;
  173. /* (re)set buffer for decompressed output for every iteration */
  174. z->next_out = (Bytef *) decomp;
  175. z->avail_out = DSIZ;
  176. #ifdef Z_BLOCK
  177. /* Z_BLOCK is only available in zlib ver. >= 1.2.0.5 */
  178. status = inflate(z, Z_BLOCK);
  179. #else
  180. /* fallback for zlib ver. < 1.2.0.5 */
  181. status = inflate(z, Z_SYNC_FLUSH);
  182. #endif
  183. /* Flush output data if some. */
  184. if(z->avail_out != DSIZ) {
  185. if(status == Z_OK || status == Z_STREAM_END) {
  186. zp->zlib_init = started; /* Data started. */
  187. result = Curl_cwriter_write(data, writer->next, type, decomp,
  188. DSIZ - z->avail_out);
  189. if(result) {
  190. exit_zlib(data, z, &zp->zlib_init, result);
  191. break;
  192. }
  193. }
  194. }
  195. /* Dispatch by inflate() status. */
  196. switch(status) {
  197. case Z_OK:
  198. /* Always loop: there may be unflushed latched data in zlib state. */
  199. done = FALSE;
  200. break;
  201. case Z_BUF_ERROR:
  202. /* No more data to flush: just exit loop. */
  203. break;
  204. case Z_STREAM_END:
  205. result = process_trailer(data, zp);
  206. break;
  207. case Z_DATA_ERROR:
  208. /* some servers seem to not generate zlib headers, so this is an attempt
  209. to fix and continue anyway */
  210. if(zp->zlib_init == ZLIB_INIT) {
  211. /* Do not use inflateReset2(): only available since zlib 1.2.3.4. */
  212. (void) inflateEnd(z); /* don't care about the return code */
  213. if(inflateInit2(z, -MAX_WBITS) == Z_OK) {
  214. z->next_in = orig_in;
  215. z->avail_in = nread;
  216. zp->zlib_init = ZLIB_INFLATING;
  217. zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */
  218. done = FALSE;
  219. break;
  220. }
  221. zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */
  222. }
  223. result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  224. break;
  225. default:
  226. result = exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  227. break;
  228. }
  229. }
  230. free(decomp);
  231. /* We're about to leave this call so the `nread' data bytes won't be seen
  232. again. If we are in a state that would wrongly allow restart in raw mode
  233. at the next call, assume output has already started. */
  234. if(nread && zp->zlib_init == ZLIB_INIT)
  235. zp->zlib_init = started; /* Cannot restart anymore. */
  236. return result;
  237. }
  238. /* Deflate handler. */
  239. static CURLcode deflate_do_init(struct Curl_easy *data,
  240. struct Curl_cwriter *writer)
  241. {
  242. struct zlib_writer *zp = (struct zlib_writer *) writer;
  243. z_stream *z = &zp->z; /* zlib state structure */
  244. /* Initialize zlib */
  245. z->zalloc = (alloc_func) zalloc_cb;
  246. z->zfree = (free_func) zfree_cb;
  247. if(inflateInit(z) != Z_OK)
  248. return process_zlib_error(data, z);
  249. zp->zlib_init = ZLIB_INIT;
  250. return CURLE_OK;
  251. }
  252. static CURLcode deflate_do_write(struct Curl_easy *data,
  253. struct Curl_cwriter *writer, int type,
  254. const char *buf, size_t nbytes)
  255. {
  256. struct zlib_writer *zp = (struct zlib_writer *) writer;
  257. z_stream *z = &zp->z; /* zlib state structure */
  258. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  259. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  260. /* Set the compressed input when this function is called */
  261. z->next_in = (Bytef *) buf;
  262. z->avail_in = (uInt) nbytes;
  263. if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER)
  264. return process_trailer(data, zp);
  265. /* Now uncompress the data */
  266. return inflate_stream(data, writer, type, ZLIB_INFLATING);
  267. }
  268. static void deflate_do_close(struct Curl_easy *data,
  269. struct Curl_cwriter *writer)
  270. {
  271. struct zlib_writer *zp = (struct zlib_writer *) writer;
  272. z_stream *z = &zp->z; /* zlib state structure */
  273. exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
  274. }
  275. static const struct Curl_cwtype deflate_encoding = {
  276. "deflate",
  277. NULL,
  278. deflate_do_init,
  279. deflate_do_write,
  280. deflate_do_close,
  281. sizeof(struct zlib_writer)
  282. };
  283. /* Gzip handler. */
  284. static CURLcode gzip_do_init(struct Curl_easy *data,
  285. struct Curl_cwriter *writer)
  286. {
  287. struct zlib_writer *zp = (struct zlib_writer *) writer;
  288. z_stream *z = &zp->z; /* zlib state structure */
  289. /* Initialize zlib */
  290. z->zalloc = (alloc_func) zalloc_cb;
  291. z->zfree = (free_func) zfree_cb;
  292. if(strcmp(zlibVersion(), "1.2.0.4") >= 0) {
  293. /* zlib ver. >= 1.2.0.4 supports transparent gzip decompressing */
  294. if(inflateInit2(z, MAX_WBITS + 32) != Z_OK) {
  295. return process_zlib_error(data, z);
  296. }
  297. zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */
  298. }
  299. else {
  300. /* we must parse the gzip header and trailer ourselves */
  301. if(inflateInit2(z, -MAX_WBITS) != Z_OK) {
  302. return process_zlib_error(data, z);
  303. }
  304. zp->trailerlen = 8; /* A CRC-32 and a 32-bit input size (RFC 1952, 2.2) */
  305. zp->zlib_init = ZLIB_INIT; /* Initial call state */
  306. }
  307. return CURLE_OK;
  308. }
  309. #ifdef OLD_ZLIB_SUPPORT
  310. /* Skip over the gzip header */
  311. typedef enum {
  312. GZIP_OK,
  313. GZIP_BAD,
  314. GZIP_UNDERFLOW
  315. } gzip_status;
  316. static gzip_status check_gzip_header(unsigned char const *data, ssize_t len,
  317. ssize_t *headerlen)
  318. {
  319. int method, flags;
  320. const ssize_t totallen = len;
  321. /* The shortest header is 10 bytes */
  322. if(len < 10)
  323. return GZIP_UNDERFLOW;
  324. if((data[0] != GZIP_MAGIC_0) || (data[1] != GZIP_MAGIC_1))
  325. return GZIP_BAD;
  326. method = data[2];
  327. flags = data[3];
  328. if(method != Z_DEFLATED || (flags & RESERVED) != 0) {
  329. /* Can't handle this compression method or unknown flag */
  330. return GZIP_BAD;
  331. }
  332. /* Skip over time, xflags, OS code and all previous bytes */
  333. len -= 10;
  334. data += 10;
  335. if(flags & EXTRA_FIELD) {
  336. ssize_t extra_len;
  337. if(len < 2)
  338. return GZIP_UNDERFLOW;
  339. extra_len = (data[1] << 8) | data[0];
  340. if(len < (extra_len + 2))
  341. return GZIP_UNDERFLOW;
  342. len -= (extra_len + 2);
  343. data += (extra_len + 2);
  344. }
  345. if(flags & ORIG_NAME) {
  346. /* Skip over NUL-terminated file name */
  347. while(len && *data) {
  348. --len;
  349. ++data;
  350. }
  351. if(!len || *data)
  352. return GZIP_UNDERFLOW;
  353. /* Skip over the NUL */
  354. --len;
  355. ++data;
  356. }
  357. if(flags & COMMENT) {
  358. /* Skip over NUL-terminated comment */
  359. while(len && *data) {
  360. --len;
  361. ++data;
  362. }
  363. if(!len || *data)
  364. return GZIP_UNDERFLOW;
  365. /* Skip over the NUL */
  366. --len;
  367. }
  368. if(flags & HEAD_CRC) {
  369. if(len < 2)
  370. return GZIP_UNDERFLOW;
  371. len -= 2;
  372. }
  373. *headerlen = totallen - len;
  374. return GZIP_OK;
  375. }
  376. #endif
  377. static CURLcode gzip_do_write(struct Curl_easy *data,
  378. struct Curl_cwriter *writer, int type,
  379. const char *buf, size_t nbytes)
  380. {
  381. struct zlib_writer *zp = (struct zlib_writer *) writer;
  382. z_stream *z = &zp->z; /* zlib state structure */
  383. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  384. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  385. if(zp->zlib_init == ZLIB_INIT_GZIP) {
  386. /* Let zlib handle the gzip decompression entirely */
  387. z->next_in = (Bytef *) buf;
  388. z->avail_in = (uInt) nbytes;
  389. /* Now uncompress the data */
  390. return inflate_stream(data, writer, type, ZLIB_INIT_GZIP);
  391. }
  392. #ifndef OLD_ZLIB_SUPPORT
  393. /* Support for old zlib versions is compiled away and we are running with
  394. an old version, so return an error. */
  395. return exit_zlib(data, z, &zp->zlib_init, CURLE_WRITE_ERROR);
  396. #else
  397. /* This next mess is to get around the potential case where there isn't
  398. * enough data passed in to skip over the gzip header. If that happens, we
  399. * malloc a block and copy what we have then wait for the next call. If
  400. * there still isn't enough (this is definitely a worst-case scenario), we
  401. * make the block bigger, copy the next part in and keep waiting.
  402. *
  403. * This is only required with zlib versions < 1.2.0.4 as newer versions
  404. * can handle the gzip header themselves.
  405. */
  406. switch(zp->zlib_init) {
  407. /* Skip over gzip header? */
  408. case ZLIB_INIT:
  409. {
  410. /* Initial call state */
  411. ssize_t hlen;
  412. switch(check_gzip_header((unsigned char *) buf, nbytes, &hlen)) {
  413. case GZIP_OK:
  414. z->next_in = (Bytef *) buf + hlen;
  415. z->avail_in = (uInt) (nbytes - hlen);
  416. zp->zlib_init = ZLIB_GZIP_INFLATING; /* Inflating stream state */
  417. break;
  418. case GZIP_UNDERFLOW:
  419. /* We need more data so we can find the end of the gzip header. It's
  420. * possible that the memory block we malloc here will never be freed if
  421. * the transfer abruptly aborts after this point. Since it's unlikely
  422. * that circumstances will be right for this code path to be followed in
  423. * the first place, and it's even more unlikely for a transfer to fail
  424. * immediately afterwards, it should seldom be a problem.
  425. */
  426. z->avail_in = (uInt) nbytes;
  427. z->next_in = malloc(z->avail_in);
  428. if(!z->next_in) {
  429. return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);
  430. }
  431. memcpy(z->next_in, buf, z->avail_in);
  432. zp->zlib_init = ZLIB_GZIP_HEADER; /* Need more gzip header data state */
  433. /* We don't have any data to inflate yet */
  434. return CURLE_OK;
  435. case GZIP_BAD:
  436. default:
  437. return exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  438. }
  439. }
  440. break;
  441. case ZLIB_GZIP_HEADER:
  442. {
  443. /* Need more gzip header data state */
  444. ssize_t hlen;
  445. z->avail_in += (uInt) nbytes;
  446. z->next_in = Curl_saferealloc(z->next_in, z->avail_in);
  447. if(!z->next_in) {
  448. return exit_zlib(data, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY);
  449. }
  450. /* Append the new block of data to the previous one */
  451. memcpy(z->next_in + z->avail_in - nbytes, buf, nbytes);
  452. switch(check_gzip_header(z->next_in, (ssize_t)z->avail_in, &hlen)) {
  453. case GZIP_OK:
  454. /* This is the zlib stream data */
  455. free(z->next_in);
  456. /* Don't point into the malloced block since we just freed it */
  457. z->next_in = (Bytef *) buf + hlen + nbytes - z->avail_in;
  458. z->avail_in = z->avail_in - (uInt)hlen;
  459. zp->zlib_init = ZLIB_GZIP_INFLATING; /* Inflating stream state */
  460. break;
  461. case GZIP_UNDERFLOW:
  462. /* We still don't have any data to inflate! */
  463. return CURLE_OK;
  464. case GZIP_BAD:
  465. default:
  466. return exit_zlib(data, z, &zp->zlib_init, process_zlib_error(data, z));
  467. }
  468. }
  469. break;
  470. case ZLIB_EXTERNAL_TRAILER:
  471. z->next_in = (Bytef *) buf;
  472. z->avail_in = (uInt) nbytes;
  473. return process_trailer(data, zp);
  474. case ZLIB_GZIP_INFLATING:
  475. default:
  476. /* Inflating stream state */
  477. z->next_in = (Bytef *) buf;
  478. z->avail_in = (uInt) nbytes;
  479. break;
  480. }
  481. if(z->avail_in == 0) {
  482. /* We don't have any data to inflate; wait until next time */
  483. return CURLE_OK;
  484. }
  485. /* We've parsed the header, now uncompress the data */
  486. return inflate_stream(data, writer, type, ZLIB_GZIP_INFLATING);
  487. #endif
  488. }
  489. static void gzip_do_close(struct Curl_easy *data,
  490. struct Curl_cwriter *writer)
  491. {
  492. struct zlib_writer *zp = (struct zlib_writer *) writer;
  493. z_stream *z = &zp->z; /* zlib state structure */
  494. exit_zlib(data, z, &zp->zlib_init, CURLE_OK);
  495. }
  496. static const struct Curl_cwtype gzip_encoding = {
  497. "gzip",
  498. "x-gzip",
  499. gzip_do_init,
  500. gzip_do_write,
  501. gzip_do_close,
  502. sizeof(struct zlib_writer)
  503. };
  504. #endif /* HAVE_LIBZ */
  505. #ifdef HAVE_BROTLI
  506. /* Brotli writer. */
  507. struct brotli_writer {
  508. struct Curl_cwriter super;
  509. BrotliDecoderState *br; /* State structure for brotli. */
  510. };
  511. static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
  512. {
  513. switch(be) {
  514. case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
  515. case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
  516. case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
  517. case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
  518. case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
  519. case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
  520. case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
  521. case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
  522. case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
  523. case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
  524. case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
  525. case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
  526. case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
  527. case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
  528. #ifdef BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY
  529. case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
  530. #endif
  531. #ifdef BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET
  532. case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
  533. #endif
  534. case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
  535. return CURLE_BAD_CONTENT_ENCODING;
  536. case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
  537. case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
  538. case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
  539. case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
  540. case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
  541. case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
  542. return CURLE_OUT_OF_MEMORY;
  543. default:
  544. break;
  545. }
  546. return CURLE_WRITE_ERROR;
  547. }
  548. static CURLcode brotli_do_init(struct Curl_easy *data,
  549. struct Curl_cwriter *writer)
  550. {
  551. struct brotli_writer *bp = (struct brotli_writer *) writer;
  552. (void) data;
  553. bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
  554. return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
  555. }
  556. static CURLcode brotli_do_write(struct Curl_easy *data,
  557. struct Curl_cwriter *writer, int type,
  558. const char *buf, size_t nbytes)
  559. {
  560. struct brotli_writer *bp = (struct brotli_writer *) writer;
  561. const uint8_t *src = (const uint8_t *) buf;
  562. char *decomp;
  563. uint8_t *dst;
  564. size_t dstleft;
  565. CURLcode result = CURLE_OK;
  566. BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
  567. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  568. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  569. if(!bp->br)
  570. return CURLE_WRITE_ERROR; /* Stream already ended. */
  571. decomp = malloc(DSIZ);
  572. if(!decomp)
  573. return CURLE_OUT_OF_MEMORY;
  574. while((nbytes || r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) &&
  575. result == CURLE_OK) {
  576. dst = (uint8_t *) decomp;
  577. dstleft = DSIZ;
  578. r = BrotliDecoderDecompressStream(bp->br,
  579. &nbytes, &src, &dstleft, &dst, NULL);
  580. result = Curl_cwriter_write(data, writer->next, type,
  581. decomp, DSIZ - dstleft);
  582. if(result)
  583. break;
  584. switch(r) {
  585. case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
  586. case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
  587. break;
  588. case BROTLI_DECODER_RESULT_SUCCESS:
  589. BrotliDecoderDestroyInstance(bp->br);
  590. bp->br = NULL;
  591. if(nbytes)
  592. result = CURLE_WRITE_ERROR;
  593. break;
  594. default:
  595. result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
  596. break;
  597. }
  598. }
  599. free(decomp);
  600. return result;
  601. }
  602. static void brotli_do_close(struct Curl_easy *data,
  603. struct Curl_cwriter *writer)
  604. {
  605. struct brotli_writer *bp = (struct brotli_writer *) writer;
  606. (void) data;
  607. if(bp->br) {
  608. BrotliDecoderDestroyInstance(bp->br);
  609. bp->br = NULL;
  610. }
  611. }
  612. static const struct Curl_cwtype brotli_encoding = {
  613. "br",
  614. NULL,
  615. brotli_do_init,
  616. brotli_do_write,
  617. brotli_do_close,
  618. sizeof(struct brotli_writer)
  619. };
  620. #endif
  621. #ifdef HAVE_ZSTD
  622. /* Zstd writer. */
  623. struct zstd_writer {
  624. struct Curl_cwriter super;
  625. ZSTD_DStream *zds; /* State structure for zstd. */
  626. void *decomp;
  627. };
  628. static CURLcode zstd_do_init(struct Curl_easy *data,
  629. struct Curl_cwriter *writer)
  630. {
  631. struct zstd_writer *zp = (struct zstd_writer *) writer;
  632. (void)data;
  633. zp->zds = ZSTD_createDStream();
  634. zp->decomp = NULL;
  635. return zp->zds ? CURLE_OK : CURLE_OUT_OF_MEMORY;
  636. }
  637. static CURLcode zstd_do_write(struct Curl_easy *data,
  638. struct Curl_cwriter *writer, int type,
  639. const char *buf, size_t nbytes)
  640. {
  641. CURLcode result = CURLE_OK;
  642. struct zstd_writer *zp = (struct zstd_writer *) writer;
  643. ZSTD_inBuffer in;
  644. ZSTD_outBuffer out;
  645. size_t errorCode;
  646. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  647. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  648. if(!zp->decomp) {
  649. zp->decomp = malloc(DSIZ);
  650. if(!zp->decomp)
  651. return CURLE_OUT_OF_MEMORY;
  652. }
  653. in.pos = 0;
  654. in.src = buf;
  655. in.size = nbytes;
  656. for(;;) {
  657. out.pos = 0;
  658. out.dst = zp->decomp;
  659. out.size = DSIZ;
  660. errorCode = ZSTD_decompressStream(zp->zds, &out, &in);
  661. if(ZSTD_isError(errorCode)) {
  662. return CURLE_BAD_CONTENT_ENCODING;
  663. }
  664. if(out.pos > 0) {
  665. result = Curl_cwriter_write(data, writer->next, type,
  666. zp->decomp, out.pos);
  667. if(result)
  668. break;
  669. }
  670. if((in.pos == nbytes) && (out.pos < out.size))
  671. break;
  672. }
  673. return result;
  674. }
  675. static void zstd_do_close(struct Curl_easy *data,
  676. struct Curl_cwriter *writer)
  677. {
  678. struct zstd_writer *zp = (struct zstd_writer *) writer;
  679. (void)data;
  680. if(zp->decomp) {
  681. free(zp->decomp);
  682. zp->decomp = NULL;
  683. }
  684. if(zp->zds) {
  685. ZSTD_freeDStream(zp->zds);
  686. zp->zds = NULL;
  687. }
  688. }
  689. static const struct Curl_cwtype zstd_encoding = {
  690. "zstd",
  691. NULL,
  692. zstd_do_init,
  693. zstd_do_write,
  694. zstd_do_close,
  695. sizeof(struct zstd_writer)
  696. };
  697. #endif
  698. /* Identity handler. */
  699. static const struct Curl_cwtype identity_encoding = {
  700. "identity",
  701. "none",
  702. Curl_cwriter_def_init,
  703. Curl_cwriter_def_write,
  704. Curl_cwriter_def_close,
  705. sizeof(struct Curl_cwriter)
  706. };
  707. /* supported general content decoders. */
  708. static const struct Curl_cwtype * const general_unencoders[] = {
  709. &identity_encoding,
  710. #ifdef HAVE_LIBZ
  711. &deflate_encoding,
  712. &gzip_encoding,
  713. #endif
  714. #ifdef HAVE_BROTLI
  715. &brotli_encoding,
  716. #endif
  717. #ifdef HAVE_ZSTD
  718. &zstd_encoding,
  719. #endif
  720. NULL
  721. };
  722. /* supported content decoders only for transfer encodings */
  723. static const struct Curl_cwtype * const transfer_unencoders[] = {
  724. #ifndef CURL_DISABLE_HTTP
  725. &Curl_httpchunk_unencoder,
  726. #endif
  727. NULL
  728. };
  729. /* Provide a list of comma-separated names of supported encodings.
  730. */
  731. void Curl_all_content_encodings(char *buf, size_t blen)
  732. {
  733. size_t len = 0;
  734. const struct Curl_cwtype * const *cep;
  735. const struct Curl_cwtype *ce;
  736. DEBUGASSERT(buf);
  737. DEBUGASSERT(blen);
  738. buf[0] = 0;
  739. for(cep = general_unencoders; *cep; cep++) {
  740. ce = *cep;
  741. if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT))
  742. len += strlen(ce->name) + 2;
  743. }
  744. if(!len) {
  745. if(blen >= sizeof(CONTENT_ENCODING_DEFAULT))
  746. strcpy(buf, CONTENT_ENCODING_DEFAULT);
  747. }
  748. else if(blen > len) {
  749. char *p = buf;
  750. for(cep = general_unencoders; *cep; cep++) {
  751. ce = *cep;
  752. if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT)) {
  753. strcpy(p, ce->name);
  754. p += strlen(p);
  755. *p++ = ',';
  756. *p++ = ' ';
  757. }
  758. }
  759. p[-2] = '\0';
  760. }
  761. }
  762. /* Deferred error dummy writer. */
  763. static CURLcode error_do_init(struct Curl_easy *data,
  764. struct Curl_cwriter *writer)
  765. {
  766. (void)data;
  767. (void)writer;
  768. return CURLE_OK;
  769. }
  770. static CURLcode error_do_write(struct Curl_easy *data,
  771. struct Curl_cwriter *writer, int type,
  772. const char *buf, size_t nbytes)
  773. {
  774. char all[256];
  775. (void)Curl_all_content_encodings(all, sizeof(all));
  776. (void) writer;
  777. (void) buf;
  778. (void) nbytes;
  779. if(!(type & CLIENTWRITE_BODY) || !nbytes)
  780. return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
  781. failf(data, "Unrecognized content encoding type. "
  782. "libcurl understands %s content encodings.", all);
  783. return CURLE_BAD_CONTENT_ENCODING;
  784. }
  785. static void error_do_close(struct Curl_easy *data,
  786. struct Curl_cwriter *writer)
  787. {
  788. (void) data;
  789. (void) writer;
  790. }
  791. static const struct Curl_cwtype error_writer = {
  792. "ce-error",
  793. NULL,
  794. error_do_init,
  795. error_do_write,
  796. error_do_close,
  797. sizeof(struct Curl_cwriter)
  798. };
  799. /* Find the content encoding by name. */
  800. static const struct Curl_cwtype *find_unencode_writer(const char *name,
  801. size_t len,
  802. Curl_cwriter_phase phase)
  803. {
  804. const struct Curl_cwtype * const *cep;
  805. if(phase == CURL_CW_TRANSFER_DECODE) {
  806. for(cep = transfer_unencoders; *cep; cep++) {
  807. const struct Curl_cwtype *ce = *cep;
  808. if((strncasecompare(name, ce->name, len) && !ce->name[len]) ||
  809. (ce->alias && strncasecompare(name, ce->alias, len)
  810. && !ce->alias[len]))
  811. return ce;
  812. }
  813. }
  814. /* look among the general decoders */
  815. for(cep = general_unencoders; *cep; cep++) {
  816. const struct Curl_cwtype *ce = *cep;
  817. if((strncasecompare(name, ce->name, len) && !ce->name[len]) ||
  818. (ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len]))
  819. return ce;
  820. }
  821. return NULL;
  822. }
  823. /* Set-up the unencoding stack from the Content-Encoding header value.
  824. * See RFC 7231 section 3.1.2.2. */
  825. CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
  826. const char *enclist, int is_transfer)
  827. {
  828. Curl_cwriter_phase phase = is_transfer?
  829. CURL_CW_TRANSFER_DECODE:CURL_CW_CONTENT_DECODE;
  830. CURLcode result;
  831. do {
  832. const char *name;
  833. size_t namelen;
  834. bool is_chunked = FALSE;
  835. /* Parse a single encoding name. */
  836. while(ISBLANK(*enclist) || *enclist == ',')
  837. enclist++;
  838. name = enclist;
  839. for(namelen = 0; *enclist && *enclist != ','; enclist++)
  840. if(!ISSPACE(*enclist))
  841. namelen = enclist - name + 1;
  842. if(namelen) {
  843. const struct Curl_cwtype *cwt;
  844. struct Curl_cwriter *writer;
  845. CURL_TRC_WRITE(data, "looking for %s decoder: %.*s",
  846. is_transfer? "transfer" : "content", (int)namelen, name);
  847. is_chunked = (is_transfer && (namelen == 7) &&
  848. strncasecompare(name, "chunked", 7));
  849. /* if we skip the decoding in this phase, do not look further.
  850. * Exception is "chunked" transfer-encoding which always must happen */
  851. if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
  852. (!is_transfer && data->set.http_ce_skip)) {
  853. /* not requested, ignore */
  854. CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
  855. (int)namelen, name);
  856. return CURLE_OK;
  857. }
  858. if(Curl_cwriter_count(data, phase) + 1 >= MAX_ENCODE_STACK) {
  859. failf(data, "Reject response due to more than %u content encodings",
  860. MAX_ENCODE_STACK);
  861. return CURLE_BAD_CONTENT_ENCODING;
  862. }
  863. cwt = find_unencode_writer(name, namelen, phase);
  864. if(cwt && is_chunked && Curl_cwriter_get_by_type(data, cwt)) {
  865. /* A 'chunked' transfer encoding has already been added.
  866. * Ignore duplicates. See #13451.
  867. * Also RFC 9112, ch. 6.1:
  868. * "A sender MUST NOT apply the chunked transfer coding more than
  869. * once to a message body."
  870. */
  871. CURL_TRC_WRITE(data, "ignoring duplicate 'chunked' decoder");
  872. return CURLE_OK;
  873. }
  874. if(is_transfer && !is_chunked &&
  875. Curl_cwriter_get_by_name(data, "chunked")) {
  876. /* RFC 9112, ch. 6.1:
  877. * "If any transfer coding other than chunked is applied to a
  878. * response's content, the sender MUST either apply chunked as the
  879. * final transfer coding or terminate the message by closing the
  880. * connection."
  881. * "chunked" must be the last added to be the first in its phase,
  882. * reject this.
  883. */
  884. failf(data, "Reject response due to 'chunked' not being the last "
  885. "Transfer-Encoding");
  886. return CURLE_BAD_CONTENT_ENCODING;
  887. }
  888. if(!cwt)
  889. cwt = &error_writer; /* Defer error at use. */
  890. result = Curl_cwriter_create(&writer, data, cwt, phase);
  891. CURL_TRC_WRITE(data, "added %s decoder %s -> %d",
  892. is_transfer? "transfer" : "content", cwt->name, result);
  893. if(result)
  894. return result;
  895. result = Curl_cwriter_add(data, writer);
  896. if(result) {
  897. Curl_cwriter_free(data, writer);
  898. return result;
  899. }
  900. }
  901. } while(*enclist);
  902. return CURLE_OK;
  903. }
  904. #else
  905. /* Stubs for builds without HTTP. */
  906. CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
  907. const char *enclist, int is_transfer)
  908. {
  909. (void) data;
  910. (void) enclist;
  911. (void) is_transfer;
  912. return CURLE_NOT_BUILT_IN;
  913. }
  914. void Curl_all_content_encodings(char *buf, size_t blen)
  915. {
  916. DEBUGASSERT(buf);
  917. DEBUGASSERT(blen);
  918. if(blen < sizeof(CONTENT_ENCODING_DEFAULT))
  919. buf[0] = 0;
  920. else
  921. strcpy(buf, CONTENT_ENCODING_DEFAULT);
  922. }
  923. #endif /* CURL_DISABLE_HTTP */