gzip.c 69 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253
  1. /* vi: set sw=4 ts=4: */
  2. /*
  3. * Gzip implementation for busybox
  4. *
  5. * Based on GNU gzip Copyright (C) 1992-1993 Jean-loup Gailly.
  6. *
  7. * Originally adjusted for busybox by Charles P. Wright <cpw@unix.asb.com>
  8. * "this is a stripped down version of gzip I put into busybox, it does
  9. * only standard in to standard out with -9 compression. It also requires
  10. * the zcat module for some important functions."
  11. *
  12. * Adjusted further by Erik Andersen <andersen@codepoet.org> to support
  13. * files as well as stdin/stdout, and to generally behave itself wrt
  14. * command line handling.
  15. *
  16. * Licensed under GPLv2 or later, see file LICENSE in this source tree.
  17. */
  18. /* big objects in bss:
  19. * 00000020 b bl_count
  20. * 00000074 b base_length
  21. * 00000078 b base_dist
  22. * 00000078 b static_dtree
  23. * 0000009c b bl_tree
  24. * 000000f4 b dyn_dtree
  25. * 00000100 b length_code
  26. * 00000200 b dist_code
  27. * 0000023d b depth
  28. * 00000400 b flag_buf
  29. * 0000047a b heap
  30. * 00000480 b static_ltree
  31. * 000008f4 b dyn_ltree
  32. */
  33. /* TODO: full support for -v for DESKTOP
  34. * "/usr/bin/gzip -v a bogus aa" should say:
  35. a: 85.1% -- replaced with a.gz
  36. gzip: bogus: No such file or directory
  37. aa: 85.1% -- replaced with aa.gz
  38. */
  39. //config:config GZIP
  40. //config: bool "gzip (19 kb)"
  41. //config: default y
  42. //config: help
  43. //config: gzip is used to compress files.
  44. //config: It's probably the most widely used UNIX compression program.
  45. //config:
  46. //config:config FEATURE_GZIP_LONG_OPTIONS
  47. //config: bool "Enable long options"
  48. //config: default y
  49. //config: depends on GZIP && LONG_OPTS
  50. //config:
  51. //config:config GZIP_FAST
  52. //config: int "Trade memory for speed (0:small,slow - 2:fast,big)"
  53. //config: default 0
  54. //config: range 0 2
  55. //config: depends on GZIP
  56. //config: help
  57. //config: Enable big memory options for gzip.
  58. //config: 0: small buffers, small hash-tables
  59. //config: 1: larger buffers, larger hash-tables
  60. //config: 2: larger buffers, largest hash-tables
  61. //config: Larger models may give slightly better compression
  62. //config:
  63. //config:config FEATURE_GZIP_LEVELS
  64. //config: bool "Enable compression levels"
  65. //config: default n
  66. //config: depends on GZIP
  67. //config: help
  68. //config: Enable support for compression levels 4-9. The default level
  69. //config: is 6. If levels 1-3 are specified, 4 is used.
  70. //config: If this option is not selected, -N options are ignored and -9
  71. //config: is used.
  72. //config:
  73. //config:config FEATURE_GZIP_DECOMPRESS
  74. //config: bool "Enable decompression"
  75. //config: default y
  76. //config: depends on GZIP || GUNZIP || ZCAT
  77. //config: help
  78. //config: Enable -d (--decompress) and -t (--test) options for gzip.
  79. //config: This will be automatically selected if gunzip or zcat is
  80. //config: enabled.
  81. //applet:IF_GZIP(APPLET(gzip, BB_DIR_BIN, BB_SUID_DROP))
  82. //kbuild:lib-$(CONFIG_GZIP) += gzip.o
  83. //usage:#define gzip_trivial_usage
  84. //usage: "[-cfk" IF_FEATURE_GZIP_DECOMPRESS("dt") IF_FEATURE_GZIP_LEVELS("123456789") "] [FILE]..."
  85. //usage:#define gzip_full_usage "\n\n"
  86. //usage: "Compress FILEs (or stdin)\n"
  87. //usage: IF_FEATURE_GZIP_LEVELS(
  88. //usage: "\n -1..9 Compression level"
  89. //usage: )
  90. //usage: IF_FEATURE_GZIP_DECOMPRESS(
  91. //usage: "\n -d Decompress"
  92. //usage: "\n -t Test file integrity"
  93. //usage: )
  94. //usage: "\n -c Write to stdout"
  95. //usage: "\n -f Force"
  96. //usage: "\n -k Keep input files"
  97. //usage:
  98. //usage:#define gzip_example_usage
  99. //usage: "$ ls -la /tmp/busybox*\n"
  100. //usage: "-rw-rw-r-- 1 andersen andersen 1761280 Apr 14 17:47 /tmp/busybox.tar\n"
  101. //usage: "$ gzip /tmp/busybox.tar\n"
  102. //usage: "$ ls -la /tmp/busybox*\n"
  103. //usage: "-rw-rw-r-- 1 andersen andersen 554058 Apr 14 17:49 /tmp/busybox.tar.gz\n"
  104. #include "libbb.h"
  105. #include "bb_archive.h"
  106. /* ===========================================================================
  107. */
  108. //#define DEBUG 1
  109. /* Diagnostic functions */
  110. #ifdef DEBUG
  111. # define Assert(cond,msg) { if (!(cond)) bb_error_msg(msg); }
  112. # define Trace(x) fprintf x
  113. # define Tracev(x) {if (verbose) fprintf x; }
  114. # define Tracevv(x) {if (verbose > 1) fprintf x; }
  115. # define Tracec(c,x) {if (verbose && (c)) fprintf x; }
  116. # define Tracecv(c,x) {if (verbose > 1 && (c)) fprintf x; }
  117. #else
  118. # define Assert(cond,msg)
  119. # define Trace(x)
  120. # define Tracev(x)
  121. # define Tracevv(x)
  122. # define Tracec(c,x)
  123. # define Tracecv(c,x)
  124. #endif
  125. /* ===========================================================================
  126. */
  127. #if CONFIG_GZIP_FAST == 0
  128. # define SMALL_MEM
  129. #elif CONFIG_GZIP_FAST == 1
  130. # define MEDIUM_MEM
  131. #elif CONFIG_GZIP_FAST == 2
  132. # define BIG_MEM
  133. #else
  134. # error "Invalid CONFIG_GZIP_FAST value"
  135. #endif
  136. #ifndef INBUFSIZ
  137. # ifdef SMALL_MEM
  138. # define INBUFSIZ 0x2000 /* input buffer size */
  139. # else
  140. # define INBUFSIZ 0x8000 /* input buffer size */
  141. # endif
  142. #endif
  143. #ifndef OUTBUFSIZ
  144. # ifdef SMALL_MEM
  145. # define OUTBUFSIZ 8192 /* output buffer size */
  146. # else
  147. # define OUTBUFSIZ 16384 /* output buffer size */
  148. # endif
  149. #endif
  150. #ifndef DIST_BUFSIZE
  151. # ifdef SMALL_MEM
  152. # define DIST_BUFSIZE 0x2000 /* buffer for distances, see trees.c */
  153. # else
  154. # define DIST_BUFSIZE 0x8000 /* buffer for distances, see trees.c */
  155. # endif
  156. #endif
  157. /* gzip flag byte */
  158. #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
  159. #define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */
  160. #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
  161. #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
  162. #define COMMENT 0x10 /* bit 4 set: file comment present */
  163. #define RESERVED 0xC0 /* bit 6,7: reserved */
  164. /* internal file attribute */
  165. #define UNKNOWN 0xffff
  166. #define BINARY 0
  167. #define ASCII 1
  168. #ifndef WSIZE
  169. # define WSIZE 0x8000 /* window size--must be a power of two, and */
  170. #endif /* at least 32K for zip's deflate method */
  171. #define MIN_MATCH 3
  172. #define MAX_MATCH 258
  173. /* The minimum and maximum match lengths */
  174. #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1)
  175. /* Minimum amount of lookahead, except at the end of the input file.
  176. * See deflate.c for comments about the MIN_MATCH+1.
  177. */
  178. #define MAX_DIST (WSIZE-MIN_LOOKAHEAD)
  179. /* In order to simplify the code, particularly on 16 bit machines, match
  180. * distances are limited to MAX_DIST instead of WSIZE.
  181. */
  182. #ifndef MAX_PATH_LEN
  183. # define MAX_PATH_LEN 1024 /* max pathname length */
  184. #endif
  185. #define seekable() 0 /* force sequential output */
  186. #define translate_eol 0 /* no option -a yet */
  187. #ifndef BITS
  188. # define BITS 16
  189. #endif
  190. #define INIT_BITS 9 /* Initial number of bits per code */
  191. #define BIT_MASK 0x1f /* Mask for 'number of compression bits' */
  192. /* Mask 0x20 is reserved to mean a fourth header byte, and 0x40 is free.
  193. * It's a pity that old uncompress does not check bit 0x20. That makes
  194. * extension of the format actually undesirable because old compress
  195. * would just crash on the new format instead of giving a meaningful
  196. * error message. It does check the number of bits, but it's more
  197. * helpful to say "unsupported format, get a new version" than
  198. * "can only handle 16 bits".
  199. */
  200. #ifdef MAX_EXT_CHARS
  201. # define MAX_SUFFIX MAX_EXT_CHARS
  202. #else
  203. # define MAX_SUFFIX 30
  204. #endif
  205. /* ===========================================================================
  206. * Compile with MEDIUM_MEM to reduce the memory requirements or
  207. * with SMALL_MEM to use as little memory as possible. Use BIG_MEM if the
  208. * entire input file can be held in memory (not possible on 16 bit systems).
  209. * Warning: defining these symbols affects HASH_BITS (see below) and thus
  210. * affects the compression ratio. The compressed output
  211. * is still correct, and might even be smaller in some cases.
  212. */
  213. #ifdef SMALL_MEM
  214. # define HASH_BITS 13 /* Number of bits used to hash strings */
  215. #endif
  216. #ifdef MEDIUM_MEM
  217. # define HASH_BITS 14
  218. #endif
  219. #ifndef HASH_BITS
  220. # define HASH_BITS 15
  221. /* For portability to 16 bit machines, do not use values above 15. */
  222. #endif
  223. #define HASH_SIZE (unsigned)(1<<HASH_BITS)
  224. #define HASH_MASK (HASH_SIZE-1)
  225. #define WMASK (WSIZE-1)
  226. /* HASH_SIZE and WSIZE must be powers of two */
  227. #ifndef TOO_FAR
  228. # define TOO_FAR 4096
  229. #endif
  230. /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
  231. /* ===========================================================================
  232. * These types are not really 'char', 'short' and 'long'
  233. */
  234. typedef uint8_t uch;
  235. typedef uint16_t ush;
  236. typedef uint32_t ulg;
  237. typedef int32_t lng;
  238. typedef ush Pos;
  239. typedef unsigned IPos;
  240. /* A Pos is an index in the character window. We use short instead of int to
  241. * save space in the various tables. IPos is used only for parameter passing.
  242. */
  243. enum {
  244. WINDOW_SIZE = 2 * WSIZE,
  245. /* window size, 2*WSIZE except for MMAP or BIG_MEM, where it is the
  246. * input file length plus MIN_LOOKAHEAD.
  247. */
  248. #if !ENABLE_FEATURE_GZIP_LEVELS
  249. max_chain_length = 4096,
  250. /* To speed up deflation, hash chains are never searched beyond this length.
  251. * A higher limit improves compression ratio but degrades the speed.
  252. */
  253. max_lazy_match = 258,
  254. /* Attempt to find a better match only when the current match is strictly
  255. * smaller than this value. This mechanism is used only for compression
  256. * levels >= 4.
  257. */
  258. max_insert_length = max_lazy_match,
  259. /* Insert new strings in the hash table only if the match length
  260. * is not greater than this length. This saves time but degrades compression.
  261. * max_insert_length is used only for compression levels <= 3.
  262. */
  263. good_match = 32,
  264. /* Use a faster search when the previous match is longer than this */
  265. /* Values for max_lazy_match, good_match and max_chain_length, depending on
  266. * the desired pack level (0..9). The values given below have been tuned to
  267. * exclude worst case performance for pathological files. Better values may be
  268. * found for specific files.
  269. */
  270. nice_match = 258, /* Stop searching when current match exceeds this */
  271. /* Note: the deflate() code requires max_lazy >= MIN_MATCH and max_chain >= 4
  272. * For deflate_fast() (levels <= 3) good is ignored and lazy has a different
  273. * meaning.
  274. */
  275. #endif /* ENABLE_FEATURE_GZIP_LEVELS */
  276. };
  277. struct globals {
  278. #if ENABLE_FEATURE_GZIP_LEVELS
  279. unsigned max_chain_length;
  280. unsigned max_lazy_match;
  281. unsigned good_match;
  282. unsigned nice_match;
  283. #define max_chain_length (G1.max_chain_length)
  284. #define max_lazy_match (G1.max_lazy_match)
  285. #define good_match (G1.good_match)
  286. #define nice_match (G1.nice_match)
  287. #endif
  288. lng block_start;
  289. /* window position at the beginning of the current output block. Gets
  290. * negative when the window is moved backwards.
  291. */
  292. unsigned ins_h; /* hash index of string to be inserted */
  293. #define H_SHIFT ((HASH_BITS+MIN_MATCH-1) / MIN_MATCH)
  294. /* Number of bits by which ins_h and del_h must be shifted at each
  295. * input step. It must be such that after MIN_MATCH steps, the oldest
  296. * byte no longer takes part in the hash key, that is:
  297. * H_SHIFT * MIN_MATCH >= HASH_BITS
  298. */
  299. unsigned prev_length;
  300. /* Length of the best match at previous step. Matches not greater than this
  301. * are discarded. This is used in the lazy match evaluation.
  302. */
  303. unsigned strstart; /* start of string to insert */
  304. unsigned match_start; /* start of matching string */
  305. unsigned lookahead; /* number of valid bytes ahead in window */
  306. /* ===========================================================================
  307. */
  308. #define DECLARE(type, array, size) \
  309. type * array
  310. #define ALLOC(type, array, size) \
  311. array = xzalloc((size_t)(((size)+1L)/2) * 2*sizeof(type))
  312. #define FREE(array) \
  313. do { free(array); array = NULL; } while (0)
  314. /* global buffers */
  315. /* buffer for literals or lengths */
  316. /* DECLARE(uch, l_buf, LIT_BUFSIZE); */
  317. DECLARE(uch, l_buf, INBUFSIZ);
  318. DECLARE(ush, d_buf, DIST_BUFSIZE);
  319. DECLARE(uch, outbuf, OUTBUFSIZ);
  320. /* Sliding window. Input bytes are read into the second half of the window,
  321. * and move to the first half later to keep a dictionary of at least WSIZE
  322. * bytes. With this organization, matches are limited to a distance of
  323. * WSIZE-MAX_MATCH bytes, but this ensures that IO is always
  324. * performed with a length multiple of the block size. Also, it limits
  325. * the window size to 64K, which is quite useful on MSDOS.
  326. * To do: limit the window size to WSIZE+BSZ if SMALL_MEM (the code would
  327. * be less efficient).
  328. */
  329. DECLARE(uch, window, 2L * WSIZE);
  330. /* Link to older string with same hash index. To limit the size of this
  331. * array to 64K, this link is maintained only for the last 32K strings.
  332. * An index in this array is thus a window index modulo 32K.
  333. */
  334. /* DECLARE(Pos, prev, WSIZE); */
  335. DECLARE(ush, prev, 1L << BITS);
  336. /* Heads of the hash chains or 0. */
  337. /* DECLARE(Pos, head, 1<<HASH_BITS); */
  338. #define head (G1.prev + WSIZE) /* hash head (see deflate.c) */
  339. /* number of input bytes */
  340. ulg isize; /* only 32 bits stored in .gz file */
  341. /* bbox always use stdin/stdout */
  342. #define ifd STDIN_FILENO /* input file descriptor */
  343. #define ofd STDOUT_FILENO /* output file descriptor */
  344. #ifdef DEBUG
  345. unsigned insize; /* valid bytes in l_buf */
  346. #endif
  347. unsigned outcnt; /* bytes in output buffer */
  348. smallint eofile; /* flag set at end of input file */
  349. /* ===========================================================================
  350. * Local data used by the "bit string" routines.
  351. */
  352. unsigned short bi_buf;
  353. /* Output buffer. bits are inserted starting at the bottom (least significant
  354. * bits).
  355. */
  356. #undef BUF_SIZE
  357. #define BUF_SIZE (8 * sizeof(G1.bi_buf))
  358. /* Number of bits used within bi_buf. (bi_buf might be implemented on
  359. * more than 16 bits on some systems.)
  360. */
  361. int bi_valid;
  362. /* Current input function. Set to mem_read for in-memory compression */
  363. #ifdef DEBUG
  364. ulg bits_sent; /* bit length of the compressed data */
  365. #endif
  366. /*uint32_t *crc_32_tab;*/
  367. uint32_t crc; /* shift register contents */
  368. };
  369. #define G1 (*(ptr_to_globals - 1))
  370. /* ===========================================================================
  371. * Write the output buffer outbuf[0..outcnt-1] and update bytes_out.
  372. * (used for the compressed data only)
  373. */
  374. static void flush_outbuf(void)
  375. {
  376. if (G1.outcnt == 0)
  377. return;
  378. xwrite(ofd, (char *) G1.outbuf, G1.outcnt);
  379. G1.outcnt = 0;
  380. }
  381. /* ===========================================================================
  382. */
  383. /* put_8bit is used for the compressed output */
  384. #define put_8bit(c) \
  385. do { \
  386. G1.outbuf[G1.outcnt++] = (c); \
  387. if (G1.outcnt == OUTBUFSIZ) \
  388. flush_outbuf(); \
  389. } while (0)
  390. /* Output a 16 bit value, lsb first */
  391. static void put_16bit(ush w)
  392. {
  393. /* GCC 4.2.1 won't optimize out redundant loads of G1.outcnt
  394. * (probably because of fear of aliasing with G1.outbuf[]
  395. * stores), do it explicitly:
  396. */
  397. unsigned outcnt = G1.outcnt;
  398. uch *dst = &G1.outbuf[outcnt];
  399. #if BB_UNALIGNED_MEMACCESS_OK && BB_LITTLE_ENDIAN
  400. if (outcnt < OUTBUFSIZ-2) {
  401. /* Common case */
  402. ush *dst16 = (void*) dst;
  403. *dst16 = w; /* unalinged LSB 16-bit store */
  404. G1.outcnt = outcnt + 2;
  405. return;
  406. }
  407. *dst = (uch)w;
  408. w >>= 8;
  409. #else
  410. *dst = (uch)w;
  411. w >>= 8;
  412. if (outcnt < OUTBUFSIZ-2) {
  413. /* Common case */
  414. dst[1] = w;
  415. G1.outcnt = outcnt + 2;
  416. return;
  417. }
  418. #endif
  419. /* Slowpath: we will need to do flush_outbuf() */
  420. G1.outcnt = ++outcnt;
  421. if (outcnt == OUTBUFSIZ)
  422. flush_outbuf();
  423. put_8bit(w);
  424. }
  425. static void put_32bit(ulg n)
  426. {
  427. put_16bit(n);
  428. put_16bit(n >> 16);
  429. }
  430. /* ===========================================================================
  431. * Run a set of bytes through the crc shift register. If s is a NULL
  432. * pointer, then initialize the crc shift register contents instead.
  433. * Return the current crc in either case.
  434. */
  435. static void updcrc(uch * s, unsigned n)
  436. {
  437. G1.crc = crc32_block_endian0(G1.crc, s, n, global_crc32_table /*G1.crc_32_tab*/);
  438. }
  439. /* ===========================================================================
  440. * Read a new buffer from the current input file, perform end-of-line
  441. * translation, and update the crc and input file size.
  442. * IN assertion: size >= 2 (for end-of-line translation)
  443. */
  444. static unsigned file_read(void *buf, unsigned size)
  445. {
  446. unsigned len;
  447. Assert(G1.insize == 0, "l_buf not empty");
  448. len = safe_read(ifd, buf, size);
  449. if (len == (unsigned)(-1) || len == 0)
  450. return len;
  451. updcrc(buf, len);
  452. G1.isize += len;
  453. return len;
  454. }
  455. /* ===========================================================================
  456. * Send a value on a given number of bits.
  457. * IN assertion: length <= 16 and value fits in length bits.
  458. */
  459. static void send_bits(int value, int length)
  460. {
  461. #ifdef DEBUG
  462. Tracev((stderr, " l %2d v %4x ", length, value));
  463. Assert(length > 0 && length <= 15, "invalid length");
  464. G1.bits_sent += length;
  465. #endif
  466. /* If not enough room in bi_buf, use (valid) bits from bi_buf and
  467. * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid))
  468. * unused bits in value.
  469. */
  470. if (G1.bi_valid > (int) BUF_SIZE - length) {
  471. G1.bi_buf |= (value << G1.bi_valid);
  472. put_16bit(G1.bi_buf);
  473. G1.bi_buf = (ush) value >> (BUF_SIZE - G1.bi_valid);
  474. G1.bi_valid += length - BUF_SIZE;
  475. } else {
  476. G1.bi_buf |= value << G1.bi_valid;
  477. G1.bi_valid += length;
  478. }
  479. }
  480. /* ===========================================================================
  481. * Reverse the first len bits of a code, using straightforward code (a faster
  482. * method would use a table)
  483. * IN assertion: 1 <= len <= 15
  484. */
  485. static unsigned bi_reverse(unsigned code, int len)
  486. {
  487. unsigned res = 0;
  488. while (1) {
  489. res |= code & 1;
  490. if (--len <= 0) return res;
  491. code >>= 1;
  492. res <<= 1;
  493. }
  494. }
  495. /* ===========================================================================
  496. * Write out any remaining bits in an incomplete byte.
  497. */
  498. static void bi_windup(void)
  499. {
  500. if (G1.bi_valid > 8) {
  501. put_16bit(G1.bi_buf);
  502. } else if (G1.bi_valid > 0) {
  503. put_8bit(G1.bi_buf);
  504. }
  505. G1.bi_buf = 0;
  506. G1.bi_valid = 0;
  507. #ifdef DEBUG
  508. G1.bits_sent = (G1.bits_sent + 7) & ~7;
  509. #endif
  510. }
  511. /* ===========================================================================
  512. * Copy a stored block to the zip file, storing first the length and its
  513. * one's complement if requested.
  514. */
  515. static void copy_block(char *buf, unsigned len, int header)
  516. {
  517. bi_windup(); /* align on byte boundary */
  518. if (header) {
  519. put_16bit(len);
  520. put_16bit(~len);
  521. #ifdef DEBUG
  522. G1.bits_sent += 2 * 16;
  523. #endif
  524. }
  525. #ifdef DEBUG
  526. G1.bits_sent += (ulg) len << 3;
  527. #endif
  528. while (len--) {
  529. put_8bit(*buf++);
  530. }
  531. }
  532. /* ===========================================================================
  533. * Fill the window when the lookahead becomes insufficient.
  534. * Updates strstart and lookahead, and sets eofile if end of input file.
  535. * IN assertion: lookahead < MIN_LOOKAHEAD && strstart + lookahead > 0
  536. * OUT assertions: at least one byte has been read, or eofile is set;
  537. * file reads are performed for at least two bytes (required for the
  538. * translate_eol option).
  539. */
  540. static void fill_window(void)
  541. {
  542. unsigned n, m;
  543. unsigned more = WINDOW_SIZE - G1.lookahead - G1.strstart;
  544. /* Amount of free space at the end of the window. */
  545. /* If the window is almost full and there is insufficient lookahead,
  546. * move the upper half to the lower one to make room in the upper half.
  547. */
  548. if (more == (unsigned) -1) {
  549. /* Very unlikely, but possible on 16 bit machine if strstart == 0
  550. * and lookahead == 1 (input done one byte at time)
  551. */
  552. more--;
  553. } else if (G1.strstart >= WSIZE + MAX_DIST) {
  554. /* By the IN assertion, the window is not empty so we can't confuse
  555. * more == 0 with more == 64K on a 16 bit machine.
  556. */
  557. Assert(WINDOW_SIZE == 2 * WSIZE, "no sliding with BIG_MEM");
  558. memcpy(G1.window, G1.window + WSIZE, WSIZE);
  559. G1.match_start -= WSIZE;
  560. G1.strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */
  561. G1.block_start -= WSIZE;
  562. for (n = 0; n < HASH_SIZE; n++) {
  563. m = head[n];
  564. head[n] = (Pos) (m >= WSIZE ? m - WSIZE : 0);
  565. }
  566. for (n = 0; n < WSIZE; n++) {
  567. m = G1.prev[n];
  568. G1.prev[n] = (Pos) (m >= WSIZE ? m - WSIZE : 0);
  569. /* If n is not on any hash chain, prev[n] is garbage but
  570. * its value will never be used.
  571. */
  572. }
  573. more += WSIZE;
  574. }
  575. /* At this point, more >= 2 */
  576. if (!G1.eofile) {
  577. n = file_read(G1.window + G1.strstart + G1.lookahead, more);
  578. if (n == 0 || n == (unsigned) -1) {
  579. G1.eofile = 1;
  580. } else {
  581. G1.lookahead += n;
  582. }
  583. }
  584. }
  585. /* ===========================================================================
  586. * Set match_start to the longest match starting at the given string and
  587. * return its length. Matches shorter or equal to prev_length are discarded,
  588. * in which case the result is equal to prev_length and match_start is
  589. * garbage.
  590. * IN assertions: cur_match is the head of the hash chain for the current
  591. * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1
  592. */
  593. /* For MSDOS, OS/2 and 386 Unix, an optimized version is in match.asm or
  594. * match.s. The code is functionally equivalent, so you can use the C version
  595. * if desired.
  596. */
  597. static int longest_match(IPos cur_match)
  598. {
  599. unsigned chain_length = max_chain_length; /* max hash chain length */
  600. uch *scan = G1.window + G1.strstart; /* current string */
  601. uch *match; /* matched string */
  602. int len; /* length of current match */
  603. int best_len = G1.prev_length; /* best match length so far */
  604. IPos limit = G1.strstart > (IPos) MAX_DIST ? G1.strstart - (IPos) MAX_DIST : 0;
  605. /* Stop when cur_match becomes <= limit. To simplify the code,
  606. * we prevent matches with the string of window index 0.
  607. */
  608. /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16.
  609. * It is easy to get rid of this optimization if necessary.
  610. */
  611. #if HASH_BITS < 8 || MAX_MATCH != 258
  612. # error Code too clever
  613. #endif
  614. uch *strend = G1.window + G1.strstart + MAX_MATCH;
  615. uch scan_end1 = scan[best_len - 1];
  616. uch scan_end = scan[best_len];
  617. /* Do not waste too much time if we already have a good match: */
  618. if (G1.prev_length >= good_match) {
  619. chain_length >>= 2;
  620. }
  621. Assert(G1.strstart <= WINDOW_SIZE - MIN_LOOKAHEAD, "insufficient lookahead");
  622. do {
  623. Assert(cur_match < G1.strstart, "no future");
  624. match = G1.window + cur_match;
  625. /* Skip to next match if the match length cannot increase
  626. * or if the match length is less than 2:
  627. */
  628. if (match[best_len] != scan_end
  629. || match[best_len - 1] != scan_end1
  630. || *match != *scan || *++match != scan[1]
  631. ) {
  632. continue;
  633. }
  634. /* The check at best_len-1 can be removed because it will be made
  635. * again later. (This heuristic is not always a win.)
  636. * It is not necessary to compare scan[2] and match[2] since they
  637. * are always equal when the other bytes match, given that
  638. * the hash keys are equal and that HASH_BITS >= 8.
  639. */
  640. scan += 2, match++;
  641. /* We check for insufficient lookahead only every 8th comparison;
  642. * the 256th check will be made at strstart+258.
  643. */
  644. do {
  645. } while (*++scan == *++match && *++scan == *++match &&
  646. *++scan == *++match && *++scan == *++match &&
  647. *++scan == *++match && *++scan == *++match &&
  648. *++scan == *++match && *++scan == *++match && scan < strend);
  649. len = MAX_MATCH - (int) (strend - scan);
  650. scan = strend - MAX_MATCH;
  651. if (len > best_len) {
  652. G1.match_start = cur_match;
  653. best_len = len;
  654. if (len >= nice_match)
  655. break;
  656. scan_end1 = scan[best_len - 1];
  657. scan_end = scan[best_len];
  658. }
  659. } while ((cur_match = G1.prev[cur_match & WMASK]) > limit
  660. && --chain_length != 0);
  661. return best_len;
  662. }
  663. #ifdef DEBUG
  664. /* ===========================================================================
  665. * Check that the match at match_start is indeed a match.
  666. */
  667. static void check_match(IPos start, IPos match, int length)
  668. {
  669. /* check that the match is indeed a match */
  670. if (memcmp(G1.window + match, G1.window + start, length) != 0) {
  671. bb_error_msg(" start %d, match %d, length %d", start, match, length);
  672. bb_error_msg("invalid match");
  673. }
  674. if (verbose > 1) {
  675. bb_error_msg("\\[%d,%d]", start - match, length);
  676. do {
  677. bb_putchar_stderr(G1.window[start++]);
  678. } while (--length != 0);
  679. }
  680. }
  681. #else
  682. # define check_match(start, match, length) ((void)0)
  683. #endif
  684. /* trees.c -- output deflated data using Huffman coding
  685. * Copyright (C) 1992-1993 Jean-loup Gailly
  686. * This is free software; you can redistribute it and/or modify it under the
  687. * terms of the GNU General Public License, see the file COPYING.
  688. */
  689. /* PURPOSE
  690. * Encode various sets of source values using variable-length
  691. * binary code trees.
  692. *
  693. * DISCUSSION
  694. * The PKZIP "deflation" process uses several Huffman trees. The more
  695. * common source values are represented by shorter bit sequences.
  696. *
  697. * Each code tree is stored in the ZIP file in a compressed form
  698. * which is itself a Huffman encoding of the lengths of
  699. * all the code strings (in ascending order by source values).
  700. * The actual code strings are reconstructed from the lengths in
  701. * the UNZIP process, as described in the "application note"
  702. * (APPNOTE.TXT) distributed as part of PKWARE's PKZIP program.
  703. *
  704. * REFERENCES
  705. * Lynch, Thomas J.
  706. * Data Compression: Techniques and Applications, pp. 53-55.
  707. * Lifetime Learning Publications, 1985. ISBN 0-534-03418-7.
  708. *
  709. * Storer, James A.
  710. * Data Compression: Methods and Theory, pp. 49-50.
  711. * Computer Science Press, 1988. ISBN 0-7167-8156-5.
  712. *
  713. * Sedgewick, R.
  714. * Algorithms, p290.
  715. * Addison-Wesley, 1983. ISBN 0-201-06672-6.
  716. *
  717. * INTERFACE
  718. * void ct_init()
  719. * Allocate the match buffer, initialize the various tables [and save
  720. * the location of the internal file attribute (ascii/binary) and
  721. * method (DEFLATE/STORE) -- deleted in bbox]
  722. *
  723. * void ct_tally(int dist, int lc);
  724. * Save the match info and tally the frequency counts.
  725. *
  726. * ulg flush_block(char *buf, ulg stored_len, int eof)
  727. * Determine the best encoding for the current block: dynamic trees,
  728. * static trees or store, and output the encoded block to the zip
  729. * file. Returns the total compressed length for the file so far.
  730. */
  731. #define MAX_BITS 15
  732. /* All codes must not exceed MAX_BITS bits */
  733. #define MAX_BL_BITS 7
  734. /* Bit length codes must not exceed MAX_BL_BITS bits */
  735. #define LENGTH_CODES 29
  736. /* number of length codes, not counting the special END_BLOCK code */
  737. #define LITERALS 256
  738. /* number of literal bytes 0..255 */
  739. #define END_BLOCK 256
  740. /* end of block literal code */
  741. #define L_CODES (LITERALS+1+LENGTH_CODES)
  742. /* number of Literal or Length codes, including the END_BLOCK code */
  743. #define D_CODES 30
  744. /* number of distance codes */
  745. #define BL_CODES 19
  746. /* number of codes used to transfer the bit lengths */
  747. /* extra bits for each length code */
  748. static const uint8_t extra_lbits[LENGTH_CODES] ALIGN1 = {
  749. 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4,
  750. 4, 4, 5, 5, 5, 5, 0
  751. };
  752. /* extra bits for each distance code */
  753. static const uint8_t extra_dbits[D_CODES] ALIGN1 = {
  754. 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9,
  755. 10, 10, 11, 11, 12, 12, 13, 13
  756. };
  757. /* extra bits for each bit length code */
  758. static const uint8_t extra_blbits[BL_CODES] ALIGN1 = {
  759. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7 };
  760. /* number of codes at each bit length for an optimal tree */
  761. static const uint8_t bl_order[BL_CODES] ALIGN1 = {
  762. 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
  763. #define STORED_BLOCK 0
  764. #define STATIC_TREES 1
  765. #define DYN_TREES 2
  766. /* The three kinds of block type */
  767. #ifndef LIT_BUFSIZE
  768. # ifdef SMALL_MEM
  769. # define LIT_BUFSIZE 0x2000
  770. # else
  771. # ifdef MEDIUM_MEM
  772. # define LIT_BUFSIZE 0x4000
  773. # else
  774. # define LIT_BUFSIZE 0x8000
  775. # endif
  776. # endif
  777. #endif
  778. #ifndef DIST_BUFSIZE
  779. # define DIST_BUFSIZE LIT_BUFSIZE
  780. #endif
  781. /* Sizes of match buffers for literals/lengths and distances. There are
  782. * 4 reasons for limiting LIT_BUFSIZE to 64K:
  783. * - frequencies can be kept in 16 bit counters
  784. * - if compression is not successful for the first block, all input data is
  785. * still in the window so we can still emit a stored block even when input
  786. * comes from standard input. (This can also be done for all blocks if
  787. * LIT_BUFSIZE is not greater than 32K.)
  788. * - if compression is not successful for a file smaller than 64K, we can
  789. * even emit a stored file instead of a stored block (saving 5 bytes).
  790. * - creating new Huffman trees less frequently may not provide fast
  791. * adaptation to changes in the input data statistics. (Take for
  792. * example a binary file with poorly compressible code followed by
  793. * a highly compressible string table.) Smaller buffer sizes give
  794. * fast adaptation but have of course the overhead of transmitting trees
  795. * more frequently.
  796. * - I can't count above 4
  797. * The current code is general and allows DIST_BUFSIZE < LIT_BUFSIZE (to save
  798. * memory at the expense of compression). Some optimizations would be possible
  799. * if we rely on DIST_BUFSIZE == LIT_BUFSIZE.
  800. */
  801. #define REP_3_6 16
  802. /* repeat previous bit length 3-6 times (2 bits of repeat count) */
  803. #define REPZ_3_10 17
  804. /* repeat a zero length 3-10 times (3 bits of repeat count) */
  805. #define REPZ_11_138 18
  806. /* repeat a zero length 11-138 times (7 bits of repeat count) */
  807. /* ===========================================================================
  808. */
  809. /* Data structure describing a single value and its code string. */
  810. typedef struct ct_data {
  811. union {
  812. ush freq; /* frequency count */
  813. ush code; /* bit string */
  814. } fc;
  815. union {
  816. ush dad; /* father node in Huffman tree */
  817. ush len; /* length of bit string */
  818. } dl;
  819. } ct_data;
  820. #define Freq fc.freq
  821. #define Code fc.code
  822. #define Dad dl.dad
  823. #define Len dl.len
  824. #define HEAP_SIZE (2*L_CODES + 1)
  825. /* maximum heap size */
  826. typedef struct tree_desc {
  827. ct_data *dyn_tree; /* the dynamic tree */
  828. ct_data *static_tree; /* corresponding static tree or NULL */
  829. const uint8_t *extra_bits; /* extra bits for each code or NULL */
  830. int extra_base; /* base index for extra_bits */
  831. int elems; /* max number of elements in the tree */
  832. int max_length; /* max bit length for the codes */
  833. int max_code; /* largest code with non zero frequency */
  834. } tree_desc;
  835. struct globals2 {
  836. ush heap[HEAP_SIZE]; /* heap used to build the Huffman trees */
  837. int heap_len; /* number of elements in the heap */
  838. int heap_max; /* element of largest frequency */
  839. /* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used.
  840. * The same heap array is used to build all trees.
  841. */
  842. ct_data dyn_ltree[HEAP_SIZE]; /* literal and length tree */
  843. ct_data dyn_dtree[2 * D_CODES + 1]; /* distance tree */
  844. ct_data static_ltree[L_CODES + 2];
  845. /* The static literal tree. Since the bit lengths are imposed, there is no
  846. * need for the L_CODES extra codes used during heap construction. However
  847. * The codes 286 and 287 are needed to build a canonical tree (see ct_init
  848. * below).
  849. */
  850. ct_data static_dtree[D_CODES];
  851. /* The static distance tree. (Actually a trivial tree since all codes use
  852. * 5 bits.)
  853. */
  854. ct_data bl_tree[2 * BL_CODES + 1];
  855. /* Huffman tree for the bit lengths */
  856. tree_desc l_desc;
  857. tree_desc d_desc;
  858. tree_desc bl_desc;
  859. ush bl_count[MAX_BITS + 1];
  860. /* The lengths of the bit length codes are sent in order of decreasing
  861. * probability, to avoid transmitting the lengths for unused bit length codes.
  862. */
  863. uch depth[2 * L_CODES + 1];
  864. /* Depth of each subtree used as tie breaker for trees of equal frequency */
  865. uch length_code[MAX_MATCH - MIN_MATCH + 1];
  866. /* length code for each normalized match length (0 == MIN_MATCH) */
  867. uch dist_code[512];
  868. /* distance codes. The first 256 values correspond to the distances
  869. * 3 .. 258, the last 256 values correspond to the top 8 bits of
  870. * the 15 bit distances.
  871. */
  872. int base_length[LENGTH_CODES];
  873. /* First normalized length for each code (0 = MIN_MATCH) */
  874. int base_dist[D_CODES];
  875. /* First normalized distance for each code (0 = distance of 1) */
  876. uch flag_buf[LIT_BUFSIZE / 8];
  877. /* flag_buf is a bit array distinguishing literals from lengths in
  878. * l_buf, thus indicating the presence or absence of a distance.
  879. */
  880. unsigned last_lit; /* running index in l_buf */
  881. unsigned last_dist; /* running index in d_buf */
  882. unsigned last_flags; /* running index in flag_buf */
  883. uch flags; /* current flags not yet saved in flag_buf */
  884. uch flag_bit; /* current bit used in flags */
  885. /* bits are filled in flags starting at bit 0 (least significant).
  886. * Note: these flags are overkill in the current code since we don't
  887. * take advantage of DIST_BUFSIZE == LIT_BUFSIZE.
  888. */
  889. ulg opt_len; /* bit length of current block with optimal trees */
  890. ulg static_len; /* bit length of current block with static trees */
  891. ulg compressed_len; /* total bit length of compressed file */
  892. };
  893. #define G2ptr ((struct globals2*)(ptr_to_globals))
  894. #define G2 (*G2ptr)
  895. /* ===========================================================================
  896. */
  897. static void gen_codes(ct_data * tree, int max_code);
  898. static void build_tree(tree_desc * desc);
  899. static void scan_tree(ct_data * tree, int max_code);
  900. static void send_tree(ct_data * tree, int max_code);
  901. static int build_bl_tree(void);
  902. static void send_all_trees(int lcodes, int dcodes, int blcodes);
  903. static void compress_block(ct_data * ltree, ct_data * dtree);
  904. #ifndef DEBUG
  905. /* Send a code of the given tree. c and tree must not have side effects */
  906. # define SEND_CODE(c, tree) send_bits(tree[c].Code, tree[c].Len)
  907. #else
  908. # define SEND_CODE(c, tree) \
  909. { \
  910. if (verbose > 1) bb_error_msg("\ncd %3d ", (c)); \
  911. send_bits(tree[c].Code, tree[c].Len); \
  912. }
  913. #endif
  914. #define D_CODE(dist) \
  915. ((dist) < 256 ? G2.dist_code[dist] : G2.dist_code[256 + ((dist)>>7)])
  916. /* Mapping from a distance to a distance code. dist is the distance - 1 and
  917. * must not have side effects. dist_code[256] and dist_code[257] are never
  918. * used.
  919. * The arguments must not have side effects.
  920. */
  921. /* ===========================================================================
  922. * Initialize a new block.
  923. */
  924. static void init_block(void)
  925. {
  926. int n; /* iterates over tree elements */
  927. /* Initialize the trees. */
  928. for (n = 0; n < L_CODES; n++)
  929. G2.dyn_ltree[n].Freq = 0;
  930. for (n = 0; n < D_CODES; n++)
  931. G2.dyn_dtree[n].Freq = 0;
  932. for (n = 0; n < BL_CODES; n++)
  933. G2.bl_tree[n].Freq = 0;
  934. G2.dyn_ltree[END_BLOCK].Freq = 1;
  935. G2.opt_len = G2.static_len = 0;
  936. G2.last_lit = G2.last_dist = G2.last_flags = 0;
  937. G2.flags = 0;
  938. G2.flag_bit = 1;
  939. }
  940. /* ===========================================================================
  941. * Restore the heap property by moving down the tree starting at node k,
  942. * exchanging a node with the smallest of its two sons if necessary, stopping
  943. * when the heap property is re-established (each father smaller than its
  944. * two sons).
  945. */
  946. /* Compares to subtrees, using the tree depth as tie breaker when
  947. * the subtrees have equal frequency. This minimizes the worst case length. */
  948. #define SMALLER(tree, n, m) \
  949. (tree[n].Freq < tree[m].Freq \
  950. || (tree[n].Freq == tree[m].Freq && G2.depth[n] <= G2.depth[m]))
  951. static void pqdownheap(ct_data * tree, int k)
  952. {
  953. int v = G2.heap[k];
  954. int j = k << 1; /* left son of k */
  955. while (j <= G2.heap_len) {
  956. /* Set j to the smallest of the two sons: */
  957. if (j < G2.heap_len && SMALLER(tree, G2.heap[j + 1], G2.heap[j]))
  958. j++;
  959. /* Exit if v is smaller than both sons */
  960. if (SMALLER(tree, v, G2.heap[j]))
  961. break;
  962. /* Exchange v with the smallest son */
  963. G2.heap[k] = G2.heap[j];
  964. k = j;
  965. /* And continue down the tree, setting j to the left son of k */
  966. j <<= 1;
  967. }
  968. G2.heap[k] = v;
  969. }
  970. /* ===========================================================================
  971. * Compute the optimal bit lengths for a tree and update the total bit length
  972. * for the current block.
  973. * IN assertion: the fields freq and dad are set, heap[heap_max] and
  974. * above are the tree nodes sorted by increasing frequency.
  975. * OUT assertions: the field len is set to the optimal bit length, the
  976. * array bl_count contains the frequencies for each bit length.
  977. * The length opt_len is updated; static_len is also updated if stree is
  978. * not null.
  979. */
  980. static void gen_bitlen(tree_desc * desc)
  981. {
  982. ct_data *tree = desc->dyn_tree;
  983. const uint8_t *extra = desc->extra_bits;
  984. int base = desc->extra_base;
  985. int max_code = desc->max_code;
  986. int max_length = desc->max_length;
  987. ct_data *stree = desc->static_tree;
  988. int h; /* heap index */
  989. int n, m; /* iterate over the tree elements */
  990. int bits; /* bit length */
  991. int xbits; /* extra bits */
  992. ush f; /* frequency */
  993. int overflow = 0; /* number of elements with bit length too large */
  994. for (bits = 0; bits <= MAX_BITS; bits++)
  995. G2.bl_count[bits] = 0;
  996. /* In a first pass, compute the optimal bit lengths (which may
  997. * overflow in the case of the bit length tree).
  998. */
  999. tree[G2.heap[G2.heap_max]].Len = 0; /* root of the heap */
  1000. for (h = G2.heap_max + 1; h < HEAP_SIZE; h++) {
  1001. n = G2.heap[h];
  1002. bits = tree[tree[n].Dad].Len + 1;
  1003. if (bits > max_length) {
  1004. bits = max_length;
  1005. overflow++;
  1006. }
  1007. tree[n].Len = (ush) bits;
  1008. /* We overwrite tree[n].Dad which is no longer needed */
  1009. if (n > max_code)
  1010. continue; /* not a leaf node */
  1011. G2.bl_count[bits]++;
  1012. xbits = 0;
  1013. if (n >= base)
  1014. xbits = extra[n - base];
  1015. f = tree[n].Freq;
  1016. G2.opt_len += (ulg) f *(bits + xbits);
  1017. if (stree)
  1018. G2.static_len += (ulg) f * (stree[n].Len + xbits);
  1019. }
  1020. if (overflow == 0)
  1021. return;
  1022. Trace((stderr, "\nbit length overflow\n"));
  1023. /* This happens for example on obj2 and pic of the Calgary corpus */
  1024. /* Find the first bit length which could increase: */
  1025. do {
  1026. bits = max_length - 1;
  1027. while (G2.bl_count[bits] == 0)
  1028. bits--;
  1029. G2.bl_count[bits]--; /* move one leaf down the tree */
  1030. G2.bl_count[bits + 1] += 2; /* move one overflow item as its brother */
  1031. G2.bl_count[max_length]--;
  1032. /* The brother of the overflow item also moves one step up,
  1033. * but this does not affect bl_count[max_length]
  1034. */
  1035. overflow -= 2;
  1036. } while (overflow > 0);
  1037. /* Now recompute all bit lengths, scanning in increasing frequency.
  1038. * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all
  1039. * lengths instead of fixing only the wrong ones. This idea is taken
  1040. * from 'ar' written by Haruhiko Okumura.)
  1041. */
  1042. for (bits = max_length; bits != 0; bits--) {
  1043. n = G2.bl_count[bits];
  1044. while (n != 0) {
  1045. m = G2.heap[--h];
  1046. if (m > max_code)
  1047. continue;
  1048. if (tree[m].Len != (unsigned) bits) {
  1049. Trace((stderr, "code %d bits %d->%d\n", m, tree[m].Len, bits));
  1050. G2.opt_len += ((int32_t) bits - tree[m].Len) * tree[m].Freq;
  1051. tree[m].Len = bits;
  1052. }
  1053. n--;
  1054. }
  1055. }
  1056. }
  1057. /* ===========================================================================
  1058. * Generate the codes for a given tree and bit counts (which need not be
  1059. * optimal).
  1060. * IN assertion: the array bl_count contains the bit length statistics for
  1061. * the given tree and the field len is set for all tree elements.
  1062. * OUT assertion: the field code is set for all tree elements of non
  1063. * zero code length.
  1064. */
  1065. static void gen_codes(ct_data * tree, int max_code)
  1066. {
  1067. ush next_code[MAX_BITS + 1]; /* next code value for each bit length */
  1068. ush code = 0; /* running code value */
  1069. int bits; /* bit index */
  1070. int n; /* code index */
  1071. /* The distribution counts are first used to generate the code values
  1072. * without bit reversal.
  1073. */
  1074. for (bits = 1; bits <= MAX_BITS; bits++) {
  1075. next_code[bits] = code = (code + G2.bl_count[bits - 1]) << 1;
  1076. }
  1077. /* Check that the bit counts in bl_count are consistent. The last code
  1078. * must be all ones.
  1079. */
  1080. Assert(code + G2.bl_count[MAX_BITS] - 1 == (1 << MAX_BITS) - 1,
  1081. "inconsistent bit counts");
  1082. Tracev((stderr, "\ngen_codes: max_code %d ", max_code));
  1083. for (n = 0; n <= max_code; n++) {
  1084. int len = tree[n].Len;
  1085. if (len == 0)
  1086. continue;
  1087. /* Now reverse the bits */
  1088. tree[n].Code = bi_reverse(next_code[len]++, len);
  1089. Tracec(tree != G2.static_ltree,
  1090. (stderr, "\nn %3d %c l %2d c %4x (%x) ", n,
  1091. (n > ' ' ? n : ' '), len, tree[n].Code,
  1092. next_code[len] - 1));
  1093. }
  1094. }
  1095. /* ===========================================================================
  1096. * Construct one Huffman tree and assigns the code bit strings and lengths.
  1097. * Update the total bit length for the current block.
  1098. * IN assertion: the field freq is set for all tree elements.
  1099. * OUT assertions: the fields len and code are set to the optimal bit length
  1100. * and corresponding code. The length opt_len is updated; static_len is
  1101. * also updated if stree is not null. The field max_code is set.
  1102. */
  1103. /* Remove the smallest element from the heap and recreate the heap with
  1104. * one less element. Updates heap and heap_len. */
  1105. #define SMALLEST 1
  1106. /* Index within the heap array of least frequent node in the Huffman tree */
  1107. #define PQREMOVE(tree, top) \
  1108. do { \
  1109. top = G2.heap[SMALLEST]; \
  1110. G2.heap[SMALLEST] = G2.heap[G2.heap_len--]; \
  1111. pqdownheap(tree, SMALLEST); \
  1112. } while (0)
  1113. static void build_tree(tree_desc * desc)
  1114. {
  1115. ct_data *tree = desc->dyn_tree;
  1116. ct_data *stree = desc->static_tree;
  1117. int elems = desc->elems;
  1118. int n, m; /* iterate over heap elements */
  1119. int max_code = -1; /* largest code with non zero frequency */
  1120. int node = elems; /* next internal node of the tree */
  1121. /* Construct the initial heap, with least frequent element in
  1122. * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1].
  1123. * heap[0] is not used.
  1124. */
  1125. G2.heap_len = 0;
  1126. G2.heap_max = HEAP_SIZE;
  1127. for (n = 0; n < elems; n++) {
  1128. if (tree[n].Freq != 0) {
  1129. G2.heap[++G2.heap_len] = max_code = n;
  1130. G2.depth[n] = 0;
  1131. } else {
  1132. tree[n].Len = 0;
  1133. }
  1134. }
  1135. /* The pkzip format requires that at least one distance code exists,
  1136. * and that at least one bit should be sent even if there is only one
  1137. * possible code. So to avoid special checks later on we force at least
  1138. * two codes of non zero frequency.
  1139. */
  1140. while (G2.heap_len < 2) {
  1141. int new = G2.heap[++G2.heap_len] = (max_code < 2 ? ++max_code : 0);
  1142. tree[new].Freq = 1;
  1143. G2.depth[new] = 0;
  1144. G2.opt_len--;
  1145. if (stree)
  1146. G2.static_len -= stree[new].Len;
  1147. /* new is 0 or 1 so it does not have extra bits */
  1148. }
  1149. desc->max_code = max_code;
  1150. /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree,
  1151. * establish sub-heaps of increasing lengths:
  1152. */
  1153. for (n = G2.heap_len / 2; n >= 1; n--)
  1154. pqdownheap(tree, n);
  1155. /* Construct the Huffman tree by repeatedly combining the least two
  1156. * frequent nodes.
  1157. */
  1158. do {
  1159. PQREMOVE(tree, n); /* n = node of least frequency */
  1160. m = G2.heap[SMALLEST]; /* m = node of next least frequency */
  1161. G2.heap[--G2.heap_max] = n; /* keep the nodes sorted by frequency */
  1162. G2.heap[--G2.heap_max] = m;
  1163. /* Create a new node father of n and m */
  1164. tree[node].Freq = tree[n].Freq + tree[m].Freq;
  1165. G2.depth[node] = MAX(G2.depth[n], G2.depth[m]) + 1;
  1166. tree[n].Dad = tree[m].Dad = (ush) node;
  1167. #ifdef DUMP_BL_TREE
  1168. if (tree == G2.bl_tree) {
  1169. bb_error_msg("\nnode %d(%d), sons %d(%d) %d(%d)",
  1170. node, tree[node].Freq, n, tree[n].Freq, m, tree[m].Freq);
  1171. }
  1172. #endif
  1173. /* and insert the new node in the heap */
  1174. G2.heap[SMALLEST] = node++;
  1175. pqdownheap(tree, SMALLEST);
  1176. } while (G2.heap_len >= 2);
  1177. G2.heap[--G2.heap_max] = G2.heap[SMALLEST];
  1178. /* At this point, the fields freq and dad are set. We can now
  1179. * generate the bit lengths.
  1180. */
  1181. gen_bitlen((tree_desc *) desc);
  1182. /* The field len is now set, we can generate the bit codes */
  1183. gen_codes((ct_data *) tree, max_code);
  1184. }
  1185. /* ===========================================================================
  1186. * Scan a literal or distance tree to determine the frequencies of the codes
  1187. * in the bit length tree. Updates opt_len to take into account the repeat
  1188. * counts. (The contribution of the bit length codes will be added later
  1189. * during the construction of bl_tree.)
  1190. */
  1191. static void scan_tree(ct_data * tree, int max_code)
  1192. {
  1193. int n; /* iterates over all tree elements */
  1194. int prevlen = -1; /* last emitted length */
  1195. int curlen; /* length of current code */
  1196. int nextlen = tree[0].Len; /* length of next code */
  1197. int count = 0; /* repeat count of the current code */
  1198. int max_count = 7; /* max repeat count */
  1199. int min_count = 4; /* min repeat count */
  1200. if (nextlen == 0) {
  1201. max_count = 138;
  1202. min_count = 3;
  1203. }
  1204. tree[max_code + 1].Len = 0xffff; /* guard */
  1205. for (n = 0; n <= max_code; n++) {
  1206. curlen = nextlen;
  1207. nextlen = tree[n + 1].Len;
  1208. if (++count < max_count && curlen == nextlen)
  1209. continue;
  1210. if (count < min_count) {
  1211. G2.bl_tree[curlen].Freq += count;
  1212. } else if (curlen != 0) {
  1213. if (curlen != prevlen)
  1214. G2.bl_tree[curlen].Freq++;
  1215. G2.bl_tree[REP_3_6].Freq++;
  1216. } else if (count <= 10) {
  1217. G2.bl_tree[REPZ_3_10].Freq++;
  1218. } else {
  1219. G2.bl_tree[REPZ_11_138].Freq++;
  1220. }
  1221. count = 0;
  1222. prevlen = curlen;
  1223. max_count = 7;
  1224. min_count = 4;
  1225. if (nextlen == 0) {
  1226. max_count = 138;
  1227. min_count = 3;
  1228. } else if (curlen == nextlen) {
  1229. max_count = 6;
  1230. min_count = 3;
  1231. }
  1232. }
  1233. }
  1234. /* ===========================================================================
  1235. * Send a literal or distance tree in compressed form, using the codes in
  1236. * bl_tree.
  1237. */
  1238. static void send_tree(ct_data * tree, int max_code)
  1239. {
  1240. int n; /* iterates over all tree elements */
  1241. int prevlen = -1; /* last emitted length */
  1242. int curlen; /* length of current code */
  1243. int nextlen = tree[0].Len; /* length of next code */
  1244. int count = 0; /* repeat count of the current code */
  1245. int max_count = 7; /* max repeat count */
  1246. int min_count = 4; /* min repeat count */
  1247. /* tree[max_code+1].Len = -1; *//* guard already set */
  1248. if (nextlen == 0)
  1249. max_count = 138, min_count = 3;
  1250. for (n = 0; n <= max_code; n++) {
  1251. curlen = nextlen;
  1252. nextlen = tree[n + 1].Len;
  1253. if (++count < max_count && curlen == nextlen) {
  1254. continue;
  1255. } else if (count < min_count) {
  1256. do {
  1257. SEND_CODE(curlen, G2.bl_tree);
  1258. } while (--count);
  1259. } else if (curlen != 0) {
  1260. if (curlen != prevlen) {
  1261. SEND_CODE(curlen, G2.bl_tree);
  1262. count--;
  1263. }
  1264. Assert(count >= 3 && count <= 6, " 3_6?");
  1265. SEND_CODE(REP_3_6, G2.bl_tree);
  1266. send_bits(count - 3, 2);
  1267. } else if (count <= 10) {
  1268. SEND_CODE(REPZ_3_10, G2.bl_tree);
  1269. send_bits(count - 3, 3);
  1270. } else {
  1271. SEND_CODE(REPZ_11_138, G2.bl_tree);
  1272. send_bits(count - 11, 7);
  1273. }
  1274. count = 0;
  1275. prevlen = curlen;
  1276. if (nextlen == 0) {
  1277. max_count = 138;
  1278. min_count = 3;
  1279. } else if (curlen == nextlen) {
  1280. max_count = 6;
  1281. min_count = 3;
  1282. } else {
  1283. max_count = 7;
  1284. min_count = 4;
  1285. }
  1286. }
  1287. }
  1288. /* ===========================================================================
  1289. * Construct the Huffman tree for the bit lengths and return the index in
  1290. * bl_order of the last bit length code to send.
  1291. */
  1292. static int build_bl_tree(void)
  1293. {
  1294. int max_blindex; /* index of last bit length code of non zero freq */
  1295. /* Determine the bit length frequencies for literal and distance trees */
  1296. scan_tree(G2.dyn_ltree, G2.l_desc.max_code);
  1297. scan_tree(G2.dyn_dtree, G2.d_desc.max_code);
  1298. /* Build the bit length tree: */
  1299. build_tree(&G2.bl_desc);
  1300. /* opt_len now includes the length of the tree representations, except
  1301. * the lengths of the bit lengths codes and the 5+5+4 bits for the counts.
  1302. */
  1303. /* Determine the number of bit length codes to send. The pkzip format
  1304. * requires that at least 4 bit length codes be sent. (appnote.txt says
  1305. * 3 but the actual value used is 4.)
  1306. */
  1307. for (max_blindex = BL_CODES - 1; max_blindex >= 3; max_blindex--) {
  1308. if (G2.bl_tree[bl_order[max_blindex]].Len != 0)
  1309. break;
  1310. }
  1311. /* Update opt_len to include the bit length tree and counts */
  1312. G2.opt_len += 3 * (max_blindex + 1) + 5 + 5 + 4;
  1313. Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", G2.opt_len, G2.static_len));
  1314. return max_blindex;
  1315. }
  1316. /* ===========================================================================
  1317. * Send the header for a block using dynamic Huffman trees: the counts, the
  1318. * lengths of the bit length codes, the literal tree and the distance tree.
  1319. * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4.
  1320. */
  1321. static void send_all_trees(int lcodes, int dcodes, int blcodes)
  1322. {
  1323. int rank; /* index in bl_order */
  1324. Assert(lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes");
  1325. Assert(lcodes <= L_CODES && dcodes <= D_CODES
  1326. && blcodes <= BL_CODES, "too many codes");
  1327. Tracev((stderr, "\nbl counts: "));
  1328. send_bits(lcodes - 257, 5); /* not +255 as stated in appnote.txt */
  1329. send_bits(dcodes - 1, 5);
  1330. send_bits(blcodes - 4, 4); /* not -3 as stated in appnote.txt */
  1331. for (rank = 0; rank < blcodes; rank++) {
  1332. Tracev((stderr, "\nbl code %2d ", bl_order[rank]));
  1333. send_bits(G2.bl_tree[bl_order[rank]].Len, 3);
  1334. }
  1335. Tracev((stderr, "\nbl tree: sent %ld", G1.bits_sent));
  1336. send_tree((ct_data *) G2.dyn_ltree, lcodes - 1); /* send the literal tree */
  1337. Tracev((stderr, "\nlit tree: sent %ld", G1.bits_sent));
  1338. send_tree((ct_data *) G2.dyn_dtree, dcodes - 1); /* send the distance tree */
  1339. Tracev((stderr, "\ndist tree: sent %ld", G1.bits_sent));
  1340. }
  1341. /* ===========================================================================
  1342. * Save the match info and tally the frequency counts. Return true if
  1343. * the current block must be flushed.
  1344. */
  1345. static int ct_tally(int dist, int lc)
  1346. {
  1347. G1.l_buf[G2.last_lit++] = lc;
  1348. if (dist == 0) {
  1349. /* lc is the unmatched char */
  1350. G2.dyn_ltree[lc].Freq++;
  1351. } else {
  1352. /* Here, lc is the match length - MIN_MATCH */
  1353. dist--; /* dist = match distance - 1 */
  1354. Assert((ush) dist < (ush) MAX_DIST
  1355. && (ush) lc <= (ush) (MAX_MATCH - MIN_MATCH)
  1356. && (ush) D_CODE(dist) < (ush) D_CODES, "ct_tally: bad match"
  1357. );
  1358. G2.dyn_ltree[G2.length_code[lc] + LITERALS + 1].Freq++;
  1359. G2.dyn_dtree[D_CODE(dist)].Freq++;
  1360. G1.d_buf[G2.last_dist++] = dist;
  1361. G2.flags |= G2.flag_bit;
  1362. }
  1363. G2.flag_bit <<= 1;
  1364. /* Output the flags if they fill a byte: */
  1365. if ((G2.last_lit & 7) == 0) {
  1366. G2.flag_buf[G2.last_flags++] = G2.flags;
  1367. G2.flags = 0;
  1368. G2.flag_bit = 1;
  1369. }
  1370. /* Try to guess if it is profitable to stop the current block here */
  1371. if ((G2.last_lit & 0xfff) == 0) {
  1372. /* Compute an upper bound for the compressed length */
  1373. ulg out_length = G2.last_lit * 8L;
  1374. ulg in_length = (ulg) G1.strstart - G1.block_start;
  1375. int dcode;
  1376. for (dcode = 0; dcode < D_CODES; dcode++) {
  1377. out_length += G2.dyn_dtree[dcode].Freq * (5L + extra_dbits[dcode]);
  1378. }
  1379. out_length >>= 3;
  1380. Trace((stderr,
  1381. "\nlast_lit %u, last_dist %u, in %ld, out ~%ld(%ld%%) ",
  1382. G2.last_lit, G2.last_dist, in_length, out_length,
  1383. 100L - out_length * 100L / in_length));
  1384. if (G2.last_dist < G2.last_lit / 2 && out_length < in_length / 2)
  1385. return 1;
  1386. }
  1387. return (G2.last_lit == LIT_BUFSIZE - 1 || G2.last_dist == DIST_BUFSIZE);
  1388. /* We avoid equality with LIT_BUFSIZE because of wraparound at 64K
  1389. * on 16 bit machines and because stored blocks are restricted to
  1390. * 64K-1 bytes.
  1391. */
  1392. }
  1393. /* ===========================================================================
  1394. * Send the block data compressed using the given Huffman trees
  1395. */
  1396. static void compress_block(ct_data * ltree, ct_data * dtree)
  1397. {
  1398. unsigned dist; /* distance of matched string */
  1399. int lc; /* match length or unmatched char (if dist == 0) */
  1400. unsigned lx = 0; /* running index in l_buf */
  1401. unsigned dx = 0; /* running index in d_buf */
  1402. unsigned fx = 0; /* running index in flag_buf */
  1403. uch flag = 0; /* current flags */
  1404. unsigned code; /* the code to send */
  1405. int extra; /* number of extra bits to send */
  1406. if (G2.last_lit != 0) do {
  1407. if ((lx & 7) == 0)
  1408. flag = G2.flag_buf[fx++];
  1409. lc = G1.l_buf[lx++];
  1410. if ((flag & 1) == 0) {
  1411. SEND_CODE(lc, ltree); /* send a literal byte */
  1412. Tracecv(lc > ' ', (stderr, " '%c' ", lc));
  1413. } else {
  1414. /* Here, lc is the match length - MIN_MATCH */
  1415. code = G2.length_code[lc];
  1416. SEND_CODE(code + LITERALS + 1, ltree); /* send the length code */
  1417. extra = extra_lbits[code];
  1418. if (extra != 0) {
  1419. lc -= G2.base_length[code];
  1420. send_bits(lc, extra); /* send the extra length bits */
  1421. }
  1422. dist = G1.d_buf[dx++];
  1423. /* Here, dist is the match distance - 1 */
  1424. code = D_CODE(dist);
  1425. Assert(code < D_CODES, "bad d_code");
  1426. SEND_CODE(code, dtree); /* send the distance code */
  1427. extra = extra_dbits[code];
  1428. if (extra != 0) {
  1429. dist -= G2.base_dist[code];
  1430. send_bits(dist, extra); /* send the extra distance bits */
  1431. }
  1432. } /* literal or match pair ? */
  1433. flag >>= 1;
  1434. } while (lx < G2.last_lit);
  1435. SEND_CODE(END_BLOCK, ltree);
  1436. }
  1437. /* ===========================================================================
  1438. * Determine the best encoding for the current block: dynamic trees, static
  1439. * trees or store, and output the encoded block to the zip file. This function
  1440. * returns the total compressed length for the file so far.
  1441. */
  1442. static ulg flush_block(char *buf, ulg stored_len, int eof)
  1443. {
  1444. ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
  1445. int max_blindex; /* index of last bit length code of non zero freq */
  1446. G2.flag_buf[G2.last_flags] = G2.flags; /* Save the flags for the last 8 items */
  1447. /* Construct the literal and distance trees */
  1448. build_tree(&G2.l_desc);
  1449. Tracev((stderr, "\nlit data: dyn %ld, stat %ld", G2.opt_len, G2.static_len));
  1450. build_tree(&G2.d_desc);
  1451. Tracev((stderr, "\ndist data: dyn %ld, stat %ld", G2.opt_len, G2.static_len));
  1452. /* At this point, opt_len and static_len are the total bit lengths of
  1453. * the compressed block data, excluding the tree representations.
  1454. */
  1455. /* Build the bit length tree for the above two trees, and get the index
  1456. * in bl_order of the last bit length code to send.
  1457. */
  1458. max_blindex = build_bl_tree();
  1459. /* Determine the best encoding. Compute first the block length in bytes */
  1460. opt_lenb = (G2.opt_len + 3 + 7) >> 3;
  1461. static_lenb = (G2.static_len + 3 + 7) >> 3;
  1462. Trace((stderr,
  1463. "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u dist %u ",
  1464. opt_lenb, G2.opt_len, static_lenb, G2.static_len, stored_len,
  1465. G2.last_lit, G2.last_dist));
  1466. if (static_lenb <= opt_lenb)
  1467. opt_lenb = static_lenb;
  1468. /* If compression failed and this is the first and last block,
  1469. * and if the zip file can be seeked (to rewrite the local header),
  1470. * the whole file is transformed into a stored file:
  1471. */
  1472. if (stored_len <= opt_lenb && eof && G2.compressed_len == 0L && seekable()) {
  1473. /* Since LIT_BUFSIZE <= 2*WSIZE, the input data must be there: */
  1474. if (buf == NULL)
  1475. bb_error_msg("block vanished");
  1476. copy_block(buf, (unsigned) stored_len, 0); /* without header */
  1477. G2.compressed_len = stored_len << 3;
  1478. } else if (stored_len + 4 <= opt_lenb && buf != NULL) {
  1479. /* 4: two words for the lengths */
  1480. /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE.
  1481. * Otherwise we can't have processed more than WSIZE input bytes since
  1482. * the last block flush, because compression would have been
  1483. * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to
  1484. * transform a block into a stored block.
  1485. */
  1486. send_bits((STORED_BLOCK << 1) + eof, 3); /* send block type */
  1487. G2.compressed_len = (G2.compressed_len + 3 + 7) & ~7L;
  1488. G2.compressed_len += (stored_len + 4) << 3;
  1489. copy_block(buf, (unsigned) stored_len, 1); /* with header */
  1490. } else if (static_lenb == opt_lenb) {
  1491. send_bits((STATIC_TREES << 1) + eof, 3);
  1492. compress_block((ct_data *) G2.static_ltree, (ct_data *) G2.static_dtree);
  1493. G2.compressed_len += 3 + G2.static_len;
  1494. } else {
  1495. send_bits((DYN_TREES << 1) + eof, 3);
  1496. send_all_trees(G2.l_desc.max_code + 1, G2.d_desc.max_code + 1,
  1497. max_blindex + 1);
  1498. compress_block((ct_data *) G2.dyn_ltree, (ct_data *) G2.dyn_dtree);
  1499. G2.compressed_len += 3 + G2.opt_len;
  1500. }
  1501. Assert(G2.compressed_len == G1.bits_sent, "bad compressed size");
  1502. init_block();
  1503. if (eof) {
  1504. bi_windup();
  1505. G2.compressed_len += 7; /* align on byte boundary */
  1506. }
  1507. Tracev((stderr, "\ncomprlen %lu(%lu) ", G2.compressed_len >> 3,
  1508. G2.compressed_len - 7 * eof));
  1509. return G2.compressed_len >> 3;
  1510. }
  1511. /* ===========================================================================
  1512. * Update a hash value with the given input byte
  1513. * IN assertion: all calls to UPDATE_HASH are made with consecutive
  1514. * input characters, so that a running hash key can be computed from the
  1515. * previous key instead of complete recalculation each time.
  1516. */
  1517. #define UPDATE_HASH(h, c) (h = (((h)<<H_SHIFT) ^ (c)) & HASH_MASK)
  1518. /* ===========================================================================
  1519. * Same as above, but achieves better compression. We use a lazy
  1520. * evaluation for matches: a match is finally adopted only if there is
  1521. * no better match at the next window position.
  1522. *
  1523. * Processes a new input file and return its compressed length. Sets
  1524. * the compressed length, crc, deflate flags and internal file
  1525. * attributes.
  1526. */
  1527. /* Flush the current block, with given end-of-file flag.
  1528. * IN assertion: strstart is set to the end of the current match. */
  1529. #define FLUSH_BLOCK(eof) \
  1530. flush_block( \
  1531. G1.block_start >= 0L \
  1532. ? (char*)&G1.window[(unsigned)G1.block_start] \
  1533. : (char*)NULL, \
  1534. (ulg)G1.strstart - G1.block_start, \
  1535. (eof) \
  1536. )
  1537. /* Insert string s in the dictionary and set match_head to the previous head
  1538. * of the hash chain (the most recent string with same hash key). Return
  1539. * the previous length of the hash chain.
  1540. * IN assertion: all calls to INSERT_STRING are made with consecutive
  1541. * input characters and the first MIN_MATCH bytes of s are valid
  1542. * (except for the last MIN_MATCH-1 bytes of the input file). */
  1543. #define INSERT_STRING(s, match_head) \
  1544. do { \
  1545. UPDATE_HASH(G1.ins_h, G1.window[(s) + MIN_MATCH-1]); \
  1546. G1.prev[(s) & WMASK] = match_head = head[G1.ins_h]; \
  1547. head[G1.ins_h] = (s); \
  1548. } while (0)
  1549. static ulg deflate(void)
  1550. {
  1551. IPos hash_head; /* head of hash chain */
  1552. IPos prev_match; /* previous match */
  1553. int flush; /* set if current block must be flushed */
  1554. int match_available = 0; /* set if previous match exists */
  1555. unsigned match_length = MIN_MATCH - 1; /* length of best match */
  1556. /* Process the input block. */
  1557. while (G1.lookahead != 0) {
  1558. /* Insert the string window[strstart .. strstart+2] in the
  1559. * dictionary, and set hash_head to the head of the hash chain:
  1560. */
  1561. INSERT_STRING(G1.strstart, hash_head);
  1562. /* Find the longest match, discarding those <= prev_length.
  1563. */
  1564. G1.prev_length = match_length;
  1565. prev_match = G1.match_start;
  1566. match_length = MIN_MATCH - 1;
  1567. if (hash_head != 0 && G1.prev_length < max_lazy_match
  1568. && G1.strstart - hash_head <= MAX_DIST
  1569. ) {
  1570. /* To simplify the code, we prevent matches with the string
  1571. * of window index 0 (in particular we have to avoid a match
  1572. * of the string with itself at the start of the input file).
  1573. */
  1574. match_length = longest_match(hash_head);
  1575. /* longest_match() sets match_start */
  1576. if (match_length > G1.lookahead)
  1577. match_length = G1.lookahead;
  1578. /* Ignore a length 3 match if it is too distant: */
  1579. if (match_length == MIN_MATCH && G1.strstart - G1.match_start > TOO_FAR) {
  1580. /* If prev_match is also MIN_MATCH, G1.match_start is garbage
  1581. * but we will ignore the current match anyway.
  1582. */
  1583. match_length--;
  1584. }
  1585. }
  1586. /* If there was a match at the previous step and the current
  1587. * match is not better, output the previous match:
  1588. */
  1589. if (G1.prev_length >= MIN_MATCH && match_length <= G1.prev_length) {
  1590. check_match(G1.strstart - 1, prev_match, G1.prev_length);
  1591. flush = ct_tally(G1.strstart - 1 - prev_match, G1.prev_length - MIN_MATCH);
  1592. /* Insert in hash table all strings up to the end of the match.
  1593. * strstart-1 and strstart are already inserted.
  1594. */
  1595. G1.lookahead -= G1.prev_length - 1;
  1596. G1.prev_length -= 2;
  1597. do {
  1598. G1.strstart++;
  1599. INSERT_STRING(G1.strstart, hash_head);
  1600. /* strstart never exceeds WSIZE-MAX_MATCH, so there are
  1601. * always MIN_MATCH bytes ahead. If lookahead < MIN_MATCH
  1602. * these bytes are garbage, but it does not matter since the
  1603. * next lookahead bytes will always be emitted as literals.
  1604. */
  1605. } while (--G1.prev_length != 0);
  1606. match_available = 0;
  1607. match_length = MIN_MATCH - 1;
  1608. G1.strstart++;
  1609. if (flush) {
  1610. FLUSH_BLOCK(0);
  1611. G1.block_start = G1.strstart;
  1612. }
  1613. } else if (match_available) {
  1614. /* If there was no match at the previous position, output a
  1615. * single literal. If there was a match but the current match
  1616. * is longer, truncate the previous match to a single literal.
  1617. */
  1618. Tracevv((stderr, "%c", G1.window[G1.strstart - 1]));
  1619. if (ct_tally(0, G1.window[G1.strstart - 1])) {
  1620. FLUSH_BLOCK(0);
  1621. G1.block_start = G1.strstart;
  1622. }
  1623. G1.strstart++;
  1624. G1.lookahead--;
  1625. } else {
  1626. /* There is no previous match to compare with, wait for
  1627. * the next step to decide.
  1628. */
  1629. match_available = 1;
  1630. G1.strstart++;
  1631. G1.lookahead--;
  1632. }
  1633. Assert(G1.strstart <= G1.isize && lookahead <= G1.isize, "a bit too far");
  1634. /* Make sure that we always have enough lookahead, except
  1635. * at the end of the input file. We need MAX_MATCH bytes
  1636. * for the next match, plus MIN_MATCH bytes to insert the
  1637. * string following the next match.
  1638. */
  1639. while (G1.lookahead < MIN_LOOKAHEAD && !G1.eofile)
  1640. fill_window();
  1641. }
  1642. if (match_available)
  1643. ct_tally(0, G1.window[G1.strstart - 1]);
  1644. return FLUSH_BLOCK(1); /* eof */
  1645. }
  1646. /* ===========================================================================
  1647. * Initialize the bit string routines.
  1648. */
  1649. static void bi_init(void)
  1650. {
  1651. G1.bi_buf = 0;
  1652. G1.bi_valid = 0;
  1653. #ifdef DEBUG
  1654. G1.bits_sent = 0L;
  1655. #endif
  1656. }
  1657. /* ===========================================================================
  1658. * Initialize the "longest match" routines for a new file
  1659. */
  1660. static void lm_init(ush * flagsp)
  1661. {
  1662. unsigned j;
  1663. /* Initialize the hash table. */
  1664. memset(head, 0, HASH_SIZE * sizeof(*head));
  1665. /* prev will be initialized on the fly */
  1666. /* speed options for the general purpose bit flag */
  1667. *flagsp |= 2; /* FAST 4, SLOW 2 */
  1668. /* ??? reduce max_chain_length for binary files */
  1669. G1.strstart = 0;
  1670. G1.block_start = 0L;
  1671. G1.lookahead = file_read(G1.window,
  1672. sizeof(int) <= 2 ? (unsigned) WSIZE : 2 * WSIZE);
  1673. if (G1.lookahead == 0 || G1.lookahead == (unsigned) -1) {
  1674. G1.eofile = 1;
  1675. G1.lookahead = 0;
  1676. return;
  1677. }
  1678. G1.eofile = 0;
  1679. /* Make sure that we always have enough lookahead. This is important
  1680. * if input comes from a device such as a tty.
  1681. */
  1682. while (G1.lookahead < MIN_LOOKAHEAD && !G1.eofile)
  1683. fill_window();
  1684. G1.ins_h = 0;
  1685. for (j = 0; j < MIN_MATCH - 1; j++)
  1686. UPDATE_HASH(G1.ins_h, G1.window[j]);
  1687. /* If lookahead < MIN_MATCH, ins_h is garbage, but this is
  1688. * not important since only literal bytes will be emitted.
  1689. */
  1690. }
  1691. /* ===========================================================================
  1692. * Allocate the match buffer, initialize the various tables and save the
  1693. * location of the internal file attribute (ascii/binary) and method
  1694. * (DEFLATE/STORE).
  1695. * One callsite in zip()
  1696. */
  1697. static void ct_init(void)
  1698. {
  1699. int n; /* iterates over tree elements */
  1700. int length; /* length value */
  1701. int code; /* code value */
  1702. int dist; /* distance index */
  1703. G2.compressed_len = 0L;
  1704. #ifdef NOT_NEEDED
  1705. if (G2.static_dtree[0].Len != 0)
  1706. return; /* ct_init already called */
  1707. #endif
  1708. /* Initialize the mapping length (0..255) -> length code (0..28) */
  1709. length = 0;
  1710. for (code = 0; code < LENGTH_CODES - 1; code++) {
  1711. G2.base_length[code] = length;
  1712. for (n = 0; n < (1 << extra_lbits[code]); n++) {
  1713. G2.length_code[length++] = code;
  1714. }
  1715. }
  1716. Assert(length == 256, "ct_init: length != 256");
  1717. /* Note that the length 255 (match length 258) can be represented
  1718. * in two different ways: code 284 + 5 bits or code 285, so we
  1719. * overwrite length_code[255] to use the best encoding:
  1720. */
  1721. G2.length_code[length - 1] = code;
  1722. /* Initialize the mapping dist (0..32K) -> dist code (0..29) */
  1723. dist = 0;
  1724. for (code = 0; code < 16; code++) {
  1725. G2.base_dist[code] = dist;
  1726. for (n = 0; n < (1 << extra_dbits[code]); n++) {
  1727. G2.dist_code[dist++] = code;
  1728. }
  1729. }
  1730. Assert(dist == 256, "ct_init: dist != 256");
  1731. dist >>= 7; /* from now on, all distances are divided by 128 */
  1732. for (; code < D_CODES; code++) {
  1733. G2.base_dist[code] = dist << 7;
  1734. for (n = 0; n < (1 << (extra_dbits[code] - 7)); n++) {
  1735. G2.dist_code[256 + dist++] = code;
  1736. }
  1737. }
  1738. Assert(dist == 256, "ct_init: 256+dist != 512");
  1739. /* Construct the codes of the static literal tree */
  1740. /* already zeroed - it's in bss
  1741. for (n = 0; n <= MAX_BITS; n++)
  1742. G2.bl_count[n] = 0; */
  1743. n = 0;
  1744. while (n <= 143) {
  1745. G2.static_ltree[n++].Len = 8;
  1746. G2.bl_count[8]++;
  1747. }
  1748. while (n <= 255) {
  1749. G2.static_ltree[n++].Len = 9;
  1750. G2.bl_count[9]++;
  1751. }
  1752. while (n <= 279) {
  1753. G2.static_ltree[n++].Len = 7;
  1754. G2.bl_count[7]++;
  1755. }
  1756. while (n <= 287) {
  1757. G2.static_ltree[n++].Len = 8;
  1758. G2.bl_count[8]++;
  1759. }
  1760. /* Codes 286 and 287 do not exist, but we must include them in the
  1761. * tree construction to get a canonical Huffman tree (longest code
  1762. * all ones)
  1763. */
  1764. gen_codes((ct_data *) G2.static_ltree, L_CODES + 1);
  1765. /* The static distance tree is trivial: */
  1766. for (n = 0; n < D_CODES; n++) {
  1767. G2.static_dtree[n].Len = 5;
  1768. G2.static_dtree[n].Code = bi_reverse(n, 5);
  1769. }
  1770. /* Initialize the first block of the first file: */
  1771. init_block();
  1772. }
  1773. /* ===========================================================================
  1774. * Deflate in to out.
  1775. * IN assertions: the input and output buffers are cleared.
  1776. */
  1777. static void zip(void)
  1778. {
  1779. ush deflate_flags = 0; /* pkzip -es, -en or -ex equivalent */
  1780. G1.outcnt = 0;
  1781. /* Write the header to the gzip file. See algorithm.doc for the format */
  1782. /* magic header for gzip files: 1F 8B */
  1783. /* compression method: 8 (DEFLATED) */
  1784. /* general flags: 0 */
  1785. put_32bit(0x00088b1f);
  1786. put_32bit(0); /* Unix timestamp */
  1787. /* Write deflated file to zip file */
  1788. G1.crc = ~0;
  1789. bi_init();
  1790. ct_init();
  1791. lm_init(&deflate_flags);
  1792. put_8bit(deflate_flags); /* extra flags */
  1793. put_8bit(3); /* OS identifier = 3 (Unix) */
  1794. deflate();
  1795. /* Write the crc and uncompressed size */
  1796. put_32bit(~G1.crc);
  1797. put_32bit(G1.isize);
  1798. flush_outbuf();
  1799. }
  1800. /* ======================================================================== */
  1801. static
  1802. IF_DESKTOP(long long) int FAST_FUNC pack_gzip(transformer_state_t *xstate UNUSED_PARAM)
  1803. {
  1804. /* Clear input and output buffers */
  1805. G1.outcnt = 0;
  1806. #ifdef DEBUG
  1807. G1.insize = 0;
  1808. #endif
  1809. G1.isize = 0;
  1810. /* Reinit G2.xxx */
  1811. memset(&G2, 0, sizeof(G2));
  1812. G2.l_desc.dyn_tree = G2.dyn_ltree;
  1813. G2.l_desc.static_tree = G2.static_ltree;
  1814. G2.l_desc.extra_bits = extra_lbits;
  1815. G2.l_desc.extra_base = LITERALS + 1;
  1816. G2.l_desc.elems = L_CODES;
  1817. G2.l_desc.max_length = MAX_BITS;
  1818. //G2.l_desc.max_code = 0;
  1819. G2.d_desc.dyn_tree = G2.dyn_dtree;
  1820. G2.d_desc.static_tree = G2.static_dtree;
  1821. G2.d_desc.extra_bits = extra_dbits;
  1822. //G2.d_desc.extra_base = 0;
  1823. G2.d_desc.elems = D_CODES;
  1824. G2.d_desc.max_length = MAX_BITS;
  1825. //G2.d_desc.max_code = 0;
  1826. G2.bl_desc.dyn_tree = G2.bl_tree;
  1827. //G2.bl_desc.static_tree = NULL;
  1828. G2.bl_desc.extra_bits = extra_blbits,
  1829. //G2.bl_desc.extra_base = 0;
  1830. G2.bl_desc.elems = BL_CODES;
  1831. G2.bl_desc.max_length = MAX_BL_BITS;
  1832. //G2.bl_desc.max_code = 0;
  1833. #if 0
  1834. /* Saving of timestamp is disabled. Why?
  1835. * - it is not Y2038-safe.
  1836. * - some people want deterministic results
  1837. * (normally they'd use -n, but our -n is a nop).
  1838. * - it's bloat.
  1839. * Per RFC 1952, gzfile.time=0 is "no timestamp".
  1840. * If users will demand this to be reinstated,
  1841. * implement -n "don't save timestamp".
  1842. */
  1843. struct stat s;
  1844. s.st_ctime = 0;
  1845. fstat(STDIN_FILENO, &s);
  1846. zip(s.st_ctime);
  1847. #else
  1848. zip();
  1849. #endif
  1850. return 0;
  1851. }
  1852. #if ENABLE_FEATURE_GZIP_LONG_OPTIONS
  1853. static const char gzip_longopts[] ALIGN1 =
  1854. "stdout\0" No_argument "c"
  1855. "to-stdout\0" No_argument "c"
  1856. "force\0" No_argument "f"
  1857. "verbose\0" No_argument "v"
  1858. #if ENABLE_FEATURE_GZIP_DECOMPRESS
  1859. "decompress\0" No_argument "d"
  1860. "uncompress\0" No_argument "d"
  1861. "test\0" No_argument "t"
  1862. #endif
  1863. "quiet\0" No_argument "q"
  1864. "fast\0" No_argument "1"
  1865. "best\0" No_argument "9"
  1866. "no-name\0" No_argument "n"
  1867. ;
  1868. #endif
  1869. /*
  1870. * Linux kernel build uses gzip -d -n. We accept and ignore -n.
  1871. * Man page says:
  1872. * -n --no-name
  1873. * gzip: do not save the original file name and time stamp.
  1874. * (The original name is always saved if the name had to be truncated.)
  1875. * gunzip: do not restore the original file name/time even if present
  1876. * (remove only the gzip suffix from the compressed file name).
  1877. * This option is the default when decompressing.
  1878. * -N --name
  1879. * gzip: always save the original file name and time stamp (this is the default)
  1880. * gunzip: restore the original file name and time stamp if present.
  1881. */
  1882. int gzip_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
  1883. #if ENABLE_FEATURE_GZIP_DECOMPRESS
  1884. int gzip_main(int argc, char **argv)
  1885. #else
  1886. int gzip_main(int argc UNUSED_PARAM, char **argv)
  1887. #endif
  1888. {
  1889. unsigned opt;
  1890. #if ENABLE_FEATURE_GZIP_LEVELS
  1891. static const struct {
  1892. uint8_t good;
  1893. uint8_t chain_shift;
  1894. uint8_t lazy2;
  1895. uint8_t nice2;
  1896. } gzip_level_config[6] = {
  1897. {4, 4, 4/2, 16/2}, /* Level 4 */
  1898. {8, 5, 16/2, 32/2}, /* Level 5 */
  1899. {8, 7, 16/2, 128/2}, /* Level 6 */
  1900. {8, 8, 32/2, 128/2}, /* Level 7 */
  1901. {32, 10, 128/2, 258/2}, /* Level 8 */
  1902. {32, 12, 258/2, 258/2}, /* Level 9 */
  1903. };
  1904. #endif
  1905. SET_PTR_TO_GLOBALS((char *)xzalloc(sizeof(struct globals)+sizeof(struct globals2))
  1906. + sizeof(struct globals));
  1907. /* Must match bbunzip's constants OPT_STDOUT, OPT_FORCE! */
  1908. #if ENABLE_FEATURE_GZIP_LONG_OPTIONS
  1909. opt = getopt32long(argv, "cfkv" IF_FEATURE_GZIP_DECOMPRESS("dt") "qn123456789", gzip_longopts);
  1910. #else
  1911. opt = getopt32(argv, "cfkv" IF_FEATURE_GZIP_DECOMPRESS("dt") "qn123456789");
  1912. #endif
  1913. #if ENABLE_FEATURE_GZIP_DECOMPRESS /* gunzip_main may not be visible... */
  1914. if (opt & 0x30) // -d and/or -t
  1915. return gunzip_main(argc, argv);
  1916. #endif
  1917. #if ENABLE_FEATURE_GZIP_LEVELS
  1918. opt >>= ENABLE_FEATURE_GZIP_DECOMPRESS ? 8 : 6; /* drop cfkv[dt]qn bits */
  1919. if (opt == 0)
  1920. opt = 1 << 6; /* default: 6 */
  1921. opt = ffs(opt >> 4); /* Maps -1..-4 to [0], -5 to [1] ... -9 to [5] */
  1922. max_chain_length = 1 << gzip_level_config[opt].chain_shift;
  1923. good_match = gzip_level_config[opt].good;
  1924. max_lazy_match = gzip_level_config[opt].lazy2 * 2;
  1925. nice_match = gzip_level_config[opt].nice2 * 2;
  1926. #endif
  1927. option_mask32 &= 0xf; /* retain only -cfkv */
  1928. /* Allocate all global buffers (for DYN_ALLOC option) */
  1929. ALLOC(uch, G1.l_buf, INBUFSIZ);
  1930. ALLOC(uch, G1.outbuf, OUTBUFSIZ);
  1931. ALLOC(ush, G1.d_buf, DIST_BUFSIZE);
  1932. ALLOC(uch, G1.window, 2L * WSIZE);
  1933. ALLOC(ush, G1.prev, 1L << BITS);
  1934. /* Initialize the CRC32 table */
  1935. global_crc32_table = crc32_filltable(NULL, 0);
  1936. argv += optind;
  1937. return bbunpack(argv, pack_gzip, append_ext, "gz");
  1938. }