gcm128.c 54 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888
  1. /*
  2. * Copyright 2010-2018 The OpenSSL Project Authors. All Rights Reserved.
  3. *
  4. * Licensed under the OpenSSL license (the "License"). You may not use
  5. * this file except in compliance with the License. You can obtain a copy
  6. * in the file LICENSE in the source distribution or at
  7. * https://www.openssl.org/source/license.html
  8. */
  9. #include <openssl/crypto.h>
  10. #include "modes_lcl.h"
  11. #include <string.h>
  12. #if defined(BSWAP4) && defined(STRICT_ALIGNMENT)
  13. /* redefine, because alignment is ensured */
  14. # undef GETU32
  15. # define GETU32(p) BSWAP4(*(const u32 *)(p))
  16. # undef PUTU32
  17. # define PUTU32(p,v) *(u32 *)(p) = BSWAP4(v)
  18. #endif
  19. #define PACK(s) ((size_t)(s)<<(sizeof(size_t)*8-16))
  20. #define REDUCE1BIT(V) do { \
  21. if (sizeof(size_t)==8) { \
  22. u64 T = U64(0xe100000000000000) & (0-(V.lo&1)); \
  23. V.lo = (V.hi<<63)|(V.lo>>1); \
  24. V.hi = (V.hi>>1 )^T; \
  25. } \
  26. else { \
  27. u32 T = 0xe1000000U & (0-(u32)(V.lo&1)); \
  28. V.lo = (V.hi<<63)|(V.lo>>1); \
  29. V.hi = (V.hi>>1 )^((u64)T<<32); \
  30. } \
  31. } while(0)
  32. /*-
  33. * Even though permitted values for TABLE_BITS are 8, 4 and 1, it should
  34. * never be set to 8. 8 is effectively reserved for testing purposes.
  35. * TABLE_BITS>1 are lookup-table-driven implementations referred to as
  36. * "Shoup's" in GCM specification. In other words OpenSSL does not cover
  37. * whole spectrum of possible table driven implementations. Why? In
  38. * non-"Shoup's" case memory access pattern is segmented in such manner,
  39. * that it's trivial to see that cache timing information can reveal
  40. * fair portion of intermediate hash value. Given that ciphertext is
  41. * always available to attacker, it's possible for him to attempt to
  42. * deduce secret parameter H and if successful, tamper with messages
  43. * [which is nothing but trivial in CTR mode]. In "Shoup's" case it's
  44. * not as trivial, but there is no reason to believe that it's resistant
  45. * to cache-timing attack. And the thing about "8-bit" implementation is
  46. * that it consumes 16 (sixteen) times more memory, 4KB per individual
  47. * key + 1KB shared. Well, on pros side it should be twice as fast as
  48. * "4-bit" version. And for gcc-generated x86[_64] code, "8-bit" version
  49. * was observed to run ~75% faster, closer to 100% for commercial
  50. * compilers... Yet "4-bit" procedure is preferred, because it's
  51. * believed to provide better security-performance balance and adequate
  52. * all-round performance. "All-round" refers to things like:
  53. *
  54. * - shorter setup time effectively improves overall timing for
  55. * handling short messages;
  56. * - larger table allocation can become unbearable because of VM
  57. * subsystem penalties (for example on Windows large enough free
  58. * results in VM working set trimming, meaning that consequent
  59. * malloc would immediately incur working set expansion);
  60. * - larger table has larger cache footprint, which can affect
  61. * performance of other code paths (not necessarily even from same
  62. * thread in Hyper-Threading world);
  63. *
  64. * Value of 1 is not appropriate for performance reasons.
  65. */
  66. #if TABLE_BITS==8
  67. static void gcm_init_8bit(u128 Htable[256], u64 H[2])
  68. {
  69. int i, j;
  70. u128 V;
  71. Htable[0].hi = 0;
  72. Htable[0].lo = 0;
  73. V.hi = H[0];
  74. V.lo = H[1];
  75. for (Htable[128] = V, i = 64; i > 0; i >>= 1) {
  76. REDUCE1BIT(V);
  77. Htable[i] = V;
  78. }
  79. for (i = 2; i < 256; i <<= 1) {
  80. u128 *Hi = Htable + i, H0 = *Hi;
  81. for (j = 1; j < i; ++j) {
  82. Hi[j].hi = H0.hi ^ Htable[j].hi;
  83. Hi[j].lo = H0.lo ^ Htable[j].lo;
  84. }
  85. }
  86. }
  87. static void gcm_gmult_8bit(u64 Xi[2], const u128 Htable[256])
  88. {
  89. u128 Z = { 0, 0 };
  90. const u8 *xi = (const u8 *)Xi + 15;
  91. size_t rem, n = *xi;
  92. const union {
  93. long one;
  94. char little;
  95. } is_endian = { 1 };
  96. static const size_t rem_8bit[256] = {
  97. PACK(0x0000), PACK(0x01C2), PACK(0x0384), PACK(0x0246),
  98. PACK(0x0708), PACK(0x06CA), PACK(0x048C), PACK(0x054E),
  99. PACK(0x0E10), PACK(0x0FD2), PACK(0x0D94), PACK(0x0C56),
  100. PACK(0x0918), PACK(0x08DA), PACK(0x0A9C), PACK(0x0B5E),
  101. PACK(0x1C20), PACK(0x1DE2), PACK(0x1FA4), PACK(0x1E66),
  102. PACK(0x1B28), PACK(0x1AEA), PACK(0x18AC), PACK(0x196E),
  103. PACK(0x1230), PACK(0x13F2), PACK(0x11B4), PACK(0x1076),
  104. PACK(0x1538), PACK(0x14FA), PACK(0x16BC), PACK(0x177E),
  105. PACK(0x3840), PACK(0x3982), PACK(0x3BC4), PACK(0x3A06),
  106. PACK(0x3F48), PACK(0x3E8A), PACK(0x3CCC), PACK(0x3D0E),
  107. PACK(0x3650), PACK(0x3792), PACK(0x35D4), PACK(0x3416),
  108. PACK(0x3158), PACK(0x309A), PACK(0x32DC), PACK(0x331E),
  109. PACK(0x2460), PACK(0x25A2), PACK(0x27E4), PACK(0x2626),
  110. PACK(0x2368), PACK(0x22AA), PACK(0x20EC), PACK(0x212E),
  111. PACK(0x2A70), PACK(0x2BB2), PACK(0x29F4), PACK(0x2836),
  112. PACK(0x2D78), PACK(0x2CBA), PACK(0x2EFC), PACK(0x2F3E),
  113. PACK(0x7080), PACK(0x7142), PACK(0x7304), PACK(0x72C6),
  114. PACK(0x7788), PACK(0x764A), PACK(0x740C), PACK(0x75CE),
  115. PACK(0x7E90), PACK(0x7F52), PACK(0x7D14), PACK(0x7CD6),
  116. PACK(0x7998), PACK(0x785A), PACK(0x7A1C), PACK(0x7BDE),
  117. PACK(0x6CA0), PACK(0x6D62), PACK(0x6F24), PACK(0x6EE6),
  118. PACK(0x6BA8), PACK(0x6A6A), PACK(0x682C), PACK(0x69EE),
  119. PACK(0x62B0), PACK(0x6372), PACK(0x6134), PACK(0x60F6),
  120. PACK(0x65B8), PACK(0x647A), PACK(0x663C), PACK(0x67FE),
  121. PACK(0x48C0), PACK(0x4902), PACK(0x4B44), PACK(0x4A86),
  122. PACK(0x4FC8), PACK(0x4E0A), PACK(0x4C4C), PACK(0x4D8E),
  123. PACK(0x46D0), PACK(0x4712), PACK(0x4554), PACK(0x4496),
  124. PACK(0x41D8), PACK(0x401A), PACK(0x425C), PACK(0x439E),
  125. PACK(0x54E0), PACK(0x5522), PACK(0x5764), PACK(0x56A6),
  126. PACK(0x53E8), PACK(0x522A), PACK(0x506C), PACK(0x51AE),
  127. PACK(0x5AF0), PACK(0x5B32), PACK(0x5974), PACK(0x58B6),
  128. PACK(0x5DF8), PACK(0x5C3A), PACK(0x5E7C), PACK(0x5FBE),
  129. PACK(0xE100), PACK(0xE0C2), PACK(0xE284), PACK(0xE346),
  130. PACK(0xE608), PACK(0xE7CA), PACK(0xE58C), PACK(0xE44E),
  131. PACK(0xEF10), PACK(0xEED2), PACK(0xEC94), PACK(0xED56),
  132. PACK(0xE818), PACK(0xE9DA), PACK(0xEB9C), PACK(0xEA5E),
  133. PACK(0xFD20), PACK(0xFCE2), PACK(0xFEA4), PACK(0xFF66),
  134. PACK(0xFA28), PACK(0xFBEA), PACK(0xF9AC), PACK(0xF86E),
  135. PACK(0xF330), PACK(0xF2F2), PACK(0xF0B4), PACK(0xF176),
  136. PACK(0xF438), PACK(0xF5FA), PACK(0xF7BC), PACK(0xF67E),
  137. PACK(0xD940), PACK(0xD882), PACK(0xDAC4), PACK(0xDB06),
  138. PACK(0xDE48), PACK(0xDF8A), PACK(0xDDCC), PACK(0xDC0E),
  139. PACK(0xD750), PACK(0xD692), PACK(0xD4D4), PACK(0xD516),
  140. PACK(0xD058), PACK(0xD19A), PACK(0xD3DC), PACK(0xD21E),
  141. PACK(0xC560), PACK(0xC4A2), PACK(0xC6E4), PACK(0xC726),
  142. PACK(0xC268), PACK(0xC3AA), PACK(0xC1EC), PACK(0xC02E),
  143. PACK(0xCB70), PACK(0xCAB2), PACK(0xC8F4), PACK(0xC936),
  144. PACK(0xCC78), PACK(0xCDBA), PACK(0xCFFC), PACK(0xCE3E),
  145. PACK(0x9180), PACK(0x9042), PACK(0x9204), PACK(0x93C6),
  146. PACK(0x9688), PACK(0x974A), PACK(0x950C), PACK(0x94CE),
  147. PACK(0x9F90), PACK(0x9E52), PACK(0x9C14), PACK(0x9DD6),
  148. PACK(0x9898), PACK(0x995A), PACK(0x9B1C), PACK(0x9ADE),
  149. PACK(0x8DA0), PACK(0x8C62), PACK(0x8E24), PACK(0x8FE6),
  150. PACK(0x8AA8), PACK(0x8B6A), PACK(0x892C), PACK(0x88EE),
  151. PACK(0x83B0), PACK(0x8272), PACK(0x8034), PACK(0x81F6),
  152. PACK(0x84B8), PACK(0x857A), PACK(0x873C), PACK(0x86FE),
  153. PACK(0xA9C0), PACK(0xA802), PACK(0xAA44), PACK(0xAB86),
  154. PACK(0xAEC8), PACK(0xAF0A), PACK(0xAD4C), PACK(0xAC8E),
  155. PACK(0xA7D0), PACK(0xA612), PACK(0xA454), PACK(0xA596),
  156. PACK(0xA0D8), PACK(0xA11A), PACK(0xA35C), PACK(0xA29E),
  157. PACK(0xB5E0), PACK(0xB422), PACK(0xB664), PACK(0xB7A6),
  158. PACK(0xB2E8), PACK(0xB32A), PACK(0xB16C), PACK(0xB0AE),
  159. PACK(0xBBF0), PACK(0xBA32), PACK(0xB874), PACK(0xB9B6),
  160. PACK(0xBCF8), PACK(0xBD3A), PACK(0xBF7C), PACK(0xBEBE)
  161. };
  162. while (1) {
  163. Z.hi ^= Htable[n].hi;
  164. Z.lo ^= Htable[n].lo;
  165. if ((u8 *)Xi == xi)
  166. break;
  167. n = *(--xi);
  168. rem = (size_t)Z.lo & 0xff;
  169. Z.lo = (Z.hi << 56) | (Z.lo >> 8);
  170. Z.hi = (Z.hi >> 8);
  171. if (sizeof(size_t) == 8)
  172. Z.hi ^= rem_8bit[rem];
  173. else
  174. Z.hi ^= (u64)rem_8bit[rem] << 32;
  175. }
  176. if (is_endian.little) {
  177. # ifdef BSWAP8
  178. Xi[0] = BSWAP8(Z.hi);
  179. Xi[1] = BSWAP8(Z.lo);
  180. # else
  181. u8 *p = (u8 *)Xi;
  182. u32 v;
  183. v = (u32)(Z.hi >> 32);
  184. PUTU32(p, v);
  185. v = (u32)(Z.hi);
  186. PUTU32(p + 4, v);
  187. v = (u32)(Z.lo >> 32);
  188. PUTU32(p + 8, v);
  189. v = (u32)(Z.lo);
  190. PUTU32(p + 12, v);
  191. # endif
  192. } else {
  193. Xi[0] = Z.hi;
  194. Xi[1] = Z.lo;
  195. }
  196. }
  197. # define GCM_MUL(ctx) gcm_gmult_8bit(ctx->Xi.u,ctx->Htable)
  198. #elif TABLE_BITS==4
  199. static void gcm_init_4bit(u128 Htable[16], u64 H[2])
  200. {
  201. u128 V;
  202. # if defined(OPENSSL_SMALL_FOOTPRINT)
  203. int i;
  204. # endif
  205. Htable[0].hi = 0;
  206. Htable[0].lo = 0;
  207. V.hi = H[0];
  208. V.lo = H[1];
  209. # if defined(OPENSSL_SMALL_FOOTPRINT)
  210. for (Htable[8] = V, i = 4; i > 0; i >>= 1) {
  211. REDUCE1BIT(V);
  212. Htable[i] = V;
  213. }
  214. for (i = 2; i < 16; i <<= 1) {
  215. u128 *Hi = Htable + i;
  216. int j;
  217. for (V = *Hi, j = 1; j < i; ++j) {
  218. Hi[j].hi = V.hi ^ Htable[j].hi;
  219. Hi[j].lo = V.lo ^ Htable[j].lo;
  220. }
  221. }
  222. # else
  223. Htable[8] = V;
  224. REDUCE1BIT(V);
  225. Htable[4] = V;
  226. REDUCE1BIT(V);
  227. Htable[2] = V;
  228. REDUCE1BIT(V);
  229. Htable[1] = V;
  230. Htable[3].hi = V.hi ^ Htable[2].hi, Htable[3].lo = V.lo ^ Htable[2].lo;
  231. V = Htable[4];
  232. Htable[5].hi = V.hi ^ Htable[1].hi, Htable[5].lo = V.lo ^ Htable[1].lo;
  233. Htable[6].hi = V.hi ^ Htable[2].hi, Htable[6].lo = V.lo ^ Htable[2].lo;
  234. Htable[7].hi = V.hi ^ Htable[3].hi, Htable[7].lo = V.lo ^ Htable[3].lo;
  235. V = Htable[8];
  236. Htable[9].hi = V.hi ^ Htable[1].hi, Htable[9].lo = V.lo ^ Htable[1].lo;
  237. Htable[10].hi = V.hi ^ Htable[2].hi, Htable[10].lo = V.lo ^ Htable[2].lo;
  238. Htable[11].hi = V.hi ^ Htable[3].hi, Htable[11].lo = V.lo ^ Htable[3].lo;
  239. Htable[12].hi = V.hi ^ Htable[4].hi, Htable[12].lo = V.lo ^ Htable[4].lo;
  240. Htable[13].hi = V.hi ^ Htable[5].hi, Htable[13].lo = V.lo ^ Htable[5].lo;
  241. Htable[14].hi = V.hi ^ Htable[6].hi, Htable[14].lo = V.lo ^ Htable[6].lo;
  242. Htable[15].hi = V.hi ^ Htable[7].hi, Htable[15].lo = V.lo ^ Htable[7].lo;
  243. # endif
  244. # if defined(GHASH_ASM) && (defined(__arm__) || defined(__arm))
  245. /*
  246. * ARM assembler expects specific dword order in Htable.
  247. */
  248. {
  249. int j;
  250. const union {
  251. long one;
  252. char little;
  253. } is_endian = { 1 };
  254. if (is_endian.little)
  255. for (j = 0; j < 16; ++j) {
  256. V = Htable[j];
  257. Htable[j].hi = V.lo;
  258. Htable[j].lo = V.hi;
  259. } else
  260. for (j = 0; j < 16; ++j) {
  261. V = Htable[j];
  262. Htable[j].hi = V.lo << 32 | V.lo >> 32;
  263. Htable[j].lo = V.hi << 32 | V.hi >> 32;
  264. }
  265. }
  266. # endif
  267. }
  268. # ifndef GHASH_ASM
  269. static const size_t rem_4bit[16] = {
  270. PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460),
  271. PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0),
  272. PACK(0xE100), PACK(0xFD20), PACK(0xD940), PACK(0xC560),
  273. PACK(0x9180), PACK(0x8DA0), PACK(0xA9C0), PACK(0xB5E0)
  274. };
  275. static void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16])
  276. {
  277. u128 Z;
  278. int cnt = 15;
  279. size_t rem, nlo, nhi;
  280. const union {
  281. long one;
  282. char little;
  283. } is_endian = { 1 };
  284. nlo = ((const u8 *)Xi)[15];
  285. nhi = nlo >> 4;
  286. nlo &= 0xf;
  287. Z.hi = Htable[nlo].hi;
  288. Z.lo = Htable[nlo].lo;
  289. while (1) {
  290. rem = (size_t)Z.lo & 0xf;
  291. Z.lo = (Z.hi << 60) | (Z.lo >> 4);
  292. Z.hi = (Z.hi >> 4);
  293. if (sizeof(size_t) == 8)
  294. Z.hi ^= rem_4bit[rem];
  295. else
  296. Z.hi ^= (u64)rem_4bit[rem] << 32;
  297. Z.hi ^= Htable[nhi].hi;
  298. Z.lo ^= Htable[nhi].lo;
  299. if (--cnt < 0)
  300. break;
  301. nlo = ((const u8 *)Xi)[cnt];
  302. nhi = nlo >> 4;
  303. nlo &= 0xf;
  304. rem = (size_t)Z.lo & 0xf;
  305. Z.lo = (Z.hi << 60) | (Z.lo >> 4);
  306. Z.hi = (Z.hi >> 4);
  307. if (sizeof(size_t) == 8)
  308. Z.hi ^= rem_4bit[rem];
  309. else
  310. Z.hi ^= (u64)rem_4bit[rem] << 32;
  311. Z.hi ^= Htable[nlo].hi;
  312. Z.lo ^= Htable[nlo].lo;
  313. }
  314. if (is_endian.little) {
  315. # ifdef BSWAP8
  316. Xi[0] = BSWAP8(Z.hi);
  317. Xi[1] = BSWAP8(Z.lo);
  318. # else
  319. u8 *p = (u8 *)Xi;
  320. u32 v;
  321. v = (u32)(Z.hi >> 32);
  322. PUTU32(p, v);
  323. v = (u32)(Z.hi);
  324. PUTU32(p + 4, v);
  325. v = (u32)(Z.lo >> 32);
  326. PUTU32(p + 8, v);
  327. v = (u32)(Z.lo);
  328. PUTU32(p + 12, v);
  329. # endif
  330. } else {
  331. Xi[0] = Z.hi;
  332. Xi[1] = Z.lo;
  333. }
  334. }
  335. # if !defined(OPENSSL_SMALL_FOOTPRINT)
  336. /*
  337. * Streamed gcm_mult_4bit, see CRYPTO_gcm128_[en|de]crypt for
  338. * details... Compiler-generated code doesn't seem to give any
  339. * performance improvement, at least not on x86[_64]. It's here
  340. * mostly as reference and a placeholder for possible future
  341. * non-trivial optimization[s]...
  342. */
  343. static void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16],
  344. const u8 *inp, size_t len)
  345. {
  346. u128 Z;
  347. int cnt;
  348. size_t rem, nlo, nhi;
  349. const union {
  350. long one;
  351. char little;
  352. } is_endian = { 1 };
  353. # if 1
  354. do {
  355. cnt = 15;
  356. nlo = ((const u8 *)Xi)[15];
  357. nlo ^= inp[15];
  358. nhi = nlo >> 4;
  359. nlo &= 0xf;
  360. Z.hi = Htable[nlo].hi;
  361. Z.lo = Htable[nlo].lo;
  362. while (1) {
  363. rem = (size_t)Z.lo & 0xf;
  364. Z.lo = (Z.hi << 60) | (Z.lo >> 4);
  365. Z.hi = (Z.hi >> 4);
  366. if (sizeof(size_t) == 8)
  367. Z.hi ^= rem_4bit[rem];
  368. else
  369. Z.hi ^= (u64)rem_4bit[rem] << 32;
  370. Z.hi ^= Htable[nhi].hi;
  371. Z.lo ^= Htable[nhi].lo;
  372. if (--cnt < 0)
  373. break;
  374. nlo = ((const u8 *)Xi)[cnt];
  375. nlo ^= inp[cnt];
  376. nhi = nlo >> 4;
  377. nlo &= 0xf;
  378. rem = (size_t)Z.lo & 0xf;
  379. Z.lo = (Z.hi << 60) | (Z.lo >> 4);
  380. Z.hi = (Z.hi >> 4);
  381. if (sizeof(size_t) == 8)
  382. Z.hi ^= rem_4bit[rem];
  383. else
  384. Z.hi ^= (u64)rem_4bit[rem] << 32;
  385. Z.hi ^= Htable[nlo].hi;
  386. Z.lo ^= Htable[nlo].lo;
  387. }
  388. # else
  389. /*
  390. * Extra 256+16 bytes per-key plus 512 bytes shared tables
  391. * [should] give ~50% improvement... One could have PACK()-ed
  392. * the rem_8bit even here, but the priority is to minimize
  393. * cache footprint...
  394. */
  395. u128 Hshr4[16]; /* Htable shifted right by 4 bits */
  396. u8 Hshl4[16]; /* Htable shifted left by 4 bits */
  397. static const unsigned short rem_8bit[256] = {
  398. 0x0000, 0x01C2, 0x0384, 0x0246, 0x0708, 0x06CA, 0x048C, 0x054E,
  399. 0x0E10, 0x0FD2, 0x0D94, 0x0C56, 0x0918, 0x08DA, 0x0A9C, 0x0B5E,
  400. 0x1C20, 0x1DE2, 0x1FA4, 0x1E66, 0x1B28, 0x1AEA, 0x18AC, 0x196E,
  401. 0x1230, 0x13F2, 0x11B4, 0x1076, 0x1538, 0x14FA, 0x16BC, 0x177E,
  402. 0x3840, 0x3982, 0x3BC4, 0x3A06, 0x3F48, 0x3E8A, 0x3CCC, 0x3D0E,
  403. 0x3650, 0x3792, 0x35D4, 0x3416, 0x3158, 0x309A, 0x32DC, 0x331E,
  404. 0x2460, 0x25A2, 0x27E4, 0x2626, 0x2368, 0x22AA, 0x20EC, 0x212E,
  405. 0x2A70, 0x2BB2, 0x29F4, 0x2836, 0x2D78, 0x2CBA, 0x2EFC, 0x2F3E,
  406. 0x7080, 0x7142, 0x7304, 0x72C6, 0x7788, 0x764A, 0x740C, 0x75CE,
  407. 0x7E90, 0x7F52, 0x7D14, 0x7CD6, 0x7998, 0x785A, 0x7A1C, 0x7BDE,
  408. 0x6CA0, 0x6D62, 0x6F24, 0x6EE6, 0x6BA8, 0x6A6A, 0x682C, 0x69EE,
  409. 0x62B0, 0x6372, 0x6134, 0x60F6, 0x65B8, 0x647A, 0x663C, 0x67FE,
  410. 0x48C0, 0x4902, 0x4B44, 0x4A86, 0x4FC8, 0x4E0A, 0x4C4C, 0x4D8E,
  411. 0x46D0, 0x4712, 0x4554, 0x4496, 0x41D8, 0x401A, 0x425C, 0x439E,
  412. 0x54E0, 0x5522, 0x5764, 0x56A6, 0x53E8, 0x522A, 0x506C, 0x51AE,
  413. 0x5AF0, 0x5B32, 0x5974, 0x58B6, 0x5DF8, 0x5C3A, 0x5E7C, 0x5FBE,
  414. 0xE100, 0xE0C2, 0xE284, 0xE346, 0xE608, 0xE7CA, 0xE58C, 0xE44E,
  415. 0xEF10, 0xEED2, 0xEC94, 0xED56, 0xE818, 0xE9DA, 0xEB9C, 0xEA5E,
  416. 0xFD20, 0xFCE2, 0xFEA4, 0xFF66, 0xFA28, 0xFBEA, 0xF9AC, 0xF86E,
  417. 0xF330, 0xF2F2, 0xF0B4, 0xF176, 0xF438, 0xF5FA, 0xF7BC, 0xF67E,
  418. 0xD940, 0xD882, 0xDAC4, 0xDB06, 0xDE48, 0xDF8A, 0xDDCC, 0xDC0E,
  419. 0xD750, 0xD692, 0xD4D4, 0xD516, 0xD058, 0xD19A, 0xD3DC, 0xD21E,
  420. 0xC560, 0xC4A2, 0xC6E4, 0xC726, 0xC268, 0xC3AA, 0xC1EC, 0xC02E,
  421. 0xCB70, 0xCAB2, 0xC8F4, 0xC936, 0xCC78, 0xCDBA, 0xCFFC, 0xCE3E,
  422. 0x9180, 0x9042, 0x9204, 0x93C6, 0x9688, 0x974A, 0x950C, 0x94CE,
  423. 0x9F90, 0x9E52, 0x9C14, 0x9DD6, 0x9898, 0x995A, 0x9B1C, 0x9ADE,
  424. 0x8DA0, 0x8C62, 0x8E24, 0x8FE6, 0x8AA8, 0x8B6A, 0x892C, 0x88EE,
  425. 0x83B0, 0x8272, 0x8034, 0x81F6, 0x84B8, 0x857A, 0x873C, 0x86FE,
  426. 0xA9C0, 0xA802, 0xAA44, 0xAB86, 0xAEC8, 0xAF0A, 0xAD4C, 0xAC8E,
  427. 0xA7D0, 0xA612, 0xA454, 0xA596, 0xA0D8, 0xA11A, 0xA35C, 0xA29E,
  428. 0xB5E0, 0xB422, 0xB664, 0xB7A6, 0xB2E8, 0xB32A, 0xB16C, 0xB0AE,
  429. 0xBBF0, 0xBA32, 0xB874, 0xB9B6, 0xBCF8, 0xBD3A, 0xBF7C, 0xBEBE
  430. };
  431. /*
  432. * This pre-processing phase slows down procedure by approximately
  433. * same time as it makes each loop spin faster. In other words
  434. * single block performance is approximately same as straightforward
  435. * "4-bit" implementation, and then it goes only faster...
  436. */
  437. for (cnt = 0; cnt < 16; ++cnt) {
  438. Z.hi = Htable[cnt].hi;
  439. Z.lo = Htable[cnt].lo;
  440. Hshr4[cnt].lo = (Z.hi << 60) | (Z.lo >> 4);
  441. Hshr4[cnt].hi = (Z.hi >> 4);
  442. Hshl4[cnt] = (u8)(Z.lo << 4);
  443. }
  444. do {
  445. for (Z.lo = 0, Z.hi = 0, cnt = 15; cnt; --cnt) {
  446. nlo = ((const u8 *)Xi)[cnt];
  447. nlo ^= inp[cnt];
  448. nhi = nlo >> 4;
  449. nlo &= 0xf;
  450. Z.hi ^= Htable[nlo].hi;
  451. Z.lo ^= Htable[nlo].lo;
  452. rem = (size_t)Z.lo & 0xff;
  453. Z.lo = (Z.hi << 56) | (Z.lo >> 8);
  454. Z.hi = (Z.hi >> 8);
  455. Z.hi ^= Hshr4[nhi].hi;
  456. Z.lo ^= Hshr4[nhi].lo;
  457. Z.hi ^= (u64)rem_8bit[rem ^ Hshl4[nhi]] << 48;
  458. }
  459. nlo = ((const u8 *)Xi)[0];
  460. nlo ^= inp[0];
  461. nhi = nlo >> 4;
  462. nlo &= 0xf;
  463. Z.hi ^= Htable[nlo].hi;
  464. Z.lo ^= Htable[nlo].lo;
  465. rem = (size_t)Z.lo & 0xf;
  466. Z.lo = (Z.hi << 60) | (Z.lo >> 4);
  467. Z.hi = (Z.hi >> 4);
  468. Z.hi ^= Htable[nhi].hi;
  469. Z.lo ^= Htable[nhi].lo;
  470. Z.hi ^= ((u64)rem_8bit[rem << 4]) << 48;
  471. # endif
  472. if (is_endian.little) {
  473. # ifdef BSWAP8
  474. Xi[0] = BSWAP8(Z.hi);
  475. Xi[1] = BSWAP8(Z.lo);
  476. # else
  477. u8 *p = (u8 *)Xi;
  478. u32 v;
  479. v = (u32)(Z.hi >> 32);
  480. PUTU32(p, v);
  481. v = (u32)(Z.hi);
  482. PUTU32(p + 4, v);
  483. v = (u32)(Z.lo >> 32);
  484. PUTU32(p + 8, v);
  485. v = (u32)(Z.lo);
  486. PUTU32(p + 12, v);
  487. # endif
  488. } else {
  489. Xi[0] = Z.hi;
  490. Xi[1] = Z.lo;
  491. }
  492. } while (inp += 16, len -= 16);
  493. }
  494. # endif
  495. # else
  496. void gcm_gmult_4bit(u64 Xi[2], const u128 Htable[16]);
  497. void gcm_ghash_4bit(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  498. size_t len);
  499. # endif
  500. # define GCM_MUL(ctx) gcm_gmult_4bit(ctx->Xi.u,ctx->Htable)
  501. # if defined(GHASH_ASM) || !defined(OPENSSL_SMALL_FOOTPRINT)
  502. # define GHASH(ctx,in,len) gcm_ghash_4bit((ctx)->Xi.u,(ctx)->Htable,in,len)
  503. /*
  504. * GHASH_CHUNK is "stride parameter" missioned to mitigate cache trashing
  505. * effect. In other words idea is to hash data while it's still in L1 cache
  506. * after encryption pass...
  507. */
  508. # define GHASH_CHUNK (3*1024)
  509. # endif
  510. #else /* TABLE_BITS */
  511. static void gcm_gmult_1bit(u64 Xi[2], const u64 H[2])
  512. {
  513. u128 V, Z = { 0, 0 };
  514. long X;
  515. int i, j;
  516. const long *xi = (const long *)Xi;
  517. const union {
  518. long one;
  519. char little;
  520. } is_endian = { 1 };
  521. V.hi = H[0]; /* H is in host byte order, no byte swapping */
  522. V.lo = H[1];
  523. for (j = 0; j < 16 / sizeof(long); ++j) {
  524. if (is_endian.little) {
  525. if (sizeof(long) == 8) {
  526. # ifdef BSWAP8
  527. X = (long)(BSWAP8(xi[j]));
  528. # else
  529. const u8 *p = (const u8 *)(xi + j);
  530. X = (long)((u64)GETU32(p) << 32 | GETU32(p + 4));
  531. # endif
  532. } else {
  533. const u8 *p = (const u8 *)(xi + j);
  534. X = (long)GETU32(p);
  535. }
  536. } else
  537. X = xi[j];
  538. for (i = 0; i < 8 * sizeof(long); ++i, X <<= 1) {
  539. u64 M = (u64)(X >> (8 * sizeof(long) - 1));
  540. Z.hi ^= V.hi & M;
  541. Z.lo ^= V.lo & M;
  542. REDUCE1BIT(V);
  543. }
  544. }
  545. if (is_endian.little) {
  546. # ifdef BSWAP8
  547. Xi[0] = BSWAP8(Z.hi);
  548. Xi[1] = BSWAP8(Z.lo);
  549. # else
  550. u8 *p = (u8 *)Xi;
  551. u32 v;
  552. v = (u32)(Z.hi >> 32);
  553. PUTU32(p, v);
  554. v = (u32)(Z.hi);
  555. PUTU32(p + 4, v);
  556. v = (u32)(Z.lo >> 32);
  557. PUTU32(p + 8, v);
  558. v = (u32)(Z.lo);
  559. PUTU32(p + 12, v);
  560. # endif
  561. } else {
  562. Xi[0] = Z.hi;
  563. Xi[1] = Z.lo;
  564. }
  565. }
  566. # define GCM_MUL(ctx) gcm_gmult_1bit(ctx->Xi.u,ctx->H.u)
  567. #endif
  568. #if TABLE_BITS==4 && (defined(GHASH_ASM) || defined(OPENSSL_CPUID_OBJ))
  569. # if !defined(I386_ONLY) && \
  570. (defined(__i386) || defined(__i386__) || \
  571. defined(__x86_64) || defined(__x86_64__) || \
  572. defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64))
  573. # define GHASH_ASM_X86_OR_64
  574. # define GCM_FUNCREF_4BIT
  575. extern unsigned int OPENSSL_ia32cap_P[];
  576. void gcm_init_clmul(u128 Htable[16], const u64 Xi[2]);
  577. void gcm_gmult_clmul(u64 Xi[2], const u128 Htable[16]);
  578. void gcm_ghash_clmul(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  579. size_t len);
  580. # if defined(__i386) || defined(__i386__) || defined(_M_IX86)
  581. # define gcm_init_avx gcm_init_clmul
  582. # define gcm_gmult_avx gcm_gmult_clmul
  583. # define gcm_ghash_avx gcm_ghash_clmul
  584. # else
  585. void gcm_init_avx(u128 Htable[16], const u64 Xi[2]);
  586. void gcm_gmult_avx(u64 Xi[2], const u128 Htable[16]);
  587. void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  588. size_t len);
  589. # endif
  590. # if defined(__i386) || defined(__i386__) || defined(_M_IX86)
  591. # define GHASH_ASM_X86
  592. void gcm_gmult_4bit_mmx(u64 Xi[2], const u128 Htable[16]);
  593. void gcm_ghash_4bit_mmx(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  594. size_t len);
  595. void gcm_gmult_4bit_x86(u64 Xi[2], const u128 Htable[16]);
  596. void gcm_ghash_4bit_x86(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  597. size_t len);
  598. # endif
  599. # elif defined(__arm__) || defined(__arm) || defined(__aarch64__)
  600. # include "arm_arch.h"
  601. # if __ARM_MAX_ARCH__>=7
  602. # define GHASH_ASM_ARM
  603. # define GCM_FUNCREF_4BIT
  604. # define PMULL_CAPABLE (OPENSSL_armcap_P & ARMV8_PMULL)
  605. # if defined(__arm__) || defined(__arm)
  606. # define NEON_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
  607. # endif
  608. void gcm_init_neon(u128 Htable[16], const u64 Xi[2]);
  609. void gcm_gmult_neon(u64 Xi[2], const u128 Htable[16]);
  610. void gcm_ghash_neon(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  611. size_t len);
  612. void gcm_init_v8(u128 Htable[16], const u64 Xi[2]);
  613. void gcm_gmult_v8(u64 Xi[2], const u128 Htable[16]);
  614. void gcm_ghash_v8(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  615. size_t len);
  616. # endif
  617. # elif defined(__sparc__) || defined(__sparc)
  618. # include "sparc_arch.h"
  619. # define GHASH_ASM_SPARC
  620. # define GCM_FUNCREF_4BIT
  621. extern unsigned int OPENSSL_sparcv9cap_P[];
  622. void gcm_init_vis3(u128 Htable[16], const u64 Xi[2]);
  623. void gcm_gmult_vis3(u64 Xi[2], const u128 Htable[16]);
  624. void gcm_ghash_vis3(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  625. size_t len);
  626. # elif defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
  627. # include "ppc_arch.h"
  628. # define GHASH_ASM_PPC
  629. # define GCM_FUNCREF_4BIT
  630. void gcm_init_p8(u128 Htable[16], const u64 Xi[2]);
  631. void gcm_gmult_p8(u64 Xi[2], const u128 Htable[16]);
  632. void gcm_ghash_p8(u64 Xi[2], const u128 Htable[16], const u8 *inp,
  633. size_t len);
  634. # endif
  635. #endif
  636. #ifdef GCM_FUNCREF_4BIT
  637. # undef GCM_MUL
  638. # define GCM_MUL(ctx) (*gcm_gmult_p)(ctx->Xi.u,ctx->Htable)
  639. # ifdef GHASH
  640. # undef GHASH
  641. # define GHASH(ctx,in,len) (*gcm_ghash_p)(ctx->Xi.u,ctx->Htable,in,len)
  642. # endif
  643. #endif
  644. void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, void *key, block128_f block)
  645. {
  646. const union {
  647. long one;
  648. char little;
  649. } is_endian = { 1 };
  650. memset(ctx, 0, sizeof(*ctx));
  651. ctx->block = block;
  652. ctx->key = key;
  653. (*block) (ctx->H.c, ctx->H.c, key);
  654. if (is_endian.little) {
  655. /* H is stored in host byte order */
  656. #ifdef BSWAP8
  657. ctx->H.u[0] = BSWAP8(ctx->H.u[0]);
  658. ctx->H.u[1] = BSWAP8(ctx->H.u[1]);
  659. #else
  660. u8 *p = ctx->H.c;
  661. u64 hi, lo;
  662. hi = (u64)GETU32(p) << 32 | GETU32(p + 4);
  663. lo = (u64)GETU32(p + 8) << 32 | GETU32(p + 12);
  664. ctx->H.u[0] = hi;
  665. ctx->H.u[1] = lo;
  666. #endif
  667. }
  668. #if TABLE_BITS==8
  669. gcm_init_8bit(ctx->Htable, ctx->H.u);
  670. #elif TABLE_BITS==4
  671. # if defined(GHASH)
  672. # define CTX__GHASH(f) (ctx->ghash = (f))
  673. # else
  674. # define CTX__GHASH(f) (ctx->ghash = NULL)
  675. # endif
  676. # if defined(GHASH_ASM_X86_OR_64)
  677. # if !defined(GHASH_ASM_X86) || defined(OPENSSL_IA32_SSE2)
  678. if (OPENSSL_ia32cap_P[1] & (1 << 1)) { /* check PCLMULQDQ bit */
  679. if (((OPENSSL_ia32cap_P[1] >> 22) & 0x41) == 0x41) { /* AVX+MOVBE */
  680. gcm_init_avx(ctx->Htable, ctx->H.u);
  681. ctx->gmult = gcm_gmult_avx;
  682. CTX__GHASH(gcm_ghash_avx);
  683. } else {
  684. gcm_init_clmul(ctx->Htable, ctx->H.u);
  685. ctx->gmult = gcm_gmult_clmul;
  686. CTX__GHASH(gcm_ghash_clmul);
  687. }
  688. return;
  689. }
  690. # endif
  691. gcm_init_4bit(ctx->Htable, ctx->H.u);
  692. # if defined(GHASH_ASM_X86) /* x86 only */
  693. # if defined(OPENSSL_IA32_SSE2)
  694. if (OPENSSL_ia32cap_P[0] & (1 << 25)) { /* check SSE bit */
  695. # else
  696. if (OPENSSL_ia32cap_P[0] & (1 << 23)) { /* check MMX bit */
  697. # endif
  698. ctx->gmult = gcm_gmult_4bit_mmx;
  699. CTX__GHASH(gcm_ghash_4bit_mmx);
  700. } else {
  701. ctx->gmult = gcm_gmult_4bit_x86;
  702. CTX__GHASH(gcm_ghash_4bit_x86);
  703. }
  704. # else
  705. ctx->gmult = gcm_gmult_4bit;
  706. CTX__GHASH(gcm_ghash_4bit);
  707. # endif
  708. # elif defined(GHASH_ASM_ARM)
  709. # ifdef PMULL_CAPABLE
  710. if (PMULL_CAPABLE) {
  711. gcm_init_v8(ctx->Htable, ctx->H.u);
  712. ctx->gmult = gcm_gmult_v8;
  713. CTX__GHASH(gcm_ghash_v8);
  714. } else
  715. # endif
  716. # ifdef NEON_CAPABLE
  717. if (NEON_CAPABLE) {
  718. gcm_init_neon(ctx->Htable, ctx->H.u);
  719. ctx->gmult = gcm_gmult_neon;
  720. CTX__GHASH(gcm_ghash_neon);
  721. } else
  722. # endif
  723. {
  724. gcm_init_4bit(ctx->Htable, ctx->H.u);
  725. ctx->gmult = gcm_gmult_4bit;
  726. CTX__GHASH(gcm_ghash_4bit);
  727. }
  728. # elif defined(GHASH_ASM_SPARC)
  729. if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) {
  730. gcm_init_vis3(ctx->Htable, ctx->H.u);
  731. ctx->gmult = gcm_gmult_vis3;
  732. CTX__GHASH(gcm_ghash_vis3);
  733. } else {
  734. gcm_init_4bit(ctx->Htable, ctx->H.u);
  735. ctx->gmult = gcm_gmult_4bit;
  736. CTX__GHASH(gcm_ghash_4bit);
  737. }
  738. # elif defined(GHASH_ASM_PPC)
  739. if (OPENSSL_ppccap_P & PPC_CRYPTO207) {
  740. gcm_init_p8(ctx->Htable, ctx->H.u);
  741. ctx->gmult = gcm_gmult_p8;
  742. CTX__GHASH(gcm_ghash_p8);
  743. } else {
  744. gcm_init_4bit(ctx->Htable, ctx->H.u);
  745. ctx->gmult = gcm_gmult_4bit;
  746. CTX__GHASH(gcm_ghash_4bit);
  747. }
  748. # else
  749. gcm_init_4bit(ctx->Htable, ctx->H.u);
  750. # endif
  751. # undef CTX__GHASH
  752. #endif
  753. }
  754. void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const unsigned char *iv,
  755. size_t len)
  756. {
  757. const union {
  758. long one;
  759. char little;
  760. } is_endian = { 1 };
  761. unsigned int ctr;
  762. #ifdef GCM_FUNCREF_4BIT
  763. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  764. #endif
  765. ctx->len.u[0] = 0; /* AAD length */
  766. ctx->len.u[1] = 0; /* message length */
  767. ctx->ares = 0;
  768. ctx->mres = 0;
  769. if (len == 12) {
  770. memcpy(ctx->Yi.c, iv, 12);
  771. ctx->Yi.c[12] = 0;
  772. ctx->Yi.c[13] = 0;
  773. ctx->Yi.c[14] = 0;
  774. ctx->Yi.c[15] = 1;
  775. ctr = 1;
  776. } else {
  777. size_t i;
  778. u64 len0 = len;
  779. /* Borrow ctx->Xi to calculate initial Yi */
  780. ctx->Xi.u[0] = 0;
  781. ctx->Xi.u[1] = 0;
  782. while (len >= 16) {
  783. for (i = 0; i < 16; ++i)
  784. ctx->Xi.c[i] ^= iv[i];
  785. GCM_MUL(ctx);
  786. iv += 16;
  787. len -= 16;
  788. }
  789. if (len) {
  790. for (i = 0; i < len; ++i)
  791. ctx->Xi.c[i] ^= iv[i];
  792. GCM_MUL(ctx);
  793. }
  794. len0 <<= 3;
  795. if (is_endian.little) {
  796. #ifdef BSWAP8
  797. ctx->Xi.u[1] ^= BSWAP8(len0);
  798. #else
  799. ctx->Xi.c[8] ^= (u8)(len0 >> 56);
  800. ctx->Xi.c[9] ^= (u8)(len0 >> 48);
  801. ctx->Xi.c[10] ^= (u8)(len0 >> 40);
  802. ctx->Xi.c[11] ^= (u8)(len0 >> 32);
  803. ctx->Xi.c[12] ^= (u8)(len0 >> 24);
  804. ctx->Xi.c[13] ^= (u8)(len0 >> 16);
  805. ctx->Xi.c[14] ^= (u8)(len0 >> 8);
  806. ctx->Xi.c[15] ^= (u8)(len0);
  807. #endif
  808. } else {
  809. ctx->Xi.u[1] ^= len0;
  810. }
  811. GCM_MUL(ctx);
  812. if (is_endian.little)
  813. #ifdef BSWAP4
  814. ctr = BSWAP4(ctx->Xi.d[3]);
  815. #else
  816. ctr = GETU32(ctx->Xi.c + 12);
  817. #endif
  818. else
  819. ctr = ctx->Xi.d[3];
  820. /* Copy borrowed Xi to Yi */
  821. ctx->Yi.u[0] = ctx->Xi.u[0];
  822. ctx->Yi.u[1] = ctx->Xi.u[1];
  823. }
  824. ctx->Xi.u[0] = 0;
  825. ctx->Xi.u[1] = 0;
  826. (*ctx->block) (ctx->Yi.c, ctx->EK0.c, ctx->key);
  827. ++ctr;
  828. if (is_endian.little)
  829. #ifdef BSWAP4
  830. ctx->Yi.d[3] = BSWAP4(ctr);
  831. #else
  832. PUTU32(ctx->Yi.c + 12, ctr);
  833. #endif
  834. else
  835. ctx->Yi.d[3] = ctr;
  836. }
  837. int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const unsigned char *aad,
  838. size_t len)
  839. {
  840. size_t i;
  841. unsigned int n;
  842. u64 alen = ctx->len.u[0];
  843. #ifdef GCM_FUNCREF_4BIT
  844. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  845. # ifdef GHASH
  846. void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16],
  847. const u8 *inp, size_t len) = ctx->ghash;
  848. # endif
  849. #endif
  850. if (ctx->len.u[1])
  851. return -2;
  852. alen += len;
  853. if (alen > (U64(1) << 61) || (sizeof(len) == 8 && alen < len))
  854. return -1;
  855. ctx->len.u[0] = alen;
  856. n = ctx->ares;
  857. if (n) {
  858. while (n && len) {
  859. ctx->Xi.c[n] ^= *(aad++);
  860. --len;
  861. n = (n + 1) % 16;
  862. }
  863. if (n == 0)
  864. GCM_MUL(ctx);
  865. else {
  866. ctx->ares = n;
  867. return 0;
  868. }
  869. }
  870. #ifdef GHASH
  871. if ((i = (len & (size_t)-16))) {
  872. GHASH(ctx, aad, i);
  873. aad += i;
  874. len -= i;
  875. }
  876. #else
  877. while (len >= 16) {
  878. for (i = 0; i < 16; ++i)
  879. ctx->Xi.c[i] ^= aad[i];
  880. GCM_MUL(ctx);
  881. aad += 16;
  882. len -= 16;
  883. }
  884. #endif
  885. if (len) {
  886. n = (unsigned int)len;
  887. for (i = 0; i < len; ++i)
  888. ctx->Xi.c[i] ^= aad[i];
  889. }
  890. ctx->ares = n;
  891. return 0;
  892. }
  893. int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
  894. const unsigned char *in, unsigned char *out,
  895. size_t len)
  896. {
  897. const union {
  898. long one;
  899. char little;
  900. } is_endian = { 1 };
  901. unsigned int n, ctr, mres;
  902. size_t i;
  903. u64 mlen = ctx->len.u[1];
  904. block128_f block = ctx->block;
  905. void *key = ctx->key;
  906. #ifdef GCM_FUNCREF_4BIT
  907. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  908. # if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  909. void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16],
  910. const u8 *inp, size_t len) = ctx->ghash;
  911. # endif
  912. #endif
  913. mlen += len;
  914. if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len))
  915. return -1;
  916. ctx->len.u[1] = mlen;
  917. mres = ctx->mres;
  918. if (ctx->ares) {
  919. /* First call to encrypt finalizes GHASH(AAD) */
  920. #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  921. if (len == 0) {
  922. GCM_MUL(ctx);
  923. ctx->ares = 0;
  924. return 0;
  925. }
  926. memcpy(ctx->Xn, ctx->Xi.c, sizeof(ctx->Xi));
  927. ctx->Xi.u[0] = 0;
  928. ctx->Xi.u[1] = 0;
  929. mres = sizeof(ctx->Xi);
  930. #else
  931. GCM_MUL(ctx);
  932. #endif
  933. ctx->ares = 0;
  934. }
  935. if (is_endian.little)
  936. #ifdef BSWAP4
  937. ctr = BSWAP4(ctx->Yi.d[3]);
  938. #else
  939. ctr = GETU32(ctx->Yi.c + 12);
  940. #endif
  941. else
  942. ctr = ctx->Yi.d[3];
  943. n = mres % 16;
  944. #if !defined(OPENSSL_SMALL_FOOTPRINT)
  945. if (16 % sizeof(size_t) == 0) { /* always true actually */
  946. do {
  947. if (n) {
  948. # if defined(GHASH)
  949. while (n && len) {
  950. ctx->Xn[mres++] = *(out++) = *(in++) ^ ctx->EKi.c[n];
  951. --len;
  952. n = (n + 1) % 16;
  953. }
  954. if (n == 0) {
  955. GHASH(ctx, ctx->Xn, mres);
  956. mres = 0;
  957. } else {
  958. ctx->mres = mres;
  959. return 0;
  960. }
  961. # else
  962. while (n && len) {
  963. ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
  964. --len;
  965. n = (n + 1) % 16;
  966. }
  967. if (n == 0) {
  968. GCM_MUL(ctx);
  969. mres = 0;
  970. } else {
  971. ctx->mres = n;
  972. return 0;
  973. }
  974. # endif
  975. }
  976. # if defined(STRICT_ALIGNMENT)
  977. if (((size_t)in | (size_t)out) % sizeof(size_t) != 0)
  978. break;
  979. # endif
  980. # if defined(GHASH)
  981. if (len >= 16 && mres) {
  982. GHASH(ctx, ctx->Xn, mres);
  983. mres = 0;
  984. }
  985. # if defined(GHASH_CHUNK)
  986. while (len >= GHASH_CHUNK) {
  987. size_t j = GHASH_CHUNK;
  988. while (j) {
  989. size_t *out_t = (size_t *)out;
  990. const size_t *in_t = (const size_t *)in;
  991. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  992. ++ctr;
  993. if (is_endian.little)
  994. # ifdef BSWAP4
  995. ctx->Yi.d[3] = BSWAP4(ctr);
  996. # else
  997. PUTU32(ctx->Yi.c + 12, ctr);
  998. # endif
  999. else
  1000. ctx->Yi.d[3] = ctr;
  1001. for (i = 0; i < 16 / sizeof(size_t); ++i)
  1002. out_t[i] = in_t[i] ^ ctx->EKi.t[i];
  1003. out += 16;
  1004. in += 16;
  1005. j -= 16;
  1006. }
  1007. GHASH(ctx, out - GHASH_CHUNK, GHASH_CHUNK);
  1008. len -= GHASH_CHUNK;
  1009. }
  1010. # endif
  1011. if ((i = (len & (size_t)-16))) {
  1012. size_t j = i;
  1013. while (len >= 16) {
  1014. size_t *out_t = (size_t *)out;
  1015. const size_t *in_t = (const size_t *)in;
  1016. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1017. ++ctr;
  1018. if (is_endian.little)
  1019. # ifdef BSWAP4
  1020. ctx->Yi.d[3] = BSWAP4(ctr);
  1021. # else
  1022. PUTU32(ctx->Yi.c + 12, ctr);
  1023. # endif
  1024. else
  1025. ctx->Yi.d[3] = ctr;
  1026. for (i = 0; i < 16 / sizeof(size_t); ++i)
  1027. out_t[i] = in_t[i] ^ ctx->EKi.t[i];
  1028. out += 16;
  1029. in += 16;
  1030. len -= 16;
  1031. }
  1032. GHASH(ctx, out - j, j);
  1033. }
  1034. # else
  1035. while (len >= 16) {
  1036. size_t *out_t = (size_t *)out;
  1037. const size_t *in_t = (const size_t *)in;
  1038. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1039. ++ctr;
  1040. if (is_endian.little)
  1041. # ifdef BSWAP4
  1042. ctx->Yi.d[3] = BSWAP4(ctr);
  1043. # else
  1044. PUTU32(ctx->Yi.c + 12, ctr);
  1045. # endif
  1046. else
  1047. ctx->Yi.d[3] = ctr;
  1048. for (i = 0; i < 16 / sizeof(size_t); ++i)
  1049. ctx->Xi.t[i] ^= out_t[i] = in_t[i] ^ ctx->EKi.t[i];
  1050. GCM_MUL(ctx);
  1051. out += 16;
  1052. in += 16;
  1053. len -= 16;
  1054. }
  1055. # endif
  1056. if (len) {
  1057. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1058. ++ctr;
  1059. if (is_endian.little)
  1060. # ifdef BSWAP4
  1061. ctx->Yi.d[3] = BSWAP4(ctr);
  1062. # else
  1063. PUTU32(ctx->Yi.c + 12, ctr);
  1064. # endif
  1065. else
  1066. ctx->Yi.d[3] = ctr;
  1067. # if defined(GHASH)
  1068. while (len--) {
  1069. ctx->Xn[mres++] = out[n] = in[n] ^ ctx->EKi.c[n];
  1070. ++n;
  1071. }
  1072. # else
  1073. while (len--) {
  1074. ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
  1075. ++n;
  1076. }
  1077. mres = n;
  1078. # endif
  1079. }
  1080. ctx->mres = mres;
  1081. return 0;
  1082. } while (0);
  1083. }
  1084. #endif
  1085. for (i = 0; i < len; ++i) {
  1086. if (n == 0) {
  1087. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1088. ++ctr;
  1089. if (is_endian.little)
  1090. #ifdef BSWAP4
  1091. ctx->Yi.d[3] = BSWAP4(ctr);
  1092. #else
  1093. PUTU32(ctx->Yi.c + 12, ctr);
  1094. #endif
  1095. else
  1096. ctx->Yi.d[3] = ctr;
  1097. }
  1098. #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1099. ctx->Xn[mres++] = out[i] = in[i] ^ ctx->EKi.c[n];
  1100. n = (n + 1) % 16;
  1101. if (mres == sizeof(ctx->Xn)) {
  1102. GHASH(ctx,ctx->Xn,sizeof(ctx->Xn));
  1103. mres = 0;
  1104. }
  1105. #else
  1106. ctx->Xi.c[n] ^= out[i] = in[i] ^ ctx->EKi.c[n];
  1107. mres = n = (n + 1) % 16;
  1108. if (n == 0)
  1109. GCM_MUL(ctx);
  1110. #endif
  1111. }
  1112. ctx->mres = mres;
  1113. return 0;
  1114. }
  1115. int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
  1116. const unsigned char *in, unsigned char *out,
  1117. size_t len)
  1118. {
  1119. const union {
  1120. long one;
  1121. char little;
  1122. } is_endian = { 1 };
  1123. unsigned int n, ctr, mres;
  1124. size_t i;
  1125. u64 mlen = ctx->len.u[1];
  1126. block128_f block = ctx->block;
  1127. void *key = ctx->key;
  1128. #ifdef GCM_FUNCREF_4BIT
  1129. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  1130. # if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1131. void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16],
  1132. const u8 *inp, size_t len) = ctx->ghash;
  1133. # endif
  1134. #endif
  1135. mlen += len;
  1136. if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len))
  1137. return -1;
  1138. ctx->len.u[1] = mlen;
  1139. mres = ctx->mres;
  1140. if (ctx->ares) {
  1141. /* First call to decrypt finalizes GHASH(AAD) */
  1142. #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1143. if (len == 0) {
  1144. GCM_MUL(ctx);
  1145. ctx->ares = 0;
  1146. return 0;
  1147. }
  1148. memcpy(ctx->Xn, ctx->Xi.c, sizeof(ctx->Xi));
  1149. ctx->Xi.u[0] = 0;
  1150. ctx->Xi.u[1] = 0;
  1151. mres = sizeof(ctx->Xi);
  1152. #else
  1153. GCM_MUL(ctx);
  1154. #endif
  1155. ctx->ares = 0;
  1156. }
  1157. if (is_endian.little)
  1158. #ifdef BSWAP4
  1159. ctr = BSWAP4(ctx->Yi.d[3]);
  1160. #else
  1161. ctr = GETU32(ctx->Yi.c + 12);
  1162. #endif
  1163. else
  1164. ctr = ctx->Yi.d[3];
  1165. n = mres % 16;
  1166. #if !defined(OPENSSL_SMALL_FOOTPRINT)
  1167. if (16 % sizeof(size_t) == 0) { /* always true actually */
  1168. do {
  1169. if (n) {
  1170. # if defined(GHASH)
  1171. while (n && len) {
  1172. *(out++) = (ctx->Xn[mres++] = *(in++)) ^ ctx->EKi.c[n];
  1173. --len;
  1174. n = (n + 1) % 16;
  1175. }
  1176. if (n == 0) {
  1177. GHASH(ctx, ctx->Xn, mres);
  1178. mres = 0;
  1179. } else {
  1180. ctx->mres = mres;
  1181. return 0;
  1182. }
  1183. # else
  1184. while (n && len) {
  1185. u8 c = *(in++);
  1186. *(out++) = c ^ ctx->EKi.c[n];
  1187. ctx->Xi.c[n] ^= c;
  1188. --len;
  1189. n = (n + 1) % 16;
  1190. }
  1191. if (n == 0) {
  1192. GCM_MUL(ctx);
  1193. mres = 0;
  1194. } else {
  1195. ctx->mres = n;
  1196. return 0;
  1197. }
  1198. # endif
  1199. }
  1200. # if defined(STRICT_ALIGNMENT)
  1201. if (((size_t)in | (size_t)out) % sizeof(size_t) != 0)
  1202. break;
  1203. # endif
  1204. # if defined(GHASH)
  1205. if (len >= 16 && mres) {
  1206. GHASH(ctx, ctx->Xn, mres);
  1207. mres = 0;
  1208. }
  1209. # if defined(GHASH_CHUNK)
  1210. while (len >= GHASH_CHUNK) {
  1211. size_t j = GHASH_CHUNK;
  1212. GHASH(ctx, in, GHASH_CHUNK);
  1213. while (j) {
  1214. size_t *out_t = (size_t *)out;
  1215. const size_t *in_t = (const size_t *)in;
  1216. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1217. ++ctr;
  1218. if (is_endian.little)
  1219. # ifdef BSWAP4
  1220. ctx->Yi.d[3] = BSWAP4(ctr);
  1221. # else
  1222. PUTU32(ctx->Yi.c + 12, ctr);
  1223. # endif
  1224. else
  1225. ctx->Yi.d[3] = ctr;
  1226. for (i = 0; i < 16 / sizeof(size_t); ++i)
  1227. out_t[i] = in_t[i] ^ ctx->EKi.t[i];
  1228. out += 16;
  1229. in += 16;
  1230. j -= 16;
  1231. }
  1232. len -= GHASH_CHUNK;
  1233. }
  1234. # endif
  1235. if ((i = (len & (size_t)-16))) {
  1236. GHASH(ctx, in, i);
  1237. while (len >= 16) {
  1238. size_t *out_t = (size_t *)out;
  1239. const size_t *in_t = (const size_t *)in;
  1240. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1241. ++ctr;
  1242. if (is_endian.little)
  1243. # ifdef BSWAP4
  1244. ctx->Yi.d[3] = BSWAP4(ctr);
  1245. # else
  1246. PUTU32(ctx->Yi.c + 12, ctr);
  1247. # endif
  1248. else
  1249. ctx->Yi.d[3] = ctr;
  1250. for (i = 0; i < 16 / sizeof(size_t); ++i)
  1251. out_t[i] = in_t[i] ^ ctx->EKi.t[i];
  1252. out += 16;
  1253. in += 16;
  1254. len -= 16;
  1255. }
  1256. }
  1257. # else
  1258. while (len >= 16) {
  1259. size_t *out_t = (size_t *)out;
  1260. const size_t *in_t = (const size_t *)in;
  1261. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1262. ++ctr;
  1263. if (is_endian.little)
  1264. # ifdef BSWAP4
  1265. ctx->Yi.d[3] = BSWAP4(ctr);
  1266. # else
  1267. PUTU32(ctx->Yi.c + 12, ctr);
  1268. # endif
  1269. else
  1270. ctx->Yi.d[3] = ctr;
  1271. for (i = 0; i < 16 / sizeof(size_t); ++i) {
  1272. size_t c = in[i];
  1273. out[i] = c ^ ctx->EKi.t[i];
  1274. ctx->Xi.t[i] ^= c;
  1275. }
  1276. GCM_MUL(ctx);
  1277. out += 16;
  1278. in += 16;
  1279. len -= 16;
  1280. }
  1281. # endif
  1282. if (len) {
  1283. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1284. ++ctr;
  1285. if (is_endian.little)
  1286. # ifdef BSWAP4
  1287. ctx->Yi.d[3] = BSWAP4(ctr);
  1288. # else
  1289. PUTU32(ctx->Yi.c + 12, ctr);
  1290. # endif
  1291. else
  1292. ctx->Yi.d[3] = ctr;
  1293. # if defined(GHASH)
  1294. while (len--) {
  1295. out[n] = (ctx->Xn[mres++] = in[n]) ^ ctx->EKi.c[n];
  1296. ++n;
  1297. }
  1298. # else
  1299. while (len--) {
  1300. u8 c = in[n];
  1301. ctx->Xi.c[n] ^= c;
  1302. out[n] = c ^ ctx->EKi.c[n];
  1303. ++n;
  1304. }
  1305. mres = n;
  1306. # endif
  1307. }
  1308. ctx->mres = mres;
  1309. return 0;
  1310. } while (0);
  1311. }
  1312. #endif
  1313. for (i = 0; i < len; ++i) {
  1314. u8 c;
  1315. if (n == 0) {
  1316. (*block) (ctx->Yi.c, ctx->EKi.c, key);
  1317. ++ctr;
  1318. if (is_endian.little)
  1319. #ifdef BSWAP4
  1320. ctx->Yi.d[3] = BSWAP4(ctr);
  1321. #else
  1322. PUTU32(ctx->Yi.c + 12, ctr);
  1323. #endif
  1324. else
  1325. ctx->Yi.d[3] = ctr;
  1326. }
  1327. #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1328. out[i] = (ctx->Xn[mres++] = c = in[i]) ^ ctx->EKi.c[n];
  1329. n = (n + 1) % 16;
  1330. if (mres == sizeof(ctx->Xn)) {
  1331. GHASH(ctx,ctx->Xn,sizeof(ctx->Xn));
  1332. mres = 0;
  1333. }
  1334. #else
  1335. c = in[i];
  1336. out[i] = c ^ ctx->EKi.c[n];
  1337. ctx->Xi.c[n] ^= c;
  1338. mres = n = (n + 1) % 16;
  1339. if (n == 0)
  1340. GCM_MUL(ctx);
  1341. #endif
  1342. }
  1343. ctx->mres = mres;
  1344. return 0;
  1345. }
  1346. int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
  1347. const unsigned char *in, unsigned char *out,
  1348. size_t len, ctr128_f stream)
  1349. {
  1350. #if defined(OPENSSL_SMALL_FOOTPRINT)
  1351. return CRYPTO_gcm128_encrypt(ctx, in, out, len);
  1352. #else
  1353. const union {
  1354. long one;
  1355. char little;
  1356. } is_endian = { 1 };
  1357. unsigned int n, ctr, mres;
  1358. size_t i;
  1359. u64 mlen = ctx->len.u[1];
  1360. void *key = ctx->key;
  1361. # ifdef GCM_FUNCREF_4BIT
  1362. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  1363. # ifdef GHASH
  1364. void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16],
  1365. const u8 *inp, size_t len) = ctx->ghash;
  1366. # endif
  1367. # endif
  1368. mlen += len;
  1369. if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len))
  1370. return -1;
  1371. ctx->len.u[1] = mlen;
  1372. mres = ctx->mres;
  1373. if (ctx->ares) {
  1374. /* First call to encrypt finalizes GHASH(AAD) */
  1375. #if defined(GHASH)
  1376. if (len == 0) {
  1377. GCM_MUL(ctx);
  1378. ctx->ares = 0;
  1379. return 0;
  1380. }
  1381. memcpy(ctx->Xn, ctx->Xi.c, sizeof(ctx->Xi));
  1382. ctx->Xi.u[0] = 0;
  1383. ctx->Xi.u[1] = 0;
  1384. mres = sizeof(ctx->Xi);
  1385. #else
  1386. GCM_MUL(ctx);
  1387. #endif
  1388. ctx->ares = 0;
  1389. }
  1390. if (is_endian.little)
  1391. # ifdef BSWAP4
  1392. ctr = BSWAP4(ctx->Yi.d[3]);
  1393. # else
  1394. ctr = GETU32(ctx->Yi.c + 12);
  1395. # endif
  1396. else
  1397. ctr = ctx->Yi.d[3];
  1398. n = mres % 16;
  1399. if (n) {
  1400. # if defined(GHASH)
  1401. while (n && len) {
  1402. ctx->Xn[mres++] = *(out++) = *(in++) ^ ctx->EKi.c[n];
  1403. --len;
  1404. n = (n + 1) % 16;
  1405. }
  1406. if (n == 0) {
  1407. GHASH(ctx, ctx->Xn, mres);
  1408. mres = 0;
  1409. } else {
  1410. ctx->mres = mres;
  1411. return 0;
  1412. }
  1413. # else
  1414. while (n && len) {
  1415. ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
  1416. --len;
  1417. n = (n + 1) % 16;
  1418. }
  1419. if (n == 0) {
  1420. GCM_MUL(ctx);
  1421. mres = 0;
  1422. } else {
  1423. ctx->mres = n;
  1424. return 0;
  1425. }
  1426. # endif
  1427. }
  1428. # if defined(GHASH)
  1429. if (len >= 16 && mres) {
  1430. GHASH(ctx, ctx->Xn, mres);
  1431. mres = 0;
  1432. }
  1433. # if defined(GHASH_CHUNK)
  1434. while (len >= GHASH_CHUNK) {
  1435. (*stream) (in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
  1436. ctr += GHASH_CHUNK / 16;
  1437. if (is_endian.little)
  1438. # ifdef BSWAP4
  1439. ctx->Yi.d[3] = BSWAP4(ctr);
  1440. # else
  1441. PUTU32(ctx->Yi.c + 12, ctr);
  1442. # endif
  1443. else
  1444. ctx->Yi.d[3] = ctr;
  1445. GHASH(ctx, out, GHASH_CHUNK);
  1446. out += GHASH_CHUNK;
  1447. in += GHASH_CHUNK;
  1448. len -= GHASH_CHUNK;
  1449. }
  1450. # endif
  1451. # endif
  1452. if ((i = (len & (size_t)-16))) {
  1453. size_t j = i / 16;
  1454. (*stream) (in, out, j, key, ctx->Yi.c);
  1455. ctr += (unsigned int)j;
  1456. if (is_endian.little)
  1457. # ifdef BSWAP4
  1458. ctx->Yi.d[3] = BSWAP4(ctr);
  1459. # else
  1460. PUTU32(ctx->Yi.c + 12, ctr);
  1461. # endif
  1462. else
  1463. ctx->Yi.d[3] = ctr;
  1464. in += i;
  1465. len -= i;
  1466. # if defined(GHASH)
  1467. GHASH(ctx, out, i);
  1468. out += i;
  1469. # else
  1470. while (j--) {
  1471. for (i = 0; i < 16; ++i)
  1472. ctx->Xi.c[i] ^= out[i];
  1473. GCM_MUL(ctx);
  1474. out += 16;
  1475. }
  1476. # endif
  1477. }
  1478. if (len) {
  1479. (*ctx->block) (ctx->Yi.c, ctx->EKi.c, key);
  1480. ++ctr;
  1481. if (is_endian.little)
  1482. # ifdef BSWAP4
  1483. ctx->Yi.d[3] = BSWAP4(ctr);
  1484. # else
  1485. PUTU32(ctx->Yi.c + 12, ctr);
  1486. # endif
  1487. else
  1488. ctx->Yi.d[3] = ctr;
  1489. while (len--) {
  1490. # if defined(GHASH)
  1491. ctx->Xn[mres++] = out[n] = in[n] ^ ctx->EKi.c[n];
  1492. # else
  1493. ctx->Xi.c[mres++] ^= out[n] = in[n] ^ ctx->EKi.c[n];
  1494. # endif
  1495. ++n;
  1496. }
  1497. }
  1498. ctx->mres = mres;
  1499. return 0;
  1500. #endif
  1501. }
  1502. int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
  1503. const unsigned char *in, unsigned char *out,
  1504. size_t len, ctr128_f stream)
  1505. {
  1506. #if defined(OPENSSL_SMALL_FOOTPRINT)
  1507. return CRYPTO_gcm128_decrypt(ctx, in, out, len);
  1508. #else
  1509. const union {
  1510. long one;
  1511. char little;
  1512. } is_endian = { 1 };
  1513. unsigned int n, ctr, mres;
  1514. size_t i;
  1515. u64 mlen = ctx->len.u[1];
  1516. void *key = ctx->key;
  1517. # ifdef GCM_FUNCREF_4BIT
  1518. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  1519. # ifdef GHASH
  1520. void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16],
  1521. const u8 *inp, size_t len) = ctx->ghash;
  1522. # endif
  1523. # endif
  1524. mlen += len;
  1525. if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len))
  1526. return -1;
  1527. ctx->len.u[1] = mlen;
  1528. mres = ctx->mres;
  1529. if (ctx->ares) {
  1530. /* First call to decrypt finalizes GHASH(AAD) */
  1531. # if defined(GHASH)
  1532. if (len == 0) {
  1533. GCM_MUL(ctx);
  1534. ctx->ares = 0;
  1535. return 0;
  1536. }
  1537. memcpy(ctx->Xn, ctx->Xi.c, sizeof(ctx->Xi));
  1538. ctx->Xi.u[0] = 0;
  1539. ctx->Xi.u[1] = 0;
  1540. mres = sizeof(ctx->Xi);
  1541. # else
  1542. GCM_MUL(ctx);
  1543. # endif
  1544. ctx->ares = 0;
  1545. }
  1546. if (is_endian.little)
  1547. # ifdef BSWAP4
  1548. ctr = BSWAP4(ctx->Yi.d[3]);
  1549. # else
  1550. ctr = GETU32(ctx->Yi.c + 12);
  1551. # endif
  1552. else
  1553. ctr = ctx->Yi.d[3];
  1554. n = mres % 16;
  1555. if (n) {
  1556. # if defined(GHASH)
  1557. while (n && len) {
  1558. *(out++) = (ctx->Xn[mres++] = *(in++)) ^ ctx->EKi.c[n];
  1559. --len;
  1560. n = (n + 1) % 16;
  1561. }
  1562. if (n == 0) {
  1563. GHASH(ctx, ctx->Xn, mres);
  1564. mres = 0;
  1565. } else {
  1566. ctx->mres = mres;
  1567. return 0;
  1568. }
  1569. # else
  1570. while (n && len) {
  1571. u8 c = *(in++);
  1572. *(out++) = c ^ ctx->EKi.c[n];
  1573. ctx->Xi.c[n] ^= c;
  1574. --len;
  1575. n = (n + 1) % 16;
  1576. }
  1577. if (n == 0) {
  1578. GCM_MUL(ctx);
  1579. mres = 0;
  1580. } else {
  1581. ctx->mres = n;
  1582. return 0;
  1583. }
  1584. # endif
  1585. }
  1586. # if defined(GHASH)
  1587. if (len >= 16 && mres) {
  1588. GHASH(ctx, ctx->Xn, mres);
  1589. mres = 0;
  1590. }
  1591. # if defined(GHASH_CHUNK)
  1592. while (len >= GHASH_CHUNK) {
  1593. GHASH(ctx, in, GHASH_CHUNK);
  1594. (*stream) (in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
  1595. ctr += GHASH_CHUNK / 16;
  1596. if (is_endian.little)
  1597. # ifdef BSWAP4
  1598. ctx->Yi.d[3] = BSWAP4(ctr);
  1599. # else
  1600. PUTU32(ctx->Yi.c + 12, ctr);
  1601. # endif
  1602. else
  1603. ctx->Yi.d[3] = ctr;
  1604. out += GHASH_CHUNK;
  1605. in += GHASH_CHUNK;
  1606. len -= GHASH_CHUNK;
  1607. }
  1608. # endif
  1609. # endif
  1610. if ((i = (len & (size_t)-16))) {
  1611. size_t j = i / 16;
  1612. # if defined(GHASH)
  1613. GHASH(ctx, in, i);
  1614. # else
  1615. while (j--) {
  1616. size_t k;
  1617. for (k = 0; k < 16; ++k)
  1618. ctx->Xi.c[k] ^= in[k];
  1619. GCM_MUL(ctx);
  1620. in += 16;
  1621. }
  1622. j = i / 16;
  1623. in -= i;
  1624. # endif
  1625. (*stream) (in, out, j, key, ctx->Yi.c);
  1626. ctr += (unsigned int)j;
  1627. if (is_endian.little)
  1628. # ifdef BSWAP4
  1629. ctx->Yi.d[3] = BSWAP4(ctr);
  1630. # else
  1631. PUTU32(ctx->Yi.c + 12, ctr);
  1632. # endif
  1633. else
  1634. ctx->Yi.d[3] = ctr;
  1635. out += i;
  1636. in += i;
  1637. len -= i;
  1638. }
  1639. if (len) {
  1640. (*ctx->block) (ctx->Yi.c, ctx->EKi.c, key);
  1641. ++ctr;
  1642. if (is_endian.little)
  1643. # ifdef BSWAP4
  1644. ctx->Yi.d[3] = BSWAP4(ctr);
  1645. # else
  1646. PUTU32(ctx->Yi.c + 12, ctr);
  1647. # endif
  1648. else
  1649. ctx->Yi.d[3] = ctr;
  1650. while (len--) {
  1651. # if defined(GHASH)
  1652. out[n] = (ctx->Xn[mres++] = in[n]) ^ ctx->EKi.c[n];
  1653. # else
  1654. u8 c = in[n];
  1655. ctx->Xi.c[mres++] ^= c;
  1656. out[n] = c ^ ctx->EKi.c[n];
  1657. # endif
  1658. ++n;
  1659. }
  1660. }
  1661. ctx->mres = mres;
  1662. return 0;
  1663. #endif
  1664. }
  1665. int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const unsigned char *tag,
  1666. size_t len)
  1667. {
  1668. const union {
  1669. long one;
  1670. char little;
  1671. } is_endian = { 1 };
  1672. u64 alen = ctx->len.u[0] << 3;
  1673. u64 clen = ctx->len.u[1] << 3;
  1674. #ifdef GCM_FUNCREF_4BIT
  1675. void (*gcm_gmult_p) (u64 Xi[2], const u128 Htable[16]) = ctx->gmult;
  1676. # if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1677. void (*gcm_ghash_p) (u64 Xi[2], const u128 Htable[16],
  1678. const u8 *inp, size_t len) = ctx->ghash;
  1679. # endif
  1680. #endif
  1681. #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1682. u128 bitlen;
  1683. unsigned int mres = ctx->mres;
  1684. if (mres) {
  1685. unsigned blocks = (mres + 15) & -16;
  1686. memset(ctx->Xn + mres, 0, blocks - mres);
  1687. mres = blocks;
  1688. if (mres == sizeof(ctx->Xn)) {
  1689. GHASH(ctx, ctx->Xn, mres);
  1690. mres = 0;
  1691. }
  1692. } else if (ctx->ares) {
  1693. GCM_MUL(ctx);
  1694. }
  1695. #else
  1696. if (ctx->mres || ctx->ares)
  1697. GCM_MUL(ctx);
  1698. #endif
  1699. if (is_endian.little) {
  1700. #ifdef BSWAP8
  1701. alen = BSWAP8(alen);
  1702. clen = BSWAP8(clen);
  1703. #else
  1704. u8 *p = ctx->len.c;
  1705. ctx->len.u[0] = alen;
  1706. ctx->len.u[1] = clen;
  1707. alen = (u64)GETU32(p) << 32 | GETU32(p + 4);
  1708. clen = (u64)GETU32(p + 8) << 32 | GETU32(p + 12);
  1709. #endif
  1710. }
  1711. #if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
  1712. bitlen.hi = alen;
  1713. bitlen.lo = clen;
  1714. memcpy(ctx->Xn + mres, &bitlen, sizeof(bitlen));
  1715. mres += sizeof(bitlen);
  1716. GHASH(ctx, ctx->Xn, mres);
  1717. #else
  1718. ctx->Xi.u[0] ^= alen;
  1719. ctx->Xi.u[1] ^= clen;
  1720. GCM_MUL(ctx);
  1721. #endif
  1722. ctx->Xi.u[0] ^= ctx->EK0.u[0];
  1723. ctx->Xi.u[1] ^= ctx->EK0.u[1];
  1724. if (tag && len <= sizeof(ctx->Xi))
  1725. return CRYPTO_memcmp(ctx->Xi.c, tag, len);
  1726. else
  1727. return -1;
  1728. }
  1729. void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, unsigned char *tag, size_t len)
  1730. {
  1731. CRYPTO_gcm128_finish(ctx, NULL, 0);
  1732. memcpy(tag, ctx->Xi.c,
  1733. len <= sizeof(ctx->Xi.c) ? len : sizeof(ctx->Xi.c));
  1734. }
  1735. GCM128_CONTEXT *CRYPTO_gcm128_new(void *key, block128_f block)
  1736. {
  1737. GCM128_CONTEXT *ret;
  1738. if ((ret = OPENSSL_malloc(sizeof(*ret))) != NULL)
  1739. CRYPTO_gcm128_init(ret, key, block);
  1740. return ret;
  1741. }
  1742. void CRYPTO_gcm128_release(GCM128_CONTEXT *ctx)
  1743. {
  1744. OPENSSL_clear_free(ctx, sizeof(*ctx));
  1745. }