aes-586.pl 102 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998
  1. #! /usr/bin/env perl
  2. # Copyright 2004-2020 The OpenSSL Project Authors. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License 2.0 (the "License"). You may not use
  5. # this file except in compliance with the License. You can obtain a copy
  6. # in the file LICENSE in the source distribution or at
  7. # https://www.openssl.org/source/license.html
  8. #
  9. # ====================================================================
  10. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  11. # project. The module is, however, dual licensed under OpenSSL and
  12. # CRYPTOGAMS licenses depending on where you obtain it. For further
  13. # details see http://www.openssl.org/~appro/cryptogams/.
  14. # ====================================================================
  15. #
  16. # Version 4.3.
  17. #
  18. # You might fail to appreciate this module performance from the first
  19. # try. If compared to "vanilla" linux-ia32-icc target, i.e. considered
  20. # to be *the* best Intel C compiler without -KPIC, performance appears
  21. # to be virtually identical... But try to re-configure with shared
  22. # library support... Aha! Intel compiler "suddenly" lags behind by 30%
  23. # [on P4, more on others]:-) And if compared to position-independent
  24. # code generated by GNU C, this code performs *more* than *twice* as
  25. # fast! Yes, all this buzz about PIC means that unlike other hand-
  26. # coded implementations, this one was explicitly designed to be safe
  27. # to use even in shared library context... This also means that this
  28. # code isn't necessarily absolutely fastest "ever," because in order
  29. # to achieve position independence an extra register has to be
  30. # off-loaded to stack, which affects the benchmark result.
  31. #
  32. # Special note about instruction choice. Do you recall RC4_INT code
  33. # performing poorly on P4? It might be the time to figure out why.
  34. # RC4_INT code implies effective address calculations in base+offset*4
  35. # form. Trouble is that it seems that offset scaling turned to be
  36. # critical path... At least eliminating scaling resulted in 2.8x RC4
  37. # performance improvement [as you might recall]. As AES code is hungry
  38. # for scaling too, I [try to] avoid the latter by favoring off-by-2
  39. # shifts and masking the result with 0xFF<<2 instead of "boring" 0xFF.
  40. #
  41. # As was shown by Dean Gaudet, the above note turned out to be
  42. # void. Performance improvement with off-by-2 shifts was observed on
  43. # intermediate implementation, which was spilling yet another register
  44. # to stack... Final offset*4 code below runs just a tad faster on P4,
  45. # but exhibits up to 10% improvement on other cores.
  46. #
  47. # Second version is "monolithic" replacement for aes_core.c, which in
  48. # addition to AES_[de|en]crypt implements AES_set_[de|en]cryption_key.
  49. # This made it possible to implement little-endian variant of the
  50. # algorithm without modifying the base C code. Motivating factor for
  51. # the undertaken effort was that it appeared that in tight IA-32
  52. # register window little-endian flavor could achieve slightly higher
  53. # Instruction Level Parallelism, and it indeed resulted in up to 15%
  54. # better performance on most recent µ-archs...
  55. #
  56. # Third version adds AES_cbc_encrypt implementation, which resulted in
  57. # up to 40% performance improvement of CBC benchmark results. 40% was
  58. # observed on P4 core, where "overall" improvement coefficient, i.e. if
  59. # compared to PIC generated by GCC and in CBC mode, was observed to be
  60. # as large as 4x:-) CBC performance is virtually identical to ECB now
  61. # and on some platforms even better, e.g. 17.6 "small" cycles/byte on
  62. # Opteron, because certain function prologues and epilogues are
  63. # effectively taken out of the loop...
  64. #
  65. # Version 3.2 implements compressed tables and prefetch of these tables
  66. # in CBC[!] mode. Former means that 3/4 of table references are now
  67. # misaligned, which unfortunately has negative impact on elder IA-32
  68. # implementations, Pentium suffered 30% penalty, PIII - 10%.
  69. #
  70. # Version 3.3 avoids L1 cache aliasing between stack frame and
  71. # S-boxes, and 3.4 - L1 cache aliasing even between key schedule. The
  72. # latter is achieved by copying the key schedule to controlled place in
  73. # stack. This unfortunately has rather strong impact on small block CBC
  74. # performance, ~2x deterioration on 16-byte block if compared to 3.3.
  75. #
  76. # Version 3.5 checks if there is L1 cache aliasing between user-supplied
  77. # key schedule and S-boxes and abstains from copying the former if
  78. # there is no. This allows end-user to consciously retain small block
  79. # performance by aligning key schedule in specific manner.
  80. #
  81. # Version 3.6 compresses Td4 to 256 bytes and prefetches it in ECB.
  82. #
  83. # Current ECB performance numbers for 128-bit key in CPU cycles per
  84. # processed byte [measure commonly used by AES benchmarkers] are:
  85. #
  86. # small footprint fully unrolled
  87. # P4 24 22
  88. # AMD K8 20 19
  89. # PIII 25 23
  90. # Pentium 81 78
  91. #
  92. # Version 3.7 reimplements outer rounds as "compact." Meaning that
  93. # first and last rounds reference compact 256 bytes S-box. This means
  94. # that first round consumes a lot more CPU cycles and that encrypt
  95. # and decrypt performance becomes asymmetric. Encrypt performance
  96. # drops by 10-12%, while decrypt - by 20-25%:-( 256 bytes S-box is
  97. # aggressively pre-fetched.
  98. #
  99. # Version 4.0 effectively rolls back to 3.6 and instead implements
  100. # additional set of functions, _[x86|sse]_AES_[en|de]crypt_compact,
  101. # which use exclusively 256 byte S-box. These functions are to be
  102. # called in modes not concealing plain text, such as ECB, or when
  103. # we're asked to process smaller amount of data [or unconditionally
  104. # on hyper-threading CPU]. Currently it's called unconditionally from
  105. # AES_[en|de]crypt, which affects all modes, but CBC. CBC routine
  106. # still needs to be modified to switch between slower and faster
  107. # mode when appropriate... But in either case benchmark landscape
  108. # changes dramatically and below numbers are CPU cycles per processed
  109. # byte for 128-bit key.
  110. #
  111. # ECB encrypt ECB decrypt CBC large chunk
  112. # P4 52[54] 83[95] 23
  113. # AMD K8 46[41] 66[70] 18
  114. # PIII 41[50] 60[77] 24
  115. # Core 2 31[36] 45[64] 18.5
  116. # Atom 76[100] 96[138] 60
  117. # Pentium 115 150 77
  118. #
  119. # Version 4.1 switches to compact S-box even in key schedule setup.
  120. #
  121. # Version 4.2 prefetches compact S-box in every SSE round or in other
  122. # words every cache-line is *guaranteed* to be accessed within ~50
  123. # cycles window. Why just SSE? Because it's needed on hyper-threading
  124. # CPU! Which is also why it's prefetched with 64 byte stride. Best
  125. # part is that it has no negative effect on performance:-)
  126. #
  127. # Version 4.3 implements switch between compact and non-compact block
  128. # functions in AES_cbc_encrypt depending on how much data was asked
  129. # to be processed in one stroke.
  130. #
  131. ######################################################################
  132. # Timing attacks are classified in two classes: synchronous when
  133. # attacker consciously initiates cryptographic operation and collects
  134. # timing data of various character afterwards, and asynchronous when
  135. # malicious code is executed on same CPU simultaneously with AES,
  136. # instruments itself and performs statistical analysis of this data.
  137. #
  138. # As far as synchronous attacks go the root to the AES timing
  139. # vulnerability is twofold. Firstly, of 256 S-box elements at most 160
  140. # are referred to in single 128-bit block operation. Well, in C
  141. # implementation with 4 distinct tables it's actually as little as 40
  142. # references per 256 elements table, but anyway... Secondly, even
  143. # though S-box elements are clustered into smaller amount of cache-
  144. # lines, smaller than 160 and even 40, it turned out that for certain
  145. # plain-text pattern[s] or simply put chosen plain-text and given key
  146. # few cache-lines remain unaccessed during block operation. Now, if
  147. # attacker can figure out this access pattern, he can deduct the key
  148. # [or at least part of it]. The natural way to mitigate this kind of
  149. # attacks is to minimize the amount of cache-lines in S-box and/or
  150. # prefetch them to ensure that every one is accessed for more uniform
  151. # timing. But note that *if* plain-text was concealed in such way that
  152. # input to block function is distributed *uniformly*, then attack
  153. # wouldn't apply. Now note that some encryption modes, most notably
  154. # CBC, do mask the plain-text in this exact way [secure cipher output
  155. # is distributed uniformly]. Yes, one still might find input that
  156. # would reveal the information about given key, but if amount of
  157. # candidate inputs to be tried is larger than amount of possible key
  158. # combinations then attack becomes infeasible. This is why revised
  159. # AES_cbc_encrypt "dares" to switch to larger S-box when larger chunk
  160. # of data is to be processed in one stroke. The current size limit of
  161. # 512 bytes is chosen to provide same [diminishingly low] probability
  162. # for cache-line to remain untouched in large chunk operation with
  163. # large S-box as for single block operation with compact S-box and
  164. # surely needs more careful consideration...
  165. #
  166. # As for asynchronous attacks. There are two flavours: attacker code
  167. # being interleaved with AES on hyper-threading CPU at *instruction*
  168. # level, and two processes time sharing single core. As for latter.
  169. # Two vectors. 1. Given that attacker process has higher priority,
  170. # yield execution to process performing AES just before timer fires
  171. # off the scheduler, immediately regain control of CPU and analyze the
  172. # cache state. For this attack to be efficient attacker would have to
  173. # effectively slow down the operation by several *orders* of magnitude,
  174. # by ratio of time slice to duration of handful of AES rounds, which
  175. # unlikely to remain unnoticed. Not to mention that this also means
  176. # that he would spend correspondingly more time to collect enough
  177. # statistical data to mount the attack. It's probably appropriate to
  178. # say that if adversary reckons that this attack is beneficial and
  179. # risks to be noticed, you probably have larger problems having him
  180. # mere opportunity. In other words suggested code design expects you
  181. # to preclude/mitigate this attack by overall system security design.
  182. # 2. Attacker manages to make his code interrupt driven. In order for
  183. # this kind of attack to be feasible, interrupt rate has to be high
  184. # enough, again comparable to duration of handful of AES rounds. But
  185. # is there interrupt source of such rate? Hardly, not even 1Gbps NIC
  186. # generates interrupts at such raging rate...
  187. #
  188. # And now back to the former, hyper-threading CPU or more specifically
  189. # Intel P4. Recall that asynchronous attack implies that malicious
  190. # code instruments itself. And naturally instrumentation granularity
  191. # has be noticeably lower than duration of codepath accessing S-box.
  192. # Given that all cache-lines are accessed during that time that is.
  193. # Current implementation accesses *all* cache-lines within ~50 cycles
  194. # window, which is actually *less* than RDTSC latency on Intel P4!
  195. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  196. push(@INC,"${dir}","${dir}../../perlasm");
  197. require "x86asm.pl";
  198. $output = pop and open STDOUT,">$output";
  199. &asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386");
  200. &static_label("AES_Te");
  201. &static_label("AES_Td");
  202. $s0="eax";
  203. $s1="ebx";
  204. $s2="ecx";
  205. $s3="edx";
  206. $key="edi";
  207. $acc="esi";
  208. $tbl="ebp";
  209. # stack frame layout in _[x86|sse]_AES_* routines, frame is allocated
  210. # by caller
  211. $__ra=&DWP(0,"esp"); # return address
  212. $__s0=&DWP(4,"esp"); # s0 backing store
  213. $__s1=&DWP(8,"esp"); # s1 backing store
  214. $__s2=&DWP(12,"esp"); # s2 backing store
  215. $__s3=&DWP(16,"esp"); # s3 backing store
  216. $__key=&DWP(20,"esp"); # pointer to key schedule
  217. $__end=&DWP(24,"esp"); # pointer to end of key schedule
  218. $__tbl=&DWP(28,"esp"); # %ebp backing store
  219. # stack frame layout in AES_[en|crypt] routines, which differs from
  220. # above by 4 and overlaps by %ebp backing store
  221. $_tbl=&DWP(24,"esp");
  222. $_esp=&DWP(28,"esp");
  223. sub _data_word() { my $i; while(defined($i=shift)) { &data_word($i,$i); } }
  224. $speed_limit=512; # chunks smaller than $speed_limit are
  225. # processed with compact routine in CBC mode
  226. $small_footprint=1; # $small_footprint=1 code is ~5% slower [on
  227. # recent µ-archs], but ~5 times smaller!
  228. # I favor compact code to minimize cache
  229. # contention and in hope to "collect" 5% back
  230. # in real-life applications...
  231. $vertical_spin=0; # shift "vertically" defaults to 0, because of
  232. # its proof-of-concept status...
  233. # Note that there is no decvert(), as well as last encryption round is
  234. # performed with "horizontal" shifts. This is because this "vertical"
  235. # implementation [one which groups shifts on a given $s[i] to form a
  236. # "column," unlike "horizontal" one, which groups shifts on different
  237. # $s[i] to form a "row"] is work in progress. It was observed to run
  238. # few percents faster on Intel cores, but not AMD. On AMD K8 core it's
  239. # whole 12% slower:-( So we face a trade-off... Shall it be resolved
  240. # some day? Till then the code is considered experimental and by
  241. # default remains dormant...
  242. sub encvert()
  243. { my ($te,@s) = @_;
  244. my ($v0,$v1) = ($acc,$key);
  245. &mov ($v0,$s[3]); # copy s3
  246. &mov (&DWP(4,"esp"),$s[2]); # save s2
  247. &mov ($v1,$s[0]); # copy s0
  248. &mov (&DWP(8,"esp"),$s[1]); # save s1
  249. &movz ($s[2],&HB($s[0]));
  250. &and ($s[0],0xFF);
  251. &mov ($s[0],&DWP(0,$te,$s[0],8)); # s0>>0
  252. &shr ($v1,16);
  253. &mov ($s[3],&DWP(3,$te,$s[2],8)); # s0>>8
  254. &movz ($s[1],&HB($v1));
  255. &and ($v1,0xFF);
  256. &mov ($s[2],&DWP(2,$te,$v1,8)); # s0>>16
  257. &mov ($v1,$v0);
  258. &mov ($s[1],&DWP(1,$te,$s[1],8)); # s0>>24
  259. &and ($v0,0xFF);
  260. &xor ($s[3],&DWP(0,$te,$v0,8)); # s3>>0
  261. &movz ($v0,&HB($v1));
  262. &shr ($v1,16);
  263. &xor ($s[2],&DWP(3,$te,$v0,8)); # s3>>8
  264. &movz ($v0,&HB($v1));
  265. &and ($v1,0xFF);
  266. &xor ($s[1],&DWP(2,$te,$v1,8)); # s3>>16
  267. &mov ($v1,&DWP(4,"esp")); # restore s2
  268. &xor ($s[0],&DWP(1,$te,$v0,8)); # s3>>24
  269. &mov ($v0,$v1);
  270. &and ($v1,0xFF);
  271. &xor ($s[2],&DWP(0,$te,$v1,8)); # s2>>0
  272. &movz ($v1,&HB($v0));
  273. &shr ($v0,16);
  274. &xor ($s[1],&DWP(3,$te,$v1,8)); # s2>>8
  275. &movz ($v1,&HB($v0));
  276. &and ($v0,0xFF);
  277. &xor ($s[0],&DWP(2,$te,$v0,8)); # s2>>16
  278. &mov ($v0,&DWP(8,"esp")); # restore s1
  279. &xor ($s[3],&DWP(1,$te,$v1,8)); # s2>>24
  280. &mov ($v1,$v0);
  281. &and ($v0,0xFF);
  282. &xor ($s[1],&DWP(0,$te,$v0,8)); # s1>>0
  283. &movz ($v0,&HB($v1));
  284. &shr ($v1,16);
  285. &xor ($s[0],&DWP(3,$te,$v0,8)); # s1>>8
  286. &movz ($v0,&HB($v1));
  287. &and ($v1,0xFF);
  288. &xor ($s[3],&DWP(2,$te,$v1,8)); # s1>>16
  289. &mov ($key,$__key); # reincarnate v1 as key
  290. &xor ($s[2],&DWP(1,$te,$v0,8)); # s1>>24
  291. }
  292. # Another experimental routine, which features "horizontal spin," but
  293. # eliminates one reference to stack. Strangely enough runs slower...
  294. sub enchoriz()
  295. { my ($v0,$v1) = ($key,$acc);
  296. &movz ($v0,&LB($s0)); # 3, 2, 1, 0*
  297. &rotr ($s2,8); # 8,11,10, 9
  298. &mov ($v1,&DWP(0,$te,$v0,8)); # 0
  299. &movz ($v0,&HB($s1)); # 7, 6, 5*, 4
  300. &rotr ($s3,16); # 13,12,15,14
  301. &xor ($v1,&DWP(3,$te,$v0,8)); # 5
  302. &movz ($v0,&HB($s2)); # 8,11,10*, 9
  303. &rotr ($s0,16); # 1, 0, 3, 2
  304. &xor ($v1,&DWP(2,$te,$v0,8)); # 10
  305. &movz ($v0,&HB($s3)); # 13,12,15*,14
  306. &xor ($v1,&DWP(1,$te,$v0,8)); # 15, t[0] collected
  307. &mov ($__s0,$v1); # t[0] saved
  308. &movz ($v0,&LB($s1)); # 7, 6, 5, 4*
  309. &shr ($s1,16); # -, -, 7, 6
  310. &mov ($v1,&DWP(0,$te,$v0,8)); # 4
  311. &movz ($v0,&LB($s3)); # 13,12,15,14*
  312. &xor ($v1,&DWP(2,$te,$v0,8)); # 14
  313. &movz ($v0,&HB($s0)); # 1, 0, 3*, 2
  314. &and ($s3,0xffff0000); # 13,12, -, -
  315. &xor ($v1,&DWP(1,$te,$v0,8)); # 3
  316. &movz ($v0,&LB($s2)); # 8,11,10, 9*
  317. &or ($s3,$s1); # 13,12, 7, 6
  318. &xor ($v1,&DWP(3,$te,$v0,8)); # 9, t[1] collected
  319. &mov ($s1,$v1); # s[1]=t[1]
  320. &movz ($v0,&LB($s0)); # 1, 0, 3, 2*
  321. &shr ($s2,16); # -, -, 8,11
  322. &mov ($v1,&DWP(2,$te,$v0,8)); # 2
  323. &movz ($v0,&HB($s3)); # 13,12, 7*, 6
  324. &xor ($v1,&DWP(1,$te,$v0,8)); # 7
  325. &movz ($v0,&HB($s2)); # -, -, 8*,11
  326. &xor ($v1,&DWP(0,$te,$v0,8)); # 8
  327. &mov ($v0,$s3);
  328. &shr ($v0,24); # 13
  329. &xor ($v1,&DWP(3,$te,$v0,8)); # 13, t[2] collected
  330. &movz ($v0,&LB($s2)); # -, -, 8,11*
  331. &shr ($s0,24); # 1*
  332. &mov ($s2,&DWP(1,$te,$v0,8)); # 11
  333. &xor ($s2,&DWP(3,$te,$s0,8)); # 1
  334. &mov ($s0,$__s0); # s[0]=t[0]
  335. &movz ($v0,&LB($s3)); # 13,12, 7, 6*
  336. &shr ($s3,16); # , ,13,12
  337. &xor ($s2,&DWP(2,$te,$v0,8)); # 6
  338. &mov ($key,$__key); # reincarnate v0 as key
  339. &and ($s3,0xff); # , ,13,12*
  340. &mov ($s3,&DWP(0,$te,$s3,8)); # 12
  341. &xor ($s3,$s2); # s[2]=t[3] collected
  342. &mov ($s2,$v1); # s[2]=t[2]
  343. }
  344. # More experimental code... SSE one... Even though this one eliminates
  345. # *all* references to stack, it's not faster...
  346. sub sse_encbody()
  347. {
  348. &movz ($acc,&LB("eax")); # 0
  349. &mov ("ecx",&DWP(0,$tbl,$acc,8)); # 0
  350. &pshufw ("mm2","mm0",0x0d); # 7, 6, 3, 2
  351. &movz ("edx",&HB("eax")); # 1
  352. &mov ("edx",&DWP(3,$tbl,"edx",8)); # 1
  353. &shr ("eax",16); # 5, 4
  354. &movz ($acc,&LB("ebx")); # 10
  355. &xor ("ecx",&DWP(2,$tbl,$acc,8)); # 10
  356. &pshufw ("mm6","mm4",0x08); # 13,12, 9, 8
  357. &movz ($acc,&HB("ebx")); # 11
  358. &xor ("edx",&DWP(1,$tbl,$acc,8)); # 11
  359. &shr ("ebx",16); # 15,14
  360. &movz ($acc,&HB("eax")); # 5
  361. &xor ("ecx",&DWP(3,$tbl,$acc,8)); # 5
  362. &movq ("mm3",QWP(16,$key));
  363. &movz ($acc,&HB("ebx")); # 15
  364. &xor ("ecx",&DWP(1,$tbl,$acc,8)); # 15
  365. &movd ("mm0","ecx"); # t[0] collected
  366. &movz ($acc,&LB("eax")); # 4
  367. &mov ("ecx",&DWP(0,$tbl,$acc,8)); # 4
  368. &movd ("eax","mm2"); # 7, 6, 3, 2
  369. &movz ($acc,&LB("ebx")); # 14
  370. &xor ("ecx",&DWP(2,$tbl,$acc,8)); # 14
  371. &movd ("ebx","mm6"); # 13,12, 9, 8
  372. &movz ($acc,&HB("eax")); # 3
  373. &xor ("ecx",&DWP(1,$tbl,$acc,8)); # 3
  374. &movz ($acc,&HB("ebx")); # 9
  375. &xor ("ecx",&DWP(3,$tbl,$acc,8)); # 9
  376. &movd ("mm1","ecx"); # t[1] collected
  377. &movz ($acc,&LB("eax")); # 2
  378. &mov ("ecx",&DWP(2,$tbl,$acc,8)); # 2
  379. &shr ("eax",16); # 7, 6
  380. &punpckldq ("mm0","mm1"); # t[0,1] collected
  381. &movz ($acc,&LB("ebx")); # 8
  382. &xor ("ecx",&DWP(0,$tbl,$acc,8)); # 8
  383. &shr ("ebx",16); # 13,12
  384. &movz ($acc,&HB("eax")); # 7
  385. &xor ("ecx",&DWP(1,$tbl,$acc,8)); # 7
  386. &pxor ("mm0","mm3");
  387. &movz ("eax",&LB("eax")); # 6
  388. &xor ("edx",&DWP(2,$tbl,"eax",8)); # 6
  389. &pshufw ("mm1","mm0",0x08); # 5, 4, 1, 0
  390. &movz ($acc,&HB("ebx")); # 13
  391. &xor ("ecx",&DWP(3,$tbl,$acc,8)); # 13
  392. &xor ("ecx",&DWP(24,$key)); # t[2]
  393. &movd ("mm4","ecx"); # t[2] collected
  394. &movz ("ebx",&LB("ebx")); # 12
  395. &xor ("edx",&DWP(0,$tbl,"ebx",8)); # 12
  396. &shr ("ecx",16);
  397. &movd ("eax","mm1"); # 5, 4, 1, 0
  398. &mov ("ebx",&DWP(28,$key)); # t[3]
  399. &xor ("ebx","edx");
  400. &movd ("mm5","ebx"); # t[3] collected
  401. &and ("ebx",0xffff0000);
  402. &or ("ebx","ecx");
  403. &punpckldq ("mm4","mm5"); # t[2,3] collected
  404. }
  405. ######################################################################
  406. # "Compact" block function
  407. ######################################################################
  408. sub enccompact()
  409. { my $Fn = \&mov;
  410. while ($#_>5) { pop(@_); $Fn=sub{}; }
  411. my ($i,$te,@s)=@_;
  412. my $tmp = $key;
  413. my $out = $i==3?$s[0]:$acc;
  414. # $Fn is used in first compact round and its purpose is to
  415. # void restoration of some values from stack, so that after
  416. # 4xenccompact with extra argument $key value is left there...
  417. if ($i==3) { &$Fn ($key,$__key); }##%edx
  418. else { &mov ($out,$s[0]); }
  419. &and ($out,0xFF);
  420. if ($i==1) { &shr ($s[0],16); }#%ebx[1]
  421. if ($i==2) { &shr ($s[0],24); }#%ecx[2]
  422. &movz ($out,&BP(-128,$te,$out,1));
  423. if ($i==3) { $tmp=$s[1]; }##%eax
  424. &movz ($tmp,&HB($s[1]));
  425. &movz ($tmp,&BP(-128,$te,$tmp,1));
  426. &shl ($tmp,8);
  427. &xor ($out,$tmp);
  428. if ($i==3) { $tmp=$s[2]; &mov ($s[1],$__s0); }##%ebx
  429. else { &mov ($tmp,$s[2]);
  430. &shr ($tmp,16); }
  431. if ($i==2) { &and ($s[1],0xFF); }#%edx[2]
  432. &and ($tmp,0xFF);
  433. &movz ($tmp,&BP(-128,$te,$tmp,1));
  434. &shl ($tmp,16);
  435. &xor ($out,$tmp);
  436. if ($i==3) { $tmp=$s[3]; &mov ($s[2],$__s1); }##%ecx
  437. elsif($i==2){ &movz ($tmp,&HB($s[3])); }#%ebx[2]
  438. else { &mov ($tmp,$s[3]);
  439. &shr ($tmp,24); }
  440. &movz ($tmp,&BP(-128,$te,$tmp,1));
  441. &shl ($tmp,24);
  442. &xor ($out,$tmp);
  443. if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
  444. if ($i==3) { &mov ($s[3],$acc); }
  445. &comment();
  446. }
  447. sub enctransform()
  448. { my @s = ($s0,$s1,$s2,$s3);
  449. my $i = shift;
  450. my $tmp = $tbl;
  451. my $r2 = $key ;
  452. &and ($tmp,$s[$i]);
  453. &lea ($r2,&DWP(0,$s[$i],$s[$i]));
  454. &mov ($acc,$tmp);
  455. &shr ($tmp,7);
  456. &and ($r2,0xfefefefe);
  457. &sub ($acc,$tmp);
  458. &mov ($tmp,$s[$i]);
  459. &and ($acc,0x1b1b1b1b);
  460. &rotr ($tmp,16);
  461. &xor ($acc,$r2); # r2
  462. &mov ($r2,$s[$i]);
  463. &xor ($s[$i],$acc); # r0 ^ r2
  464. &rotr ($r2,16+8);
  465. &xor ($acc,$tmp);
  466. &rotl ($s[$i],24);
  467. &xor ($acc,$r2);
  468. &mov ($tmp,0x80808080) if ($i!=1);
  469. &xor ($s[$i],$acc); # ROTATE(r2^r0,24) ^ r2
  470. }
  471. &function_begin_B("_x86_AES_encrypt_compact");
  472. # note that caller is expected to allocate stack frame for me!
  473. &mov ($__key,$key); # save key
  474. &xor ($s0,&DWP(0,$key)); # xor with key
  475. &xor ($s1,&DWP(4,$key));
  476. &xor ($s2,&DWP(8,$key));
  477. &xor ($s3,&DWP(12,$key));
  478. &mov ($acc,&DWP(240,$key)); # load key->rounds
  479. &lea ($acc,&DWP(-2,$acc,$acc));
  480. &lea ($acc,&DWP(0,$key,$acc,8));
  481. &mov ($__end,$acc); # end of key schedule
  482. # prefetch Te4
  483. &mov ($key,&DWP(0-128,$tbl));
  484. &mov ($acc,&DWP(32-128,$tbl));
  485. &mov ($key,&DWP(64-128,$tbl));
  486. &mov ($acc,&DWP(96-128,$tbl));
  487. &mov ($key,&DWP(128-128,$tbl));
  488. &mov ($acc,&DWP(160-128,$tbl));
  489. &mov ($key,&DWP(192-128,$tbl));
  490. &mov ($acc,&DWP(224-128,$tbl));
  491. &set_label("loop",16);
  492. &enccompact(0,$tbl,$s0,$s1,$s2,$s3,1);
  493. &enccompact(1,$tbl,$s1,$s2,$s3,$s0,1);
  494. &enccompact(2,$tbl,$s2,$s3,$s0,$s1,1);
  495. &enccompact(3,$tbl,$s3,$s0,$s1,$s2,1);
  496. &mov ($tbl,0x80808080);
  497. &enctransform(2);
  498. &enctransform(3);
  499. &enctransform(0);
  500. &enctransform(1);
  501. &mov ($key,$__key);
  502. &mov ($tbl,$__tbl);
  503. &add ($key,16); # advance rd_key
  504. &xor ($s0,&DWP(0,$key));
  505. &xor ($s1,&DWP(4,$key));
  506. &xor ($s2,&DWP(8,$key));
  507. &xor ($s3,&DWP(12,$key));
  508. &cmp ($key,$__end);
  509. &mov ($__key,$key);
  510. &jb (&label("loop"));
  511. &enccompact(0,$tbl,$s0,$s1,$s2,$s3);
  512. &enccompact(1,$tbl,$s1,$s2,$s3,$s0);
  513. &enccompact(2,$tbl,$s2,$s3,$s0,$s1);
  514. &enccompact(3,$tbl,$s3,$s0,$s1,$s2);
  515. &xor ($s0,&DWP(16,$key));
  516. &xor ($s1,&DWP(20,$key));
  517. &xor ($s2,&DWP(24,$key));
  518. &xor ($s3,&DWP(28,$key));
  519. &ret ();
  520. &function_end_B("_x86_AES_encrypt_compact");
  521. ######################################################################
  522. # "Compact" SSE block function.
  523. ######################################################################
  524. #
  525. # Performance is not actually extraordinary in comparison to pure
  526. # x86 code. In particular encrypt performance is virtually the same.
  527. # Decrypt performance on the other hand is 15-20% better on newer
  528. # µ-archs [but we're thankful for *any* improvement here], and ~50%
  529. # better on PIII:-) And additionally on the pros side this code
  530. # eliminates redundant references to stack and thus relieves/
  531. # minimizes the pressure on the memory bus.
  532. #
  533. # MMX register layout lsb
  534. # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  535. # | mm4 | mm0 |
  536. # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  537. # | s3 | s2 | s1 | s0 |
  538. # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  539. # |15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
  540. # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
  541. #
  542. # Indexes translate as s[N/4]>>(8*(N%4)), e.g. 5 means s1>>8.
  543. # In this terms encryption and decryption "compact" permutation
  544. # matrices can be depicted as following:
  545. #
  546. # encryption lsb # decryption lsb
  547. # +----++----+----+----+----+ # +----++----+----+----+----+
  548. # | t0 || 15 | 10 | 5 | 0 | # | t0 || 7 | 10 | 13 | 0 |
  549. # +----++----+----+----+----+ # +----++----+----+----+----+
  550. # | t1 || 3 | 14 | 9 | 4 | # | t1 || 11 | 14 | 1 | 4 |
  551. # +----++----+----+----+----+ # +----++----+----+----+----+
  552. # | t2 || 7 | 2 | 13 | 8 | # | t2 || 15 | 2 | 5 | 8 |
  553. # +----++----+----+----+----+ # +----++----+----+----+----+
  554. # | t3 || 11 | 6 | 1 | 12 | # | t3 || 3 | 6 | 9 | 12 |
  555. # +----++----+----+----+----+ # +----++----+----+----+----+
  556. #
  557. ######################################################################
  558. # Why not xmm registers? Short answer. It was actually tested and
  559. # was not any faster, but *contrary*, most notably on Intel CPUs.
  560. # Longer answer. Main advantage of using mm registers is that movd
  561. # latency is lower, especially on Intel P4. While arithmetic
  562. # instructions are twice as many, they can be scheduled every cycle
  563. # and not every second one when they are operating on xmm register,
  564. # so that "arithmetic throughput" remains virtually the same. And
  565. # finally the code can be executed even on elder SSE-only CPUs:-)
  566. sub sse_enccompact()
  567. {
  568. &pshufw ("mm1","mm0",0x08); # 5, 4, 1, 0
  569. &pshufw ("mm5","mm4",0x0d); # 15,14,11,10
  570. &movd ("eax","mm1"); # 5, 4, 1, 0
  571. &movd ("ebx","mm5"); # 15,14,11,10
  572. &mov ($__key,$key);
  573. &movz ($acc,&LB("eax")); # 0
  574. &movz ("edx",&HB("eax")); # 1
  575. &pshufw ("mm2","mm0",0x0d); # 7, 6, 3, 2
  576. &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 0
  577. &movz ($key,&LB("ebx")); # 10
  578. &movz ("edx",&BP(-128,$tbl,"edx",1)); # 1
  579. &shr ("eax",16); # 5, 4
  580. &shl ("edx",8); # 1
  581. &movz ($acc,&BP(-128,$tbl,$key,1)); # 10
  582. &movz ($key,&HB("ebx")); # 11
  583. &shl ($acc,16); # 10
  584. &pshufw ("mm6","mm4",0x08); # 13,12, 9, 8
  585. &or ("ecx",$acc); # 10
  586. &movz ($acc,&BP(-128,$tbl,$key,1)); # 11
  587. &movz ($key,&HB("eax")); # 5
  588. &shl ($acc,24); # 11
  589. &shr ("ebx",16); # 15,14
  590. &or ("edx",$acc); # 11
  591. &movz ($acc,&BP(-128,$tbl,$key,1)); # 5
  592. &movz ($key,&HB("ebx")); # 15
  593. &shl ($acc,8); # 5
  594. &or ("ecx",$acc); # 5
  595. &movz ($acc,&BP(-128,$tbl,$key,1)); # 15
  596. &movz ($key,&LB("eax")); # 4
  597. &shl ($acc,24); # 15
  598. &or ("ecx",$acc); # 15
  599. &movz ($acc,&BP(-128,$tbl,$key,1)); # 4
  600. &movz ($key,&LB("ebx")); # 14
  601. &movd ("eax","mm2"); # 7, 6, 3, 2
  602. &movd ("mm0","ecx"); # t[0] collected
  603. &movz ("ecx",&BP(-128,$tbl,$key,1)); # 14
  604. &movz ($key,&HB("eax")); # 3
  605. &shl ("ecx",16); # 14
  606. &movd ("ebx","mm6"); # 13,12, 9, 8
  607. &or ("ecx",$acc); # 14
  608. &movz ($acc,&BP(-128,$tbl,$key,1)); # 3
  609. &movz ($key,&HB("ebx")); # 9
  610. &shl ($acc,24); # 3
  611. &or ("ecx",$acc); # 3
  612. &movz ($acc,&BP(-128,$tbl,$key,1)); # 9
  613. &movz ($key,&LB("ebx")); # 8
  614. &shl ($acc,8); # 9
  615. &shr ("ebx",16); # 13,12
  616. &or ("ecx",$acc); # 9
  617. &movz ($acc,&BP(-128,$tbl,$key,1)); # 8
  618. &movz ($key,&LB("eax")); # 2
  619. &shr ("eax",16); # 7, 6
  620. &movd ("mm1","ecx"); # t[1] collected
  621. &movz ("ecx",&BP(-128,$tbl,$key,1)); # 2
  622. &movz ($key,&HB("eax")); # 7
  623. &shl ("ecx",16); # 2
  624. &and ("eax",0xff); # 6
  625. &or ("ecx",$acc); # 2
  626. &punpckldq ("mm0","mm1"); # t[0,1] collected
  627. &movz ($acc,&BP(-128,$tbl,$key,1)); # 7
  628. &movz ($key,&HB("ebx")); # 13
  629. &shl ($acc,24); # 7
  630. &and ("ebx",0xff); # 12
  631. &movz ("eax",&BP(-128,$tbl,"eax",1)); # 6
  632. &or ("ecx",$acc); # 7
  633. &shl ("eax",16); # 6
  634. &movz ($acc,&BP(-128,$tbl,$key,1)); # 13
  635. &or ("edx","eax"); # 6
  636. &shl ($acc,8); # 13
  637. &movz ("ebx",&BP(-128,$tbl,"ebx",1)); # 12
  638. &or ("ecx",$acc); # 13
  639. &or ("edx","ebx"); # 12
  640. &mov ($key,$__key);
  641. &movd ("mm4","ecx"); # t[2] collected
  642. &movd ("mm5","edx"); # t[3] collected
  643. &punpckldq ("mm4","mm5"); # t[2,3] collected
  644. }
  645. if (!$x86only) {
  646. &function_begin_B("_sse_AES_encrypt_compact");
  647. &pxor ("mm0",&QWP(0,$key)); # 7, 6, 5, 4, 3, 2, 1, 0
  648. &pxor ("mm4",&QWP(8,$key)); # 15,14,13,12,11,10, 9, 8
  649. # note that caller is expected to allocate stack frame for me!
  650. &mov ($acc,&DWP(240,$key)); # load key->rounds
  651. &lea ($acc,&DWP(-2,$acc,$acc));
  652. &lea ($acc,&DWP(0,$key,$acc,8));
  653. &mov ($__end,$acc); # end of key schedule
  654. &mov ($s0,0x1b1b1b1b); # magic constant
  655. &mov (&DWP(8,"esp"),$s0);
  656. &mov (&DWP(12,"esp"),$s0);
  657. # prefetch Te4
  658. &mov ($s0,&DWP(0-128,$tbl));
  659. &mov ($s1,&DWP(32-128,$tbl));
  660. &mov ($s2,&DWP(64-128,$tbl));
  661. &mov ($s3,&DWP(96-128,$tbl));
  662. &mov ($s0,&DWP(128-128,$tbl));
  663. &mov ($s1,&DWP(160-128,$tbl));
  664. &mov ($s2,&DWP(192-128,$tbl));
  665. &mov ($s3,&DWP(224-128,$tbl));
  666. &set_label("loop",16);
  667. &sse_enccompact();
  668. &add ($key,16);
  669. &cmp ($key,$__end);
  670. &ja (&label("out"));
  671. &movq ("mm2",&QWP(8,"esp"));
  672. &pxor ("mm3","mm3"); &pxor ("mm7","mm7");
  673. &movq ("mm1","mm0"); &movq ("mm5","mm4"); # r0
  674. &pcmpgtb("mm3","mm0"); &pcmpgtb("mm7","mm4");
  675. &pand ("mm3","mm2"); &pand ("mm7","mm2");
  676. &pshufw ("mm2","mm0",0xb1); &pshufw ("mm6","mm4",0xb1);# ROTATE(r0,16)
  677. &paddb ("mm0","mm0"); &paddb ("mm4","mm4");
  678. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # = r2
  679. &pshufw ("mm3","mm2",0xb1); &pshufw ("mm7","mm6",0xb1);# r0
  680. &pxor ("mm1","mm0"); &pxor ("mm5","mm4"); # r0^r2
  681. &pxor ("mm0","mm2"); &pxor ("mm4","mm6"); # ^= ROTATE(r0,16)
  682. &movq ("mm2","mm3"); &movq ("mm6","mm7");
  683. &pslld ("mm3",8); &pslld ("mm7",8);
  684. &psrld ("mm2",24); &psrld ("mm6",24);
  685. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= r0<<8
  686. &pxor ("mm0","mm2"); &pxor ("mm4","mm6"); # ^= r0>>24
  687. &movq ("mm3","mm1"); &movq ("mm7","mm5");
  688. &movq ("mm2",&QWP(0,$key)); &movq ("mm6",&QWP(8,$key));
  689. &psrld ("mm1",8); &psrld ("mm5",8);
  690. &mov ($s0,&DWP(0-128,$tbl));
  691. &pslld ("mm3",24); &pslld ("mm7",24);
  692. &mov ($s1,&DWP(64-128,$tbl));
  693. &pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= (r2^r0)<<8
  694. &mov ($s2,&DWP(128-128,$tbl));
  695. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= (r2^r0)>>24
  696. &mov ($s3,&DWP(192-128,$tbl));
  697. &pxor ("mm0","mm2"); &pxor ("mm4","mm6");
  698. &jmp (&label("loop"));
  699. &set_label("out",16);
  700. &pxor ("mm0",&QWP(0,$key));
  701. &pxor ("mm4",&QWP(8,$key));
  702. &ret ();
  703. &function_end_B("_sse_AES_encrypt_compact");
  704. }
  705. ######################################################################
  706. # Vanilla block function.
  707. ######################################################################
  708. sub encstep()
  709. { my ($i,$te,@s) = @_;
  710. my $tmp = $key;
  711. my $out = $i==3?$s[0]:$acc;
  712. # lines marked with #%e?x[i] denote "reordered" instructions...
  713. if ($i==3) { &mov ($key,$__key); }##%edx
  714. else { &mov ($out,$s[0]);
  715. &and ($out,0xFF); }
  716. if ($i==1) { &shr ($s[0],16); }#%ebx[1]
  717. if ($i==2) { &shr ($s[0],24); }#%ecx[2]
  718. &mov ($out,&DWP(0,$te,$out,8));
  719. if ($i==3) { $tmp=$s[1]; }##%eax
  720. &movz ($tmp,&HB($s[1]));
  721. &xor ($out,&DWP(3,$te,$tmp,8));
  722. if ($i==3) { $tmp=$s[2]; &mov ($s[1],$__s0); }##%ebx
  723. else { &mov ($tmp,$s[2]);
  724. &shr ($tmp,16); }
  725. if ($i==2) { &and ($s[1],0xFF); }#%edx[2]
  726. &and ($tmp,0xFF);
  727. &xor ($out,&DWP(2,$te,$tmp,8));
  728. if ($i==3) { $tmp=$s[3]; &mov ($s[2],$__s1); }##%ecx
  729. elsif($i==2){ &movz ($tmp,&HB($s[3])); }#%ebx[2]
  730. else { &mov ($tmp,$s[3]);
  731. &shr ($tmp,24) }
  732. &xor ($out,&DWP(1,$te,$tmp,8));
  733. if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
  734. if ($i==3) { &mov ($s[3],$acc); }
  735. &comment();
  736. }
  737. sub enclast()
  738. { my ($i,$te,@s)=@_;
  739. my $tmp = $key;
  740. my $out = $i==3?$s[0]:$acc;
  741. if ($i==3) { &mov ($key,$__key); }##%edx
  742. else { &mov ($out,$s[0]); }
  743. &and ($out,0xFF);
  744. if ($i==1) { &shr ($s[0],16); }#%ebx[1]
  745. if ($i==2) { &shr ($s[0],24); }#%ecx[2]
  746. &mov ($out,&DWP(2,$te,$out,8));
  747. &and ($out,0x000000ff);
  748. if ($i==3) { $tmp=$s[1]; }##%eax
  749. &movz ($tmp,&HB($s[1]));
  750. &mov ($tmp,&DWP(0,$te,$tmp,8));
  751. &and ($tmp,0x0000ff00);
  752. &xor ($out,$tmp);
  753. if ($i==3) { $tmp=$s[2]; &mov ($s[1],$__s0); }##%ebx
  754. else { &mov ($tmp,$s[2]);
  755. &shr ($tmp,16); }
  756. if ($i==2) { &and ($s[1],0xFF); }#%edx[2]
  757. &and ($tmp,0xFF);
  758. &mov ($tmp,&DWP(0,$te,$tmp,8));
  759. &and ($tmp,0x00ff0000);
  760. &xor ($out,$tmp);
  761. if ($i==3) { $tmp=$s[3]; &mov ($s[2],$__s1); }##%ecx
  762. elsif($i==2){ &movz ($tmp,&HB($s[3])); }#%ebx[2]
  763. else { &mov ($tmp,$s[3]);
  764. &shr ($tmp,24); }
  765. &mov ($tmp,&DWP(2,$te,$tmp,8));
  766. &and ($tmp,0xff000000);
  767. &xor ($out,$tmp);
  768. if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
  769. if ($i==3) { &mov ($s[3],$acc); }
  770. }
  771. &function_begin_B("_x86_AES_encrypt");
  772. if ($vertical_spin) {
  773. # I need high parts of volatile registers to be accessible...
  774. &exch ($s1="edi",$key="ebx");
  775. &mov ($s2="esi",$acc="ecx");
  776. }
  777. # note that caller is expected to allocate stack frame for me!
  778. &mov ($__key,$key); # save key
  779. &xor ($s0,&DWP(0,$key)); # xor with key
  780. &xor ($s1,&DWP(4,$key));
  781. &xor ($s2,&DWP(8,$key));
  782. &xor ($s3,&DWP(12,$key));
  783. &mov ($acc,&DWP(240,$key)); # load key->rounds
  784. if ($small_footprint) {
  785. &lea ($acc,&DWP(-2,$acc,$acc));
  786. &lea ($acc,&DWP(0,$key,$acc,8));
  787. &mov ($__end,$acc); # end of key schedule
  788. &set_label("loop",16);
  789. if ($vertical_spin) {
  790. &encvert($tbl,$s0,$s1,$s2,$s3);
  791. } else {
  792. &encstep(0,$tbl,$s0,$s1,$s2,$s3);
  793. &encstep(1,$tbl,$s1,$s2,$s3,$s0);
  794. &encstep(2,$tbl,$s2,$s3,$s0,$s1);
  795. &encstep(3,$tbl,$s3,$s0,$s1,$s2);
  796. }
  797. &add ($key,16); # advance rd_key
  798. &xor ($s0,&DWP(0,$key));
  799. &xor ($s1,&DWP(4,$key));
  800. &xor ($s2,&DWP(8,$key));
  801. &xor ($s3,&DWP(12,$key));
  802. &cmp ($key,$__end);
  803. &mov ($__key,$key);
  804. &jb (&label("loop"));
  805. }
  806. else {
  807. &cmp ($acc,10);
  808. &jle (&label("10rounds"));
  809. &cmp ($acc,12);
  810. &jle (&label("12rounds"));
  811. &set_label("14rounds",4);
  812. for ($i=1;$i<3;$i++) {
  813. if ($vertical_spin) {
  814. &encvert($tbl,$s0,$s1,$s2,$s3);
  815. } else {
  816. &encstep(0,$tbl,$s0,$s1,$s2,$s3);
  817. &encstep(1,$tbl,$s1,$s2,$s3,$s0);
  818. &encstep(2,$tbl,$s2,$s3,$s0,$s1);
  819. &encstep(3,$tbl,$s3,$s0,$s1,$s2);
  820. }
  821. &xor ($s0,&DWP(16*$i+0,$key));
  822. &xor ($s1,&DWP(16*$i+4,$key));
  823. &xor ($s2,&DWP(16*$i+8,$key));
  824. &xor ($s3,&DWP(16*$i+12,$key));
  825. }
  826. &add ($key,32);
  827. &mov ($__key,$key); # advance rd_key
  828. &set_label("12rounds",4);
  829. for ($i=1;$i<3;$i++) {
  830. if ($vertical_spin) {
  831. &encvert($tbl,$s0,$s1,$s2,$s3);
  832. } else {
  833. &encstep(0,$tbl,$s0,$s1,$s2,$s3);
  834. &encstep(1,$tbl,$s1,$s2,$s3,$s0);
  835. &encstep(2,$tbl,$s2,$s3,$s0,$s1);
  836. &encstep(3,$tbl,$s3,$s0,$s1,$s2);
  837. }
  838. &xor ($s0,&DWP(16*$i+0,$key));
  839. &xor ($s1,&DWP(16*$i+4,$key));
  840. &xor ($s2,&DWP(16*$i+8,$key));
  841. &xor ($s3,&DWP(16*$i+12,$key));
  842. }
  843. &add ($key,32);
  844. &mov ($__key,$key); # advance rd_key
  845. &set_label("10rounds",4);
  846. for ($i=1;$i<10;$i++) {
  847. if ($vertical_spin) {
  848. &encvert($tbl,$s0,$s1,$s2,$s3);
  849. } else {
  850. &encstep(0,$tbl,$s0,$s1,$s2,$s3);
  851. &encstep(1,$tbl,$s1,$s2,$s3,$s0);
  852. &encstep(2,$tbl,$s2,$s3,$s0,$s1);
  853. &encstep(3,$tbl,$s3,$s0,$s1,$s2);
  854. }
  855. &xor ($s0,&DWP(16*$i+0,$key));
  856. &xor ($s1,&DWP(16*$i+4,$key));
  857. &xor ($s2,&DWP(16*$i+8,$key));
  858. &xor ($s3,&DWP(16*$i+12,$key));
  859. }
  860. }
  861. if ($vertical_spin) {
  862. # "reincarnate" some registers for "horizontal" spin...
  863. &mov ($s1="ebx",$key="edi");
  864. &mov ($s2="ecx",$acc="esi");
  865. }
  866. &enclast(0,$tbl,$s0,$s1,$s2,$s3);
  867. &enclast(1,$tbl,$s1,$s2,$s3,$s0);
  868. &enclast(2,$tbl,$s2,$s3,$s0,$s1);
  869. &enclast(3,$tbl,$s3,$s0,$s1,$s2);
  870. &add ($key,$small_footprint?16:160);
  871. &xor ($s0,&DWP(0,$key));
  872. &xor ($s1,&DWP(4,$key));
  873. &xor ($s2,&DWP(8,$key));
  874. &xor ($s3,&DWP(12,$key));
  875. &ret ();
  876. &set_label("AES_Te",64); # Yes! I keep it in the code segment!
  877. &_data_word(0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6);
  878. &_data_word(0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591);
  879. &_data_word(0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56);
  880. &_data_word(0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec);
  881. &_data_word(0x45caca8f, 0x9d82821f, 0x40c9c989, 0x877d7dfa);
  882. &_data_word(0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb);
  883. &_data_word(0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45);
  884. &_data_word(0xbf9c9c23, 0xf7a4a453, 0x967272e4, 0x5bc0c09b);
  885. &_data_word(0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c);
  886. &_data_word(0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83);
  887. &_data_word(0x5c343468, 0xf4a5a551, 0x34e5e5d1, 0x08f1f1f9);
  888. &_data_word(0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a);
  889. &_data_word(0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d);
  890. &_data_word(0x28181830, 0xa1969637, 0x0f05050a, 0xb59a9a2f);
  891. &_data_word(0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df);
  892. &_data_word(0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea);
  893. &_data_word(0x1b090912, 0x9e83831d, 0x742c2c58, 0x2e1a1a34);
  894. &_data_word(0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b);
  895. &_data_word(0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d);
  896. &_data_word(0x7b292952, 0x3ee3e3dd, 0x712f2f5e, 0x97848413);
  897. &_data_word(0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1);
  898. &_data_word(0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6);
  899. &_data_word(0xbe6a6ad4, 0x46cbcb8d, 0xd9bebe67, 0x4b393972);
  900. &_data_word(0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85);
  901. &_data_word(0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed);
  902. &_data_word(0xc5434386, 0xd74d4d9a, 0x55333366, 0x94858511);
  903. &_data_word(0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe);
  904. &_data_word(0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b);
  905. &_data_word(0xf35151a2, 0xfea3a35d, 0xc0404080, 0x8a8f8f05);
  906. &_data_word(0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1);
  907. &_data_word(0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142);
  908. &_data_word(0x30101020, 0x1affffe5, 0x0ef3f3fd, 0x6dd2d2bf);
  909. &_data_word(0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3);
  910. &_data_word(0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e);
  911. &_data_word(0x57c4c493, 0xf2a7a755, 0x827e7efc, 0x473d3d7a);
  912. &_data_word(0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6);
  913. &_data_word(0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3);
  914. &_data_word(0x66222244, 0x7e2a2a54, 0xab90903b, 0x8388880b);
  915. &_data_word(0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428);
  916. &_data_word(0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad);
  917. &_data_word(0x3be0e0db, 0x56323264, 0x4e3a3a74, 0x1e0a0a14);
  918. &_data_word(0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8);
  919. &_data_word(0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4);
  920. &_data_word(0xa8919139, 0xa4959531, 0x37e4e4d3, 0x8b7979f2);
  921. &_data_word(0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda);
  922. &_data_word(0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949);
  923. &_data_word(0xb46c6cd8, 0xfa5656ac, 0x07f4f4f3, 0x25eaeacf);
  924. &_data_word(0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810);
  925. &_data_word(0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c);
  926. &_data_word(0x241c1c38, 0xf1a6a657, 0xc7b4b473, 0x51c6c697);
  927. &_data_word(0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e);
  928. &_data_word(0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f);
  929. &_data_word(0x907070e0, 0x423e3e7c, 0xc4b5b571, 0xaa6666cc);
  930. &_data_word(0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c);
  931. &_data_word(0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969);
  932. &_data_word(0x91868617, 0x58c1c199, 0x271d1d3a, 0xb99e9e27);
  933. &_data_word(0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122);
  934. &_data_word(0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433);
  935. &_data_word(0xb69b9b2d, 0x221e1e3c, 0x92878715, 0x20e9e9c9);
  936. &_data_word(0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5);
  937. &_data_word(0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a);
  938. &_data_word(0xdabfbf65, 0x31e6e6d7, 0xc6424284, 0xb86868d0);
  939. &_data_word(0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e);
  940. &_data_word(0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c);
  941. #Te4 # four copies of Te4 to choose from to avoid L1 aliasing
  942. &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5);
  943. &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76);
  944. &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0);
  945. &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0);
  946. &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc);
  947. &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15);
  948. &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a);
  949. &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75);
  950. &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0);
  951. &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84);
  952. &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b);
  953. &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf);
  954. &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85);
  955. &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8);
  956. &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5);
  957. &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2);
  958. &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17);
  959. &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73);
  960. &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88);
  961. &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb);
  962. &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c);
  963. &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79);
  964. &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9);
  965. &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08);
  966. &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6);
  967. &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a);
  968. &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e);
  969. &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e);
  970. &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94);
  971. &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf);
  972. &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68);
  973. &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16);
  974. &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5);
  975. &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76);
  976. &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0);
  977. &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0);
  978. &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc);
  979. &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15);
  980. &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a);
  981. &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75);
  982. &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0);
  983. &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84);
  984. &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b);
  985. &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf);
  986. &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85);
  987. &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8);
  988. &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5);
  989. &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2);
  990. &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17);
  991. &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73);
  992. &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88);
  993. &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb);
  994. &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c);
  995. &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79);
  996. &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9);
  997. &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08);
  998. &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6);
  999. &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a);
  1000. &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e);
  1001. &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e);
  1002. &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94);
  1003. &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf);
  1004. &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68);
  1005. &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16);
  1006. &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5);
  1007. &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76);
  1008. &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0);
  1009. &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0);
  1010. &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc);
  1011. &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15);
  1012. &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a);
  1013. &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75);
  1014. &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0);
  1015. &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84);
  1016. &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b);
  1017. &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf);
  1018. &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85);
  1019. &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8);
  1020. &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5);
  1021. &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2);
  1022. &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17);
  1023. &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73);
  1024. &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88);
  1025. &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb);
  1026. &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c);
  1027. &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79);
  1028. &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9);
  1029. &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08);
  1030. &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6);
  1031. &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a);
  1032. &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e);
  1033. &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e);
  1034. &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94);
  1035. &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf);
  1036. &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68);
  1037. &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16);
  1038. &data_byte(0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5);
  1039. &data_byte(0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76);
  1040. &data_byte(0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0);
  1041. &data_byte(0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0);
  1042. &data_byte(0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc);
  1043. &data_byte(0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15);
  1044. &data_byte(0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a);
  1045. &data_byte(0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75);
  1046. &data_byte(0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0);
  1047. &data_byte(0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84);
  1048. &data_byte(0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b);
  1049. &data_byte(0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf);
  1050. &data_byte(0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85);
  1051. &data_byte(0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8);
  1052. &data_byte(0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5);
  1053. &data_byte(0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2);
  1054. &data_byte(0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17);
  1055. &data_byte(0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73);
  1056. &data_byte(0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88);
  1057. &data_byte(0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb);
  1058. &data_byte(0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c);
  1059. &data_byte(0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79);
  1060. &data_byte(0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9);
  1061. &data_byte(0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08);
  1062. &data_byte(0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6);
  1063. &data_byte(0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a);
  1064. &data_byte(0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e);
  1065. &data_byte(0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e);
  1066. &data_byte(0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94);
  1067. &data_byte(0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf);
  1068. &data_byte(0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68);
  1069. &data_byte(0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16);
  1070. #rcon:
  1071. &data_word(0x00000001, 0x00000002, 0x00000004, 0x00000008);
  1072. &data_word(0x00000010, 0x00000020, 0x00000040, 0x00000080);
  1073. &data_word(0x0000001b, 0x00000036, 0x00000000, 0x00000000);
  1074. &data_word(0x00000000, 0x00000000, 0x00000000, 0x00000000);
  1075. &function_end_B("_x86_AES_encrypt");
  1076. # void AES_encrypt (const void *inp,void *out,const AES_KEY *key);
  1077. &function_begin("AES_encrypt");
  1078. &mov ($acc,&wparam(0)); # load inp
  1079. &mov ($key,&wparam(2)); # load key
  1080. &mov ($s0,"esp");
  1081. &sub ("esp",36);
  1082. &and ("esp",-64); # align to cache-line
  1083. # place stack frame just "above" the key schedule
  1084. &lea ($s1,&DWP(-64-63,$key));
  1085. &sub ($s1,"esp");
  1086. &neg ($s1);
  1087. &and ($s1,0x3C0); # modulo 1024, but aligned to cache-line
  1088. &sub ("esp",$s1);
  1089. &add ("esp",4); # 4 is reserved for caller's return address
  1090. &mov ($_esp,$s0); # save stack pointer
  1091. &call (&label("pic_point")); # make it PIC!
  1092. &set_label("pic_point");
  1093. &blindpop($tbl);
  1094. &picmeup($s0,"OPENSSL_ia32cap_P",$tbl,&label("pic_point")) if (!$x86only);
  1095. &lea ($tbl,&DWP(&label("AES_Te")."-".&label("pic_point"),$tbl));
  1096. # pick Te4 copy which can't "overlap" with stack frame or key schedule
  1097. &lea ($s1,&DWP(768-4,"esp"));
  1098. &sub ($s1,$tbl);
  1099. &and ($s1,0x300);
  1100. &lea ($tbl,&DWP(2048+128,$tbl,$s1));
  1101. if (!$x86only) {
  1102. &bt (&DWP(0,$s0),25); # check for SSE bit
  1103. &jnc (&label("x86"));
  1104. &movq ("mm0",&QWP(0,$acc));
  1105. &movq ("mm4",&QWP(8,$acc));
  1106. &call ("_sse_AES_encrypt_compact");
  1107. &mov ("esp",$_esp); # restore stack pointer
  1108. &mov ($acc,&wparam(1)); # load out
  1109. &movq (&QWP(0,$acc),"mm0"); # write output data
  1110. &movq (&QWP(8,$acc),"mm4");
  1111. &emms ();
  1112. &function_end_A();
  1113. }
  1114. &set_label("x86",16);
  1115. &mov ($_tbl,$tbl);
  1116. &mov ($s0,&DWP(0,$acc)); # load input data
  1117. &mov ($s1,&DWP(4,$acc));
  1118. &mov ($s2,&DWP(8,$acc));
  1119. &mov ($s3,&DWP(12,$acc));
  1120. &call ("_x86_AES_encrypt_compact");
  1121. &mov ("esp",$_esp); # restore stack pointer
  1122. &mov ($acc,&wparam(1)); # load out
  1123. &mov (&DWP(0,$acc),$s0); # write output data
  1124. &mov (&DWP(4,$acc),$s1);
  1125. &mov (&DWP(8,$acc),$s2);
  1126. &mov (&DWP(12,$acc),$s3);
  1127. &function_end("AES_encrypt");
  1128. #--------------------------------------------------------------------#
  1129. ######################################################################
  1130. # "Compact" block function
  1131. ######################################################################
  1132. sub deccompact()
  1133. { my $Fn = \&mov;
  1134. while ($#_>5) { pop(@_); $Fn=sub{}; }
  1135. my ($i,$td,@s)=@_;
  1136. my $tmp = $key;
  1137. my $out = $i==3?$s[0]:$acc;
  1138. # $Fn is used in first compact round and its purpose is to
  1139. # void restoration of some values from stack, so that after
  1140. # 4xdeccompact with extra argument $key, $s0 and $s1 values
  1141. # are left there...
  1142. if($i==3) { &$Fn ($key,$__key); }
  1143. else { &mov ($out,$s[0]); }
  1144. &and ($out,0xFF);
  1145. &movz ($out,&BP(-128,$td,$out,1));
  1146. if ($i==3) { $tmp=$s[1]; }
  1147. &movz ($tmp,&HB($s[1]));
  1148. &movz ($tmp,&BP(-128,$td,$tmp,1));
  1149. &shl ($tmp,8);
  1150. &xor ($out,$tmp);
  1151. if ($i==3) { $tmp=$s[2]; &mov ($s[1],$acc); }
  1152. else { mov ($tmp,$s[2]); }
  1153. &shr ($tmp,16);
  1154. &and ($tmp,0xFF);
  1155. &movz ($tmp,&BP(-128,$td,$tmp,1));
  1156. &shl ($tmp,16);
  1157. &xor ($out,$tmp);
  1158. if ($i==3) { $tmp=$s[3]; &$Fn ($s[2],$__s1); }
  1159. else { &mov ($tmp,$s[3]); }
  1160. &shr ($tmp,24);
  1161. &movz ($tmp,&BP(-128,$td,$tmp,1));
  1162. &shl ($tmp,24);
  1163. &xor ($out,$tmp);
  1164. if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
  1165. if ($i==3) { &$Fn ($s[3],$__s0); }
  1166. }
  1167. # must be called with 2,3,0,1 as argument sequence!!!
  1168. sub dectransform()
  1169. { my @s = ($s0,$s1,$s2,$s3);
  1170. my $i = shift;
  1171. my $tmp = $key;
  1172. my $tp2 = @s[($i+2)%4]; $tp2 = @s[2] if ($i==1);
  1173. my $tp4 = @s[($i+3)%4]; $tp4 = @s[3] if ($i==1);
  1174. my $tp8 = $tbl;
  1175. &mov ($tmp,0x80808080);
  1176. &and ($tmp,$s[$i]);
  1177. &mov ($acc,$tmp);
  1178. &shr ($tmp,7);
  1179. &lea ($tp2,&DWP(0,$s[$i],$s[$i]));
  1180. &sub ($acc,$tmp);
  1181. &and ($tp2,0xfefefefe);
  1182. &and ($acc,0x1b1b1b1b);
  1183. &xor ($tp2,$acc);
  1184. &mov ($tmp,0x80808080);
  1185. &and ($tmp,$tp2);
  1186. &mov ($acc,$tmp);
  1187. &shr ($tmp,7);
  1188. &lea ($tp4,&DWP(0,$tp2,$tp2));
  1189. &sub ($acc,$tmp);
  1190. &and ($tp4,0xfefefefe);
  1191. &and ($acc,0x1b1b1b1b);
  1192. &xor ($tp2,$s[$i]); # tp2^tp1
  1193. &xor ($tp4,$acc);
  1194. &mov ($tmp,0x80808080);
  1195. &and ($tmp,$tp4);
  1196. &mov ($acc,$tmp);
  1197. &shr ($tmp,7);
  1198. &lea ($tp8,&DWP(0,$tp4,$tp4));
  1199. &sub ($acc,$tmp);
  1200. &and ($tp8,0xfefefefe);
  1201. &and ($acc,0x1b1b1b1b);
  1202. &xor ($tp4,$s[$i]); # tp4^tp1
  1203. &rotl ($s[$i],8); # = ROTATE(tp1,8)
  1204. &xor ($tp8,$acc);
  1205. &xor ($s[$i],$tp2);
  1206. &xor ($tp2,$tp8);
  1207. &xor ($s[$i],$tp4);
  1208. &xor ($tp4,$tp8);
  1209. &rotl ($tp2,24);
  1210. &xor ($s[$i],$tp8); # ^= tp8^(tp4^tp1)^(tp2^tp1)
  1211. &rotl ($tp4,16);
  1212. &xor ($s[$i],$tp2); # ^= ROTATE(tp8^tp2^tp1,24)
  1213. &rotl ($tp8,8);
  1214. &xor ($s[$i],$tp4); # ^= ROTATE(tp8^tp4^tp1,16)
  1215. &mov ($s[0],$__s0) if($i==2); #prefetch $s0
  1216. &mov ($s[1],$__s1) if($i==3); #prefetch $s1
  1217. &mov ($s[2],$__s2) if($i==1);
  1218. &xor ($s[$i],$tp8); # ^= ROTATE(tp8,8)
  1219. &mov ($s[3],$__s3) if($i==1);
  1220. &mov (&DWP(4+4*$i,"esp"),$s[$i]) if($i>=2);
  1221. }
  1222. &function_begin_B("_x86_AES_decrypt_compact");
  1223. # note that caller is expected to allocate stack frame for me!
  1224. &mov ($__key,$key); # save key
  1225. &xor ($s0,&DWP(0,$key)); # xor with key
  1226. &xor ($s1,&DWP(4,$key));
  1227. &xor ($s2,&DWP(8,$key));
  1228. &xor ($s3,&DWP(12,$key));
  1229. &mov ($acc,&DWP(240,$key)); # load key->rounds
  1230. &lea ($acc,&DWP(-2,$acc,$acc));
  1231. &lea ($acc,&DWP(0,$key,$acc,8));
  1232. &mov ($__end,$acc); # end of key schedule
  1233. # prefetch Td4
  1234. &mov ($key,&DWP(0-128,$tbl));
  1235. &mov ($acc,&DWP(32-128,$tbl));
  1236. &mov ($key,&DWP(64-128,$tbl));
  1237. &mov ($acc,&DWP(96-128,$tbl));
  1238. &mov ($key,&DWP(128-128,$tbl));
  1239. &mov ($acc,&DWP(160-128,$tbl));
  1240. &mov ($key,&DWP(192-128,$tbl));
  1241. &mov ($acc,&DWP(224-128,$tbl));
  1242. &set_label("loop",16);
  1243. &deccompact(0,$tbl,$s0,$s3,$s2,$s1,1);
  1244. &deccompact(1,$tbl,$s1,$s0,$s3,$s2,1);
  1245. &deccompact(2,$tbl,$s2,$s1,$s0,$s3,1);
  1246. &deccompact(3,$tbl,$s3,$s2,$s1,$s0,1);
  1247. &dectransform(2);
  1248. &dectransform(3);
  1249. &dectransform(0);
  1250. &dectransform(1);
  1251. &mov ($key,$__key);
  1252. &mov ($tbl,$__tbl);
  1253. &add ($key,16); # advance rd_key
  1254. &xor ($s0,&DWP(0,$key));
  1255. &xor ($s1,&DWP(4,$key));
  1256. &xor ($s2,&DWP(8,$key));
  1257. &xor ($s3,&DWP(12,$key));
  1258. &cmp ($key,$__end);
  1259. &mov ($__key,$key);
  1260. &jb (&label("loop"));
  1261. &deccompact(0,$tbl,$s0,$s3,$s2,$s1);
  1262. &deccompact(1,$tbl,$s1,$s0,$s3,$s2);
  1263. &deccompact(2,$tbl,$s2,$s1,$s0,$s3);
  1264. &deccompact(3,$tbl,$s3,$s2,$s1,$s0);
  1265. &xor ($s0,&DWP(16,$key));
  1266. &xor ($s1,&DWP(20,$key));
  1267. &xor ($s2,&DWP(24,$key));
  1268. &xor ($s3,&DWP(28,$key));
  1269. &ret ();
  1270. &function_end_B("_x86_AES_decrypt_compact");
  1271. ######################################################################
  1272. # "Compact" SSE block function.
  1273. ######################################################################
  1274. sub sse_deccompact()
  1275. {
  1276. &pshufw ("mm1","mm0",0x0c); # 7, 6, 1, 0
  1277. &pshufw ("mm5","mm4",0x09); # 13,12,11,10
  1278. &movd ("eax","mm1"); # 7, 6, 1, 0
  1279. &movd ("ebx","mm5"); # 13,12,11,10
  1280. &mov ($__key,$key);
  1281. &movz ($acc,&LB("eax")); # 0
  1282. &movz ("edx",&HB("eax")); # 1
  1283. &pshufw ("mm2","mm0",0x06); # 3, 2, 5, 4
  1284. &movz ("ecx",&BP(-128,$tbl,$acc,1)); # 0
  1285. &movz ($key,&LB("ebx")); # 10
  1286. &movz ("edx",&BP(-128,$tbl,"edx",1)); # 1
  1287. &shr ("eax",16); # 7, 6
  1288. &shl ("edx",8); # 1
  1289. &movz ($acc,&BP(-128,$tbl,$key,1)); # 10
  1290. &movz ($key,&HB("ebx")); # 11
  1291. &shl ($acc,16); # 10
  1292. &pshufw ("mm6","mm4",0x03); # 9, 8,15,14
  1293. &or ("ecx",$acc); # 10
  1294. &movz ($acc,&BP(-128,$tbl,$key,1)); # 11
  1295. &movz ($key,&HB("eax")); # 7
  1296. &shl ($acc,24); # 11
  1297. &shr ("ebx",16); # 13,12
  1298. &or ("edx",$acc); # 11
  1299. &movz ($acc,&BP(-128,$tbl,$key,1)); # 7
  1300. &movz ($key,&HB("ebx")); # 13
  1301. &shl ($acc,24); # 7
  1302. &or ("ecx",$acc); # 7
  1303. &movz ($acc,&BP(-128,$tbl,$key,1)); # 13
  1304. &movz ($key,&LB("eax")); # 6
  1305. &shl ($acc,8); # 13
  1306. &movd ("eax","mm2"); # 3, 2, 5, 4
  1307. &or ("ecx",$acc); # 13
  1308. &movz ($acc,&BP(-128,$tbl,$key,1)); # 6
  1309. &movz ($key,&LB("ebx")); # 12
  1310. &shl ($acc,16); # 6
  1311. &movd ("ebx","mm6"); # 9, 8,15,14
  1312. &movd ("mm0","ecx"); # t[0] collected
  1313. &movz ("ecx",&BP(-128,$tbl,$key,1)); # 12
  1314. &movz ($key,&LB("eax")); # 4
  1315. &or ("ecx",$acc); # 12
  1316. &movz ($acc,&BP(-128,$tbl,$key,1)); # 4
  1317. &movz ($key,&LB("ebx")); # 14
  1318. &or ("edx",$acc); # 4
  1319. &movz ($acc,&BP(-128,$tbl,$key,1)); # 14
  1320. &movz ($key,&HB("eax")); # 5
  1321. &shl ($acc,16); # 14
  1322. &shr ("eax",16); # 3, 2
  1323. &or ("edx",$acc); # 14
  1324. &movz ($acc,&BP(-128,$tbl,$key,1)); # 5
  1325. &movz ($key,&HB("ebx")); # 15
  1326. &shr ("ebx",16); # 9, 8
  1327. &shl ($acc,8); # 5
  1328. &movd ("mm1","edx"); # t[1] collected
  1329. &movz ("edx",&BP(-128,$tbl,$key,1)); # 15
  1330. &movz ($key,&HB("ebx")); # 9
  1331. &shl ("edx",24); # 15
  1332. &and ("ebx",0xff); # 8
  1333. &or ("edx",$acc); # 15
  1334. &punpckldq ("mm0","mm1"); # t[0,1] collected
  1335. &movz ($acc,&BP(-128,$tbl,$key,1)); # 9
  1336. &movz ($key,&LB("eax")); # 2
  1337. &shl ($acc,8); # 9
  1338. &movz ("eax",&HB("eax")); # 3
  1339. &movz ("ebx",&BP(-128,$tbl,"ebx",1)); # 8
  1340. &or ("ecx",$acc); # 9
  1341. &movz ($acc,&BP(-128,$tbl,$key,1)); # 2
  1342. &or ("edx","ebx"); # 8
  1343. &shl ($acc,16); # 2
  1344. &movz ("eax",&BP(-128,$tbl,"eax",1)); # 3
  1345. &or ("edx",$acc); # 2
  1346. &shl ("eax",24); # 3
  1347. &or ("ecx","eax"); # 3
  1348. &mov ($key,$__key);
  1349. &movd ("mm4","edx"); # t[2] collected
  1350. &movd ("mm5","ecx"); # t[3] collected
  1351. &punpckldq ("mm4","mm5"); # t[2,3] collected
  1352. }
  1353. if (!$x86only) {
  1354. &function_begin_B("_sse_AES_decrypt_compact");
  1355. &pxor ("mm0",&QWP(0,$key)); # 7, 6, 5, 4, 3, 2, 1, 0
  1356. &pxor ("mm4",&QWP(8,$key)); # 15,14,13,12,11,10, 9, 8
  1357. # note that caller is expected to allocate stack frame for me!
  1358. &mov ($acc,&DWP(240,$key)); # load key->rounds
  1359. &lea ($acc,&DWP(-2,$acc,$acc));
  1360. &lea ($acc,&DWP(0,$key,$acc,8));
  1361. &mov ($__end,$acc); # end of key schedule
  1362. &mov ($s0,0x1b1b1b1b); # magic constant
  1363. &mov (&DWP(8,"esp"),$s0);
  1364. &mov (&DWP(12,"esp"),$s0);
  1365. # prefetch Td4
  1366. &mov ($s0,&DWP(0-128,$tbl));
  1367. &mov ($s1,&DWP(32-128,$tbl));
  1368. &mov ($s2,&DWP(64-128,$tbl));
  1369. &mov ($s3,&DWP(96-128,$tbl));
  1370. &mov ($s0,&DWP(128-128,$tbl));
  1371. &mov ($s1,&DWP(160-128,$tbl));
  1372. &mov ($s2,&DWP(192-128,$tbl));
  1373. &mov ($s3,&DWP(224-128,$tbl));
  1374. &set_label("loop",16);
  1375. &sse_deccompact();
  1376. &add ($key,16);
  1377. &cmp ($key,$__end);
  1378. &ja (&label("out"));
  1379. # ROTATE(x^y,N) == ROTATE(x,N)^ROTATE(y,N)
  1380. &movq ("mm3","mm0"); &movq ("mm7","mm4");
  1381. &movq ("mm2","mm0",1); &movq ("mm6","mm4",1);
  1382. &movq ("mm1","mm0"); &movq ("mm5","mm4");
  1383. &pshufw ("mm0","mm0",0xb1); &pshufw ("mm4","mm4",0xb1);# = ROTATE(tp0,16)
  1384. &pslld ("mm2",8); &pslld ("mm6",8);
  1385. &psrld ("mm3",8); &psrld ("mm7",8);
  1386. &pxor ("mm0","mm2"); &pxor ("mm4","mm6"); # ^= tp0<<8
  1387. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= tp0>>8
  1388. &pslld ("mm2",16); &pslld ("mm6",16);
  1389. &psrld ("mm3",16); &psrld ("mm7",16);
  1390. &pxor ("mm0","mm2"); &pxor ("mm4","mm6"); # ^= tp0<<24
  1391. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= tp0>>24
  1392. &movq ("mm3",&QWP(8,"esp"));
  1393. &pxor ("mm2","mm2"); &pxor ("mm6","mm6");
  1394. &pcmpgtb("mm2","mm1"); &pcmpgtb("mm6","mm5");
  1395. &pand ("mm2","mm3"); &pand ("mm6","mm3");
  1396. &paddb ("mm1","mm1"); &paddb ("mm5","mm5");
  1397. &pxor ("mm1","mm2"); &pxor ("mm5","mm6"); # tp2
  1398. &movq ("mm3","mm1"); &movq ("mm7","mm5");
  1399. &movq ("mm2","mm1"); &movq ("mm6","mm5");
  1400. &pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= tp2
  1401. &pslld ("mm3",24); &pslld ("mm7",24);
  1402. &psrld ("mm2",8); &psrld ("mm6",8);
  1403. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= tp2<<24
  1404. &pxor ("mm0","mm2"); &pxor ("mm4","mm6"); # ^= tp2>>8
  1405. &movq ("mm2",&QWP(8,"esp"));
  1406. &pxor ("mm3","mm3"); &pxor ("mm7","mm7");
  1407. &pcmpgtb("mm3","mm1"); &pcmpgtb("mm7","mm5");
  1408. &pand ("mm3","mm2"); &pand ("mm7","mm2");
  1409. &paddb ("mm1","mm1"); &paddb ("mm5","mm5");
  1410. &pxor ("mm1","mm3"); &pxor ("mm5","mm7"); # tp4
  1411. &pshufw ("mm3","mm1",0xb1); &pshufw ("mm7","mm5",0xb1);
  1412. &pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= tp4
  1413. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= ROTATE(tp4,16)
  1414. &pxor ("mm3","mm3"); &pxor ("mm7","mm7");
  1415. &pcmpgtb("mm3","mm1"); &pcmpgtb("mm7","mm5");
  1416. &pand ("mm3","mm2"); &pand ("mm7","mm2");
  1417. &paddb ("mm1","mm1"); &paddb ("mm5","mm5");
  1418. &pxor ("mm1","mm3"); &pxor ("mm5","mm7"); # tp8
  1419. &pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= tp8
  1420. &movq ("mm3","mm1"); &movq ("mm7","mm5");
  1421. &pshufw ("mm2","mm1",0xb1); &pshufw ("mm6","mm5",0xb1);
  1422. &pxor ("mm0","mm2"); &pxor ("mm4","mm6"); # ^= ROTATE(tp8,16)
  1423. &pslld ("mm1",8); &pslld ("mm5",8);
  1424. &psrld ("mm3",8); &psrld ("mm7",8);
  1425. &movq ("mm2",&QWP(0,$key)); &movq ("mm6",&QWP(8,$key));
  1426. &pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= tp8<<8
  1427. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= tp8>>8
  1428. &mov ($s0,&DWP(0-128,$tbl));
  1429. &pslld ("mm1",16); &pslld ("mm5",16);
  1430. &mov ($s1,&DWP(64-128,$tbl));
  1431. &psrld ("mm3",16); &psrld ("mm7",16);
  1432. &mov ($s2,&DWP(128-128,$tbl));
  1433. &pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= tp8<<24
  1434. &mov ($s3,&DWP(192-128,$tbl));
  1435. &pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= tp8>>24
  1436. &pxor ("mm0","mm2"); &pxor ("mm4","mm6");
  1437. &jmp (&label("loop"));
  1438. &set_label("out",16);
  1439. &pxor ("mm0",&QWP(0,$key));
  1440. &pxor ("mm4",&QWP(8,$key));
  1441. &ret ();
  1442. &function_end_B("_sse_AES_decrypt_compact");
  1443. }
  1444. ######################################################################
  1445. # Vanilla block function.
  1446. ######################################################################
  1447. sub decstep()
  1448. { my ($i,$td,@s) = @_;
  1449. my $tmp = $key;
  1450. my $out = $i==3?$s[0]:$acc;
  1451. # no instructions are reordered, as performance appears
  1452. # optimal... or rather that all attempts to reorder didn't
  1453. # result in better performance [which by the way is not a
  1454. # bit lower than encryption].
  1455. if($i==3) { &mov ($key,$__key); }
  1456. else { &mov ($out,$s[0]); }
  1457. &and ($out,0xFF);
  1458. &mov ($out,&DWP(0,$td,$out,8));
  1459. if ($i==3) { $tmp=$s[1]; }
  1460. &movz ($tmp,&HB($s[1]));
  1461. &xor ($out,&DWP(3,$td,$tmp,8));
  1462. if ($i==3) { $tmp=$s[2]; &mov ($s[1],$acc); }
  1463. else { &mov ($tmp,$s[2]); }
  1464. &shr ($tmp,16);
  1465. &and ($tmp,0xFF);
  1466. &xor ($out,&DWP(2,$td,$tmp,8));
  1467. if ($i==3) { $tmp=$s[3]; &mov ($s[2],$__s1); }
  1468. else { &mov ($tmp,$s[3]); }
  1469. &shr ($tmp,24);
  1470. &xor ($out,&DWP(1,$td,$tmp,8));
  1471. if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
  1472. if ($i==3) { &mov ($s[3],$__s0); }
  1473. &comment();
  1474. }
  1475. sub declast()
  1476. { my ($i,$td,@s)=@_;
  1477. my $tmp = $key;
  1478. my $out = $i==3?$s[0]:$acc;
  1479. if($i==0) { &lea ($td,&DWP(2048+128,$td));
  1480. &mov ($tmp,&DWP(0-128,$td));
  1481. &mov ($acc,&DWP(32-128,$td));
  1482. &mov ($tmp,&DWP(64-128,$td));
  1483. &mov ($acc,&DWP(96-128,$td));
  1484. &mov ($tmp,&DWP(128-128,$td));
  1485. &mov ($acc,&DWP(160-128,$td));
  1486. &mov ($tmp,&DWP(192-128,$td));
  1487. &mov ($acc,&DWP(224-128,$td));
  1488. &lea ($td,&DWP(-128,$td)); }
  1489. if($i==3) { &mov ($key,$__key); }
  1490. else { &mov ($out,$s[0]); }
  1491. &and ($out,0xFF);
  1492. &movz ($out,&BP(0,$td,$out,1));
  1493. if ($i==3) { $tmp=$s[1]; }
  1494. &movz ($tmp,&HB($s[1]));
  1495. &movz ($tmp,&BP(0,$td,$tmp,1));
  1496. &shl ($tmp,8);
  1497. &xor ($out,$tmp);
  1498. if ($i==3) { $tmp=$s[2]; &mov ($s[1],$acc); }
  1499. else { mov ($tmp,$s[2]); }
  1500. &shr ($tmp,16);
  1501. &and ($tmp,0xFF);
  1502. &movz ($tmp,&BP(0,$td,$tmp,1));
  1503. &shl ($tmp,16);
  1504. &xor ($out,$tmp);
  1505. if ($i==3) { $tmp=$s[3]; &mov ($s[2],$__s1); }
  1506. else { &mov ($tmp,$s[3]); }
  1507. &shr ($tmp,24);
  1508. &movz ($tmp,&BP(0,$td,$tmp,1));
  1509. &shl ($tmp,24);
  1510. &xor ($out,$tmp);
  1511. if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
  1512. if ($i==3) { &mov ($s[3],$__s0);
  1513. &lea ($td,&DWP(-2048,$td)); }
  1514. }
  1515. &function_begin_B("_x86_AES_decrypt");
  1516. # note that caller is expected to allocate stack frame for me!
  1517. &mov ($__key,$key); # save key
  1518. &xor ($s0,&DWP(0,$key)); # xor with key
  1519. &xor ($s1,&DWP(4,$key));
  1520. &xor ($s2,&DWP(8,$key));
  1521. &xor ($s3,&DWP(12,$key));
  1522. &mov ($acc,&DWP(240,$key)); # load key->rounds
  1523. if ($small_footprint) {
  1524. &lea ($acc,&DWP(-2,$acc,$acc));
  1525. &lea ($acc,&DWP(0,$key,$acc,8));
  1526. &mov ($__end,$acc); # end of key schedule
  1527. &set_label("loop",16);
  1528. &decstep(0,$tbl,$s0,$s3,$s2,$s1);
  1529. &decstep(1,$tbl,$s1,$s0,$s3,$s2);
  1530. &decstep(2,$tbl,$s2,$s1,$s0,$s3);
  1531. &decstep(3,$tbl,$s3,$s2,$s1,$s0);
  1532. &add ($key,16); # advance rd_key
  1533. &xor ($s0,&DWP(0,$key));
  1534. &xor ($s1,&DWP(4,$key));
  1535. &xor ($s2,&DWP(8,$key));
  1536. &xor ($s3,&DWP(12,$key));
  1537. &cmp ($key,$__end);
  1538. &mov ($__key,$key);
  1539. &jb (&label("loop"));
  1540. }
  1541. else {
  1542. &cmp ($acc,10);
  1543. &jle (&label("10rounds"));
  1544. &cmp ($acc,12);
  1545. &jle (&label("12rounds"));
  1546. &set_label("14rounds",4);
  1547. for ($i=1;$i<3;$i++) {
  1548. &decstep(0,$tbl,$s0,$s3,$s2,$s1);
  1549. &decstep(1,$tbl,$s1,$s0,$s3,$s2);
  1550. &decstep(2,$tbl,$s2,$s1,$s0,$s3);
  1551. &decstep(3,$tbl,$s3,$s2,$s1,$s0);
  1552. &xor ($s0,&DWP(16*$i+0,$key));
  1553. &xor ($s1,&DWP(16*$i+4,$key));
  1554. &xor ($s2,&DWP(16*$i+8,$key));
  1555. &xor ($s3,&DWP(16*$i+12,$key));
  1556. }
  1557. &add ($key,32);
  1558. &mov ($__key,$key); # advance rd_key
  1559. &set_label("12rounds",4);
  1560. for ($i=1;$i<3;$i++) {
  1561. &decstep(0,$tbl,$s0,$s3,$s2,$s1);
  1562. &decstep(1,$tbl,$s1,$s0,$s3,$s2);
  1563. &decstep(2,$tbl,$s2,$s1,$s0,$s3);
  1564. &decstep(3,$tbl,$s3,$s2,$s1,$s0);
  1565. &xor ($s0,&DWP(16*$i+0,$key));
  1566. &xor ($s1,&DWP(16*$i+4,$key));
  1567. &xor ($s2,&DWP(16*$i+8,$key));
  1568. &xor ($s3,&DWP(16*$i+12,$key));
  1569. }
  1570. &add ($key,32);
  1571. &mov ($__key,$key); # advance rd_key
  1572. &set_label("10rounds",4);
  1573. for ($i=1;$i<10;$i++) {
  1574. &decstep(0,$tbl,$s0,$s3,$s2,$s1);
  1575. &decstep(1,$tbl,$s1,$s0,$s3,$s2);
  1576. &decstep(2,$tbl,$s2,$s1,$s0,$s3);
  1577. &decstep(3,$tbl,$s3,$s2,$s1,$s0);
  1578. &xor ($s0,&DWP(16*$i+0,$key));
  1579. &xor ($s1,&DWP(16*$i+4,$key));
  1580. &xor ($s2,&DWP(16*$i+8,$key));
  1581. &xor ($s3,&DWP(16*$i+12,$key));
  1582. }
  1583. }
  1584. &declast(0,$tbl,$s0,$s3,$s2,$s1);
  1585. &declast(1,$tbl,$s1,$s0,$s3,$s2);
  1586. &declast(2,$tbl,$s2,$s1,$s0,$s3);
  1587. &declast(3,$tbl,$s3,$s2,$s1,$s0);
  1588. &add ($key,$small_footprint?16:160);
  1589. &xor ($s0,&DWP(0,$key));
  1590. &xor ($s1,&DWP(4,$key));
  1591. &xor ($s2,&DWP(8,$key));
  1592. &xor ($s3,&DWP(12,$key));
  1593. &ret ();
  1594. &set_label("AES_Td",64); # Yes! I keep it in the code segment!
  1595. &_data_word(0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a);
  1596. &_data_word(0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b);
  1597. &_data_word(0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5);
  1598. &_data_word(0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5);
  1599. &_data_word(0x495ab1de, 0x671bba25, 0x980eea45, 0xe1c0fe5d);
  1600. &_data_word(0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b);
  1601. &_data_word(0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295);
  1602. &_data_word(0x2d83bed4, 0xd3217458, 0x2969e049, 0x44c8c98e);
  1603. &_data_word(0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927);
  1604. &_data_word(0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d);
  1605. &_data_word(0x184adf63, 0x82311ae5, 0x60335197, 0x457f5362);
  1606. &_data_word(0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9);
  1607. &_data_word(0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52);
  1608. &_data_word(0x23d373ab, 0xe2024b72, 0x578f1fe3, 0x2aab5566);
  1609. &_data_word(0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3);
  1610. &_data_word(0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed);
  1611. &_data_word(0x2b1ccf8a, 0x92b479a7, 0xf0f207f3, 0xa1e2694e);
  1612. &_data_word(0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4);
  1613. &_data_word(0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4);
  1614. &_data_word(0x39ec830b, 0xaaef6040, 0x069f715e, 0x51106ebd);
  1615. &_data_word(0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d);
  1616. &_data_word(0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060);
  1617. &_data_word(0x24fb9819, 0x97e9bdd6, 0xcc434089, 0x779ed967);
  1618. &_data_word(0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879);
  1619. &_data_word(0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000);
  1620. &_data_word(0x83868009, 0x48ed2b32, 0xac70111e, 0x4e725a6c);
  1621. &_data_word(0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36);
  1622. &_data_word(0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624);
  1623. &_data_word(0xb1670a0c, 0x0fe75793, 0xd296eeb4, 0x9e919b1b);
  1624. &_data_word(0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c);
  1625. &_data_word(0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12);
  1626. &_data_word(0x0b0d090e, 0xadc78bf2, 0xb9a8b62d, 0xc8a91e14);
  1627. &_data_word(0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3);
  1628. &_data_word(0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b);
  1629. &_data_word(0x7629438b, 0xdcc623cb, 0x68fcedb6, 0x63f1e4b8);
  1630. &_data_word(0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684);
  1631. &_data_word(0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7);
  1632. &_data_word(0x4b2f9e1d, 0xf330b2dc, 0xec52860d, 0xd0e3c177);
  1633. &_data_word(0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947);
  1634. &_data_word(0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322);
  1635. &_data_word(0xc74e4987, 0xc1d138d9, 0xfea2ca8c, 0x360bd498);
  1636. &_data_word(0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f);
  1637. &_data_word(0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54);
  1638. &_data_word(0xc2138df6, 0xe8b8d890, 0x5ef7392e, 0xf5afc382);
  1639. &_data_word(0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf);
  1640. &_data_word(0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb);
  1641. &_data_word(0x097826cd, 0xf418596e, 0x01b79aec, 0xa89a4f83);
  1642. &_data_word(0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef);
  1643. &_data_word(0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029);
  1644. &_data_word(0xafb2a431, 0x31233f2a, 0x3094a5c6, 0xc066a235);
  1645. &_data_word(0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733);
  1646. &_data_word(0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117);
  1647. &_data_word(0x8dd64d76, 0x4db0ef43, 0x544daacc, 0xdf0496e4);
  1648. &_data_word(0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546);
  1649. &_data_word(0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb);
  1650. &_data_word(0x5a1d67b3, 0x52d2db92, 0x335610e9, 0x1347d66d);
  1651. &_data_word(0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb);
  1652. &_data_word(0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a);
  1653. &_data_word(0x59dfd29c, 0x3f73f255, 0x79ce1418, 0xbf37c773);
  1654. &_data_word(0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478);
  1655. &_data_word(0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2);
  1656. &_data_word(0x72c31d16, 0x0c25e2bc, 0x8b493c28, 0x41950dff);
  1657. &_data_word(0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664);
  1658. &_data_word(0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0);
  1659. #Td4: # four copies of Td4 to choose from to avoid L1 aliasing
  1660. &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38);
  1661. &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb);
  1662. &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87);
  1663. &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb);
  1664. &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d);
  1665. &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e);
  1666. &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2);
  1667. &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25);
  1668. &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16);
  1669. &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92);
  1670. &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda);
  1671. &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84);
  1672. &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a);
  1673. &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06);
  1674. &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02);
  1675. &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b);
  1676. &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea);
  1677. &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73);
  1678. &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85);
  1679. &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e);
  1680. &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89);
  1681. &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b);
  1682. &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20);
  1683. &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4);
  1684. &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31);
  1685. &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f);
  1686. &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d);
  1687. &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef);
  1688. &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0);
  1689. &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61);
  1690. &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26);
  1691. &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d);
  1692. &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38);
  1693. &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb);
  1694. &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87);
  1695. &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb);
  1696. &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d);
  1697. &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e);
  1698. &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2);
  1699. &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25);
  1700. &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16);
  1701. &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92);
  1702. &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda);
  1703. &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84);
  1704. &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a);
  1705. &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06);
  1706. &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02);
  1707. &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b);
  1708. &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea);
  1709. &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73);
  1710. &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85);
  1711. &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e);
  1712. &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89);
  1713. &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b);
  1714. &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20);
  1715. &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4);
  1716. &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31);
  1717. &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f);
  1718. &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d);
  1719. &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef);
  1720. &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0);
  1721. &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61);
  1722. &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26);
  1723. &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d);
  1724. &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38);
  1725. &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb);
  1726. &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87);
  1727. &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb);
  1728. &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d);
  1729. &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e);
  1730. &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2);
  1731. &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25);
  1732. &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16);
  1733. &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92);
  1734. &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda);
  1735. &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84);
  1736. &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a);
  1737. &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06);
  1738. &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02);
  1739. &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b);
  1740. &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea);
  1741. &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73);
  1742. &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85);
  1743. &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e);
  1744. &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89);
  1745. &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b);
  1746. &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20);
  1747. &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4);
  1748. &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31);
  1749. &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f);
  1750. &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d);
  1751. &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef);
  1752. &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0);
  1753. &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61);
  1754. &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26);
  1755. &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d);
  1756. &data_byte(0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38);
  1757. &data_byte(0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb);
  1758. &data_byte(0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87);
  1759. &data_byte(0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb);
  1760. &data_byte(0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d);
  1761. &data_byte(0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e);
  1762. &data_byte(0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2);
  1763. &data_byte(0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25);
  1764. &data_byte(0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16);
  1765. &data_byte(0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92);
  1766. &data_byte(0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda);
  1767. &data_byte(0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84);
  1768. &data_byte(0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a);
  1769. &data_byte(0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06);
  1770. &data_byte(0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02);
  1771. &data_byte(0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b);
  1772. &data_byte(0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea);
  1773. &data_byte(0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73);
  1774. &data_byte(0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85);
  1775. &data_byte(0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e);
  1776. &data_byte(0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89);
  1777. &data_byte(0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b);
  1778. &data_byte(0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20);
  1779. &data_byte(0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4);
  1780. &data_byte(0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31);
  1781. &data_byte(0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f);
  1782. &data_byte(0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d);
  1783. &data_byte(0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef);
  1784. &data_byte(0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0);
  1785. &data_byte(0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61);
  1786. &data_byte(0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26);
  1787. &data_byte(0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d);
  1788. &function_end_B("_x86_AES_decrypt");
  1789. # void AES_decrypt (const void *inp,void *out,const AES_KEY *key);
  1790. &function_begin("AES_decrypt");
  1791. &mov ($acc,&wparam(0)); # load inp
  1792. &mov ($key,&wparam(2)); # load key
  1793. &mov ($s0,"esp");
  1794. &sub ("esp",36);
  1795. &and ("esp",-64); # align to cache-line
  1796. # place stack frame just "above" the key schedule
  1797. &lea ($s1,&DWP(-64-63,$key));
  1798. &sub ($s1,"esp");
  1799. &neg ($s1);
  1800. &and ($s1,0x3C0); # modulo 1024, but aligned to cache-line
  1801. &sub ("esp",$s1);
  1802. &add ("esp",4); # 4 is reserved for caller's return address
  1803. &mov ($_esp,$s0); # save stack pointer
  1804. &call (&label("pic_point")); # make it PIC!
  1805. &set_label("pic_point");
  1806. &blindpop($tbl);
  1807. &picmeup($s0,"OPENSSL_ia32cap_P",$tbl,&label("pic_point")) if(!$x86only);
  1808. &lea ($tbl,&DWP(&label("AES_Td")."-".&label("pic_point"),$tbl));
  1809. # pick Td4 copy which can't "overlap" with stack frame or key schedule
  1810. &lea ($s1,&DWP(768-4,"esp"));
  1811. &sub ($s1,$tbl);
  1812. &and ($s1,0x300);
  1813. &lea ($tbl,&DWP(2048+128,$tbl,$s1));
  1814. if (!$x86only) {
  1815. &bt (&DWP(0,$s0),25); # check for SSE bit
  1816. &jnc (&label("x86"));
  1817. &movq ("mm0",&QWP(0,$acc));
  1818. &movq ("mm4",&QWP(8,$acc));
  1819. &call ("_sse_AES_decrypt_compact");
  1820. &mov ("esp",$_esp); # restore stack pointer
  1821. &mov ($acc,&wparam(1)); # load out
  1822. &movq (&QWP(0,$acc),"mm0"); # write output data
  1823. &movq (&QWP(8,$acc),"mm4");
  1824. &emms ();
  1825. &function_end_A();
  1826. }
  1827. &set_label("x86",16);
  1828. &mov ($_tbl,$tbl);
  1829. &mov ($s0,&DWP(0,$acc)); # load input data
  1830. &mov ($s1,&DWP(4,$acc));
  1831. &mov ($s2,&DWP(8,$acc));
  1832. &mov ($s3,&DWP(12,$acc));
  1833. &call ("_x86_AES_decrypt_compact");
  1834. &mov ("esp",$_esp); # restore stack pointer
  1835. &mov ($acc,&wparam(1)); # load out
  1836. &mov (&DWP(0,$acc),$s0); # write output data
  1837. &mov (&DWP(4,$acc),$s1);
  1838. &mov (&DWP(8,$acc),$s2);
  1839. &mov (&DWP(12,$acc),$s3);
  1840. &function_end("AES_decrypt");
  1841. # void AES_cbc_encrypt (const void char *inp, unsigned char *out,
  1842. # size_t length, const AES_KEY *key,
  1843. # unsigned char *ivp,const int enc);
  1844. {
  1845. # stack frame layout
  1846. # -4(%esp) # return address 0(%esp)
  1847. # 0(%esp) # s0 backing store 4(%esp)
  1848. # 4(%esp) # s1 backing store 8(%esp)
  1849. # 8(%esp) # s2 backing store 12(%esp)
  1850. # 12(%esp) # s3 backing store 16(%esp)
  1851. # 16(%esp) # key backup 20(%esp)
  1852. # 20(%esp) # end of key schedule 24(%esp)
  1853. # 24(%esp) # %ebp backup 28(%esp)
  1854. # 28(%esp) # %esp backup
  1855. my $_inp=&DWP(32,"esp"); # copy of wparam(0)
  1856. my $_out=&DWP(36,"esp"); # copy of wparam(1)
  1857. my $_len=&DWP(40,"esp"); # copy of wparam(2)
  1858. my $_key=&DWP(44,"esp"); # copy of wparam(3)
  1859. my $_ivp=&DWP(48,"esp"); # copy of wparam(4)
  1860. my $_tmp=&DWP(52,"esp"); # volatile variable
  1861. #
  1862. my $ivec=&DWP(60,"esp"); # ivec[16]
  1863. my $aes_key=&DWP(76,"esp"); # copy of aes_key
  1864. my $mark=&DWP(76+240,"esp"); # copy of aes_key->rounds
  1865. &function_begin("AES_cbc_encrypt");
  1866. &mov ($s2 eq "ecx"? $s2 : "",&wparam(2)); # load len
  1867. &cmp ($s2,0);
  1868. &je (&label("drop_out"));
  1869. &call (&label("pic_point")); # make it PIC!
  1870. &set_label("pic_point");
  1871. &blindpop($tbl);
  1872. &picmeup($s0,"OPENSSL_ia32cap_P",$tbl,&label("pic_point")) if(!$x86only);
  1873. &cmp (&wparam(5),0);
  1874. &lea ($tbl,&DWP(&label("AES_Te")."-".&label("pic_point"),$tbl));
  1875. &jne (&label("picked_te"));
  1876. &lea ($tbl,&DWP(&label("AES_Td")."-".&label("AES_Te"),$tbl));
  1877. &set_label("picked_te");
  1878. # one can argue if this is required
  1879. &pushf ();
  1880. &cld ();
  1881. &cmp ($s2,$speed_limit);
  1882. &jb (&label("slow_way"));
  1883. &test ($s2,15);
  1884. &jnz (&label("slow_way"));
  1885. if (!$x86only) {
  1886. &bt (&DWP(0,$s0),28); # check for hyper-threading bit
  1887. &jc (&label("slow_way"));
  1888. }
  1889. # pre-allocate aligned stack frame...
  1890. &lea ($acc,&DWP(-80-244,"esp"));
  1891. &and ($acc,-64);
  1892. # ... and make sure it doesn't alias with $tbl modulo 4096
  1893. &mov ($s0,$tbl);
  1894. &lea ($s1,&DWP(2048+256,$tbl));
  1895. &mov ($s3,$acc);
  1896. &and ($s0,0xfff); # s = %ebp&0xfff
  1897. &and ($s1,0xfff); # e = (%ebp+2048+256)&0xfff
  1898. &and ($s3,0xfff); # p = %esp&0xfff
  1899. &cmp ($s3,$s1); # if (p>=e) %esp =- (p-e);
  1900. &jb (&label("tbl_break_out"));
  1901. &sub ($s3,$s1);
  1902. &sub ($acc,$s3);
  1903. &jmp (&label("tbl_ok"));
  1904. &set_label("tbl_break_out",4); # else %esp -= (p-s)&0xfff + framesz;
  1905. &sub ($s3,$s0);
  1906. &and ($s3,0xfff);
  1907. &add ($s3,384);
  1908. &sub ($acc,$s3);
  1909. &set_label("tbl_ok",4);
  1910. &lea ($s3,&wparam(0)); # obtain pointer to parameter block
  1911. &exch ("esp",$acc); # allocate stack frame
  1912. &add ("esp",4); # reserve for return address!
  1913. &mov ($_tbl,$tbl); # save %ebp
  1914. &mov ($_esp,$acc); # save %esp
  1915. &mov ($s0,&DWP(0,$s3)); # load inp
  1916. &mov ($s1,&DWP(4,$s3)); # load out
  1917. #&mov ($s2,&DWP(8,$s3)); # load len
  1918. &mov ($key,&DWP(12,$s3)); # load key
  1919. &mov ($acc,&DWP(16,$s3)); # load ivp
  1920. &mov ($s3,&DWP(20,$s3)); # load enc flag
  1921. &mov ($_inp,$s0); # save copy of inp
  1922. &mov ($_out,$s1); # save copy of out
  1923. &mov ($_len,$s2); # save copy of len
  1924. &mov ($_key,$key); # save copy of key
  1925. &mov ($_ivp,$acc); # save copy of ivp
  1926. &mov ($mark,0); # copy of aes_key->rounds = 0;
  1927. # do we copy key schedule to stack?
  1928. &mov ($s1 eq "ebx" ? $s1 : "",$key);
  1929. &mov ($s2 eq "ecx" ? $s2 : "",244/4);
  1930. &sub ($s1,$tbl);
  1931. &mov ("esi",$key);
  1932. &and ($s1,0xfff);
  1933. &lea ("edi",$aes_key);
  1934. &cmp ($s1,2048+256);
  1935. &jb (&label("do_copy"));
  1936. &cmp ($s1,4096-244);
  1937. &jb (&label("skip_copy"));
  1938. &set_label("do_copy",4);
  1939. &mov ($_key,"edi");
  1940. &data_word(0xA5F3F689); # rep movsd
  1941. &set_label("skip_copy");
  1942. &mov ($key,16);
  1943. &set_label("prefetch_tbl",4);
  1944. &mov ($s0,&DWP(0,$tbl));
  1945. &mov ($s1,&DWP(32,$tbl));
  1946. &mov ($s2,&DWP(64,$tbl));
  1947. &mov ($acc,&DWP(96,$tbl));
  1948. &lea ($tbl,&DWP(128,$tbl));
  1949. &sub ($key,1);
  1950. &jnz (&label("prefetch_tbl"));
  1951. &sub ($tbl,2048);
  1952. &mov ($acc,$_inp);
  1953. &mov ($key,$_ivp);
  1954. &cmp ($s3,0);
  1955. &je (&label("fast_decrypt"));
  1956. #----------------------------- ENCRYPT -----------------------------#
  1957. &mov ($s0,&DWP(0,$key)); # load iv
  1958. &mov ($s1,&DWP(4,$key));
  1959. &set_label("fast_enc_loop",16);
  1960. &mov ($s2,&DWP(8,$key));
  1961. &mov ($s3,&DWP(12,$key));
  1962. &xor ($s0,&DWP(0,$acc)); # xor input data
  1963. &xor ($s1,&DWP(4,$acc));
  1964. &xor ($s2,&DWP(8,$acc));
  1965. &xor ($s3,&DWP(12,$acc));
  1966. &mov ($key,$_key); # load key
  1967. &call ("_x86_AES_encrypt");
  1968. &mov ($acc,$_inp); # load inp
  1969. &mov ($key,$_out); # load out
  1970. &mov (&DWP(0,$key),$s0); # save output data
  1971. &mov (&DWP(4,$key),$s1);
  1972. &mov (&DWP(8,$key),$s2);
  1973. &mov (&DWP(12,$key),$s3);
  1974. &lea ($acc,&DWP(16,$acc)); # advance inp
  1975. &mov ($s2,$_len); # load len
  1976. &mov ($_inp,$acc); # save inp
  1977. &lea ($s3,&DWP(16,$key)); # advance out
  1978. &mov ($_out,$s3); # save out
  1979. &sub ($s2,16); # decrease len
  1980. &mov ($_len,$s2); # save len
  1981. &jnz (&label("fast_enc_loop"));
  1982. &mov ($acc,$_ivp); # load ivp
  1983. &mov ($s2,&DWP(8,$key)); # restore last 2 dwords
  1984. &mov ($s3,&DWP(12,$key));
  1985. &mov (&DWP(0,$acc),$s0); # save ivec
  1986. &mov (&DWP(4,$acc),$s1);
  1987. &mov (&DWP(8,$acc),$s2);
  1988. &mov (&DWP(12,$acc),$s3);
  1989. &cmp ($mark,0); # was the key schedule copied?
  1990. &mov ("edi",$_key);
  1991. &je (&label("skip_ezero"));
  1992. # zero copy of key schedule
  1993. &mov ("ecx",240/4);
  1994. &xor ("eax","eax");
  1995. &align (4);
  1996. &data_word(0xABF3F689); # rep stosd
  1997. &set_label("skip_ezero");
  1998. &mov ("esp",$_esp);
  1999. &popf ();
  2000. &set_label("drop_out");
  2001. &function_end_A();
  2002. &pushf (); # kludge, never executed
  2003. #----------------------------- DECRYPT -----------------------------#
  2004. &set_label("fast_decrypt",16);
  2005. &cmp ($acc,$_out);
  2006. &je (&label("fast_dec_in_place")); # in-place processing...
  2007. &mov ($_tmp,$key);
  2008. &align (4);
  2009. &set_label("fast_dec_loop",16);
  2010. &mov ($s0,&DWP(0,$acc)); # read input
  2011. &mov ($s1,&DWP(4,$acc));
  2012. &mov ($s2,&DWP(8,$acc));
  2013. &mov ($s3,&DWP(12,$acc));
  2014. &mov ($key,$_key); # load key
  2015. &call ("_x86_AES_decrypt");
  2016. &mov ($key,$_tmp); # load ivp
  2017. &mov ($acc,$_len); # load len
  2018. &xor ($s0,&DWP(0,$key)); # xor iv
  2019. &xor ($s1,&DWP(4,$key));
  2020. &xor ($s2,&DWP(8,$key));
  2021. &xor ($s3,&DWP(12,$key));
  2022. &mov ($key,$_out); # load out
  2023. &mov ($acc,$_inp); # load inp
  2024. &mov (&DWP(0,$key),$s0); # write output
  2025. &mov (&DWP(4,$key),$s1);
  2026. &mov (&DWP(8,$key),$s2);
  2027. &mov (&DWP(12,$key),$s3);
  2028. &mov ($s2,$_len); # load len
  2029. &mov ($_tmp,$acc); # save ivp
  2030. &lea ($acc,&DWP(16,$acc)); # advance inp
  2031. &mov ($_inp,$acc); # save inp
  2032. &lea ($key,&DWP(16,$key)); # advance out
  2033. &mov ($_out,$key); # save out
  2034. &sub ($s2,16); # decrease len
  2035. &mov ($_len,$s2); # save len
  2036. &jnz (&label("fast_dec_loop"));
  2037. &mov ($key,$_tmp); # load temp ivp
  2038. &mov ($acc,$_ivp); # load user ivp
  2039. &mov ($s0,&DWP(0,$key)); # load iv
  2040. &mov ($s1,&DWP(4,$key));
  2041. &mov ($s2,&DWP(8,$key));
  2042. &mov ($s3,&DWP(12,$key));
  2043. &mov (&DWP(0,$acc),$s0); # copy back to user
  2044. &mov (&DWP(4,$acc),$s1);
  2045. &mov (&DWP(8,$acc),$s2);
  2046. &mov (&DWP(12,$acc),$s3);
  2047. &jmp (&label("fast_dec_out"));
  2048. &set_label("fast_dec_in_place",16);
  2049. &set_label("fast_dec_in_place_loop");
  2050. &mov ($s0,&DWP(0,$acc)); # read input
  2051. &mov ($s1,&DWP(4,$acc));
  2052. &mov ($s2,&DWP(8,$acc));
  2053. &mov ($s3,&DWP(12,$acc));
  2054. &lea ($key,$ivec);
  2055. &mov (&DWP(0,$key),$s0); # copy to temp
  2056. &mov (&DWP(4,$key),$s1);
  2057. &mov (&DWP(8,$key),$s2);
  2058. &mov (&DWP(12,$key),$s3);
  2059. &mov ($key,$_key); # load key
  2060. &call ("_x86_AES_decrypt");
  2061. &mov ($key,$_ivp); # load ivp
  2062. &mov ($acc,$_out); # load out
  2063. &xor ($s0,&DWP(0,$key)); # xor iv
  2064. &xor ($s1,&DWP(4,$key));
  2065. &xor ($s2,&DWP(8,$key));
  2066. &xor ($s3,&DWP(12,$key));
  2067. &mov (&DWP(0,$acc),$s0); # write output
  2068. &mov (&DWP(4,$acc),$s1);
  2069. &mov (&DWP(8,$acc),$s2);
  2070. &mov (&DWP(12,$acc),$s3);
  2071. &lea ($acc,&DWP(16,$acc)); # advance out
  2072. &mov ($_out,$acc); # save out
  2073. &lea ($acc,$ivec);
  2074. &mov ($s0,&DWP(0,$acc)); # read temp
  2075. &mov ($s1,&DWP(4,$acc));
  2076. &mov ($s2,&DWP(8,$acc));
  2077. &mov ($s3,&DWP(12,$acc));
  2078. &mov (&DWP(0,$key),$s0); # copy iv
  2079. &mov (&DWP(4,$key),$s1);
  2080. &mov (&DWP(8,$key),$s2);
  2081. &mov (&DWP(12,$key),$s3);
  2082. &mov ($acc,$_inp); # load inp
  2083. &mov ($s2,$_len); # load len
  2084. &lea ($acc,&DWP(16,$acc)); # advance inp
  2085. &mov ($_inp,$acc); # save inp
  2086. &sub ($s2,16); # decrease len
  2087. &mov ($_len,$s2); # save len
  2088. &jnz (&label("fast_dec_in_place_loop"));
  2089. &set_label("fast_dec_out",4);
  2090. &cmp ($mark,0); # was the key schedule copied?
  2091. &mov ("edi",$_key);
  2092. &je (&label("skip_dzero"));
  2093. # zero copy of key schedule
  2094. &mov ("ecx",240/4);
  2095. &xor ("eax","eax");
  2096. &align (4);
  2097. &data_word(0xABF3F689); # rep stosd
  2098. &set_label("skip_dzero");
  2099. &mov ("esp",$_esp);
  2100. &popf ();
  2101. &function_end_A();
  2102. &pushf (); # kludge, never executed
  2103. #--------------------------- SLOW ROUTINE ---------------------------#
  2104. &set_label("slow_way",16);
  2105. &mov ($s0,&DWP(0,$s0)) if (!$x86only);# load OPENSSL_ia32cap
  2106. &mov ($key,&wparam(3)); # load key
  2107. # pre-allocate aligned stack frame...
  2108. &lea ($acc,&DWP(-80,"esp"));
  2109. &and ($acc,-64);
  2110. # ... and make sure it doesn't alias with $key modulo 1024
  2111. &lea ($s1,&DWP(-80-63,$key));
  2112. &sub ($s1,$acc);
  2113. &neg ($s1);
  2114. &and ($s1,0x3C0); # modulo 1024, but aligned to cache-line
  2115. &sub ($acc,$s1);
  2116. # pick S-box copy which can't overlap with stack frame or $key
  2117. &lea ($s1,&DWP(768,$acc));
  2118. &sub ($s1,$tbl);
  2119. &and ($s1,0x300);
  2120. &lea ($tbl,&DWP(2048+128,$tbl,$s1));
  2121. &lea ($s3,&wparam(0)); # pointer to parameter block
  2122. &exch ("esp",$acc);
  2123. &add ("esp",4); # reserve for return address!
  2124. &mov ($_tbl,$tbl); # save %ebp
  2125. &mov ($_esp,$acc); # save %esp
  2126. &mov ($_tmp,$s0); # save OPENSSL_ia32cap
  2127. &mov ($s0,&DWP(0,$s3)); # load inp
  2128. &mov ($s1,&DWP(4,$s3)); # load out
  2129. #&mov ($s2,&DWP(8,$s3)); # load len
  2130. #&mov ($key,&DWP(12,$s3)); # load key
  2131. &mov ($acc,&DWP(16,$s3)); # load ivp
  2132. &mov ($s3,&DWP(20,$s3)); # load enc flag
  2133. &mov ($_inp,$s0); # save copy of inp
  2134. &mov ($_out,$s1); # save copy of out
  2135. &mov ($_len,$s2); # save copy of len
  2136. &mov ($_key,$key); # save copy of key
  2137. &mov ($_ivp,$acc); # save copy of ivp
  2138. &mov ($key,$acc);
  2139. &mov ($acc,$s0);
  2140. &cmp ($s3,0);
  2141. &je (&label("slow_decrypt"));
  2142. #--------------------------- SLOW ENCRYPT ---------------------------#
  2143. &cmp ($s2,16);
  2144. &mov ($s3,$s1);
  2145. &jb (&label("slow_enc_tail"));
  2146. if (!$x86only) {
  2147. &bt ($_tmp,25); # check for SSE bit
  2148. &jnc (&label("slow_enc_x86"));
  2149. &movq ("mm0",&QWP(0,$key)); # load iv
  2150. &movq ("mm4",&QWP(8,$key));
  2151. &set_label("slow_enc_loop_sse",16);
  2152. &pxor ("mm0",&QWP(0,$acc)); # xor input data
  2153. &pxor ("mm4",&QWP(8,$acc));
  2154. &mov ($key,$_key);
  2155. &call ("_sse_AES_encrypt_compact");
  2156. &mov ($acc,$_inp); # load inp
  2157. &mov ($key,$_out); # load out
  2158. &mov ($s2,$_len); # load len
  2159. &movq (&QWP(0,$key),"mm0"); # save output data
  2160. &movq (&QWP(8,$key),"mm4");
  2161. &lea ($acc,&DWP(16,$acc)); # advance inp
  2162. &mov ($_inp,$acc); # save inp
  2163. &lea ($s3,&DWP(16,$key)); # advance out
  2164. &mov ($_out,$s3); # save out
  2165. &sub ($s2,16); # decrease len
  2166. &cmp ($s2,16);
  2167. &mov ($_len,$s2); # save len
  2168. &jae (&label("slow_enc_loop_sse"));
  2169. &test ($s2,15);
  2170. &jnz (&label("slow_enc_tail"));
  2171. &mov ($acc,$_ivp); # load ivp
  2172. &movq (&QWP(0,$acc),"mm0"); # save ivec
  2173. &movq (&QWP(8,$acc),"mm4");
  2174. &emms ();
  2175. &mov ("esp",$_esp);
  2176. &popf ();
  2177. &function_end_A();
  2178. &pushf (); # kludge, never executed
  2179. }
  2180. &set_label("slow_enc_x86",16);
  2181. &mov ($s0,&DWP(0,$key)); # load iv
  2182. &mov ($s1,&DWP(4,$key));
  2183. &set_label("slow_enc_loop_x86",4);
  2184. &mov ($s2,&DWP(8,$key));
  2185. &mov ($s3,&DWP(12,$key));
  2186. &xor ($s0,&DWP(0,$acc)); # xor input data
  2187. &xor ($s1,&DWP(4,$acc));
  2188. &xor ($s2,&DWP(8,$acc));
  2189. &xor ($s3,&DWP(12,$acc));
  2190. &mov ($key,$_key); # load key
  2191. &call ("_x86_AES_encrypt_compact");
  2192. &mov ($acc,$_inp); # load inp
  2193. &mov ($key,$_out); # load out
  2194. &mov (&DWP(0,$key),$s0); # save output data
  2195. &mov (&DWP(4,$key),$s1);
  2196. &mov (&DWP(8,$key),$s2);
  2197. &mov (&DWP(12,$key),$s3);
  2198. &mov ($s2,$_len); # load len
  2199. &lea ($acc,&DWP(16,$acc)); # advance inp
  2200. &mov ($_inp,$acc); # save inp
  2201. &lea ($s3,&DWP(16,$key)); # advance out
  2202. &mov ($_out,$s3); # save out
  2203. &sub ($s2,16); # decrease len
  2204. &cmp ($s2,16);
  2205. &mov ($_len,$s2); # save len
  2206. &jae (&label("slow_enc_loop_x86"));
  2207. &test ($s2,15);
  2208. &jnz (&label("slow_enc_tail"));
  2209. &mov ($acc,$_ivp); # load ivp
  2210. &mov ($s2,&DWP(8,$key)); # restore last dwords
  2211. &mov ($s3,&DWP(12,$key));
  2212. &mov (&DWP(0,$acc),$s0); # save ivec
  2213. &mov (&DWP(4,$acc),$s1);
  2214. &mov (&DWP(8,$acc),$s2);
  2215. &mov (&DWP(12,$acc),$s3);
  2216. &mov ("esp",$_esp);
  2217. &popf ();
  2218. &function_end_A();
  2219. &pushf (); # kludge, never executed
  2220. &set_label("slow_enc_tail",16);
  2221. &emms () if (!$x86only);
  2222. &mov ($key eq "edi"? $key:"",$s3); # load out to edi
  2223. &mov ($s1,16);
  2224. &sub ($s1,$s2);
  2225. &cmp ($key,$acc eq "esi"? $acc:""); # compare with inp
  2226. &je (&label("enc_in_place"));
  2227. &align (4);
  2228. &data_word(0xA4F3F689); # rep movsb # copy input
  2229. &jmp (&label("enc_skip_in_place"));
  2230. &set_label("enc_in_place");
  2231. &lea ($key,&DWP(0,$key,$s2));
  2232. &set_label("enc_skip_in_place");
  2233. &mov ($s2,$s1);
  2234. &xor ($s0,$s0);
  2235. &align (4);
  2236. &data_word(0xAAF3F689); # rep stosb # zero tail
  2237. &mov ($key,$_ivp); # restore ivp
  2238. &mov ($acc,$s3); # output as input
  2239. &mov ($s0,&DWP(0,$key));
  2240. &mov ($s1,&DWP(4,$key));
  2241. &mov ($_len,16); # len=16
  2242. &jmp (&label("slow_enc_loop_x86")); # one more spin...
  2243. #--------------------------- SLOW DECRYPT ---------------------------#
  2244. &set_label("slow_decrypt",16);
  2245. if (!$x86only) {
  2246. &bt ($_tmp,25); # check for SSE bit
  2247. &jnc (&label("slow_dec_loop_x86"));
  2248. &set_label("slow_dec_loop_sse",4);
  2249. &movq ("mm0",&QWP(0,$acc)); # read input
  2250. &movq ("mm4",&QWP(8,$acc));
  2251. &mov ($key,$_key);
  2252. &call ("_sse_AES_decrypt_compact");
  2253. &mov ($acc,$_inp); # load inp
  2254. &lea ($s0,$ivec);
  2255. &mov ($s1,$_out); # load out
  2256. &mov ($s2,$_len); # load len
  2257. &mov ($key,$_ivp); # load ivp
  2258. &movq ("mm1",&QWP(0,$acc)); # re-read input
  2259. &movq ("mm5",&QWP(8,$acc));
  2260. &pxor ("mm0",&QWP(0,$key)); # xor iv
  2261. &pxor ("mm4",&QWP(8,$key));
  2262. &movq (&QWP(0,$key),"mm1"); # copy input to iv
  2263. &movq (&QWP(8,$key),"mm5");
  2264. &sub ($s2,16); # decrease len
  2265. &jc (&label("slow_dec_partial_sse"));
  2266. &movq (&QWP(0,$s1),"mm0"); # write output
  2267. &movq (&QWP(8,$s1),"mm4");
  2268. &lea ($s1,&DWP(16,$s1)); # advance out
  2269. &mov ($_out,$s1); # save out
  2270. &lea ($acc,&DWP(16,$acc)); # advance inp
  2271. &mov ($_inp,$acc); # save inp
  2272. &mov ($_len,$s2); # save len
  2273. &jnz (&label("slow_dec_loop_sse"));
  2274. &emms ();
  2275. &mov ("esp",$_esp);
  2276. &popf ();
  2277. &function_end_A();
  2278. &pushf (); # kludge, never executed
  2279. &set_label("slow_dec_partial_sse",16);
  2280. &movq (&QWP(0,$s0),"mm0"); # save output to temp
  2281. &movq (&QWP(8,$s0),"mm4");
  2282. &emms ();
  2283. &add ($s2 eq "ecx" ? "ecx":"",16);
  2284. &mov ("edi",$s1); # out
  2285. &mov ("esi",$s0); # temp
  2286. &align (4);
  2287. &data_word(0xA4F3F689); # rep movsb # copy partial output
  2288. &mov ("esp",$_esp);
  2289. &popf ();
  2290. &function_end_A();
  2291. &pushf (); # kludge, never executed
  2292. }
  2293. &set_label("slow_dec_loop_x86",16);
  2294. &mov ($s0,&DWP(0,$acc)); # read input
  2295. &mov ($s1,&DWP(4,$acc));
  2296. &mov ($s2,&DWP(8,$acc));
  2297. &mov ($s3,&DWP(12,$acc));
  2298. &lea ($key,$ivec);
  2299. &mov (&DWP(0,$key),$s0); # copy to temp
  2300. &mov (&DWP(4,$key),$s1);
  2301. &mov (&DWP(8,$key),$s2);
  2302. &mov (&DWP(12,$key),$s3);
  2303. &mov ($key,$_key); # load key
  2304. &call ("_x86_AES_decrypt_compact");
  2305. &mov ($key,$_ivp); # load ivp
  2306. &mov ($acc,$_len); # load len
  2307. &xor ($s0,&DWP(0,$key)); # xor iv
  2308. &xor ($s1,&DWP(4,$key));
  2309. &xor ($s2,&DWP(8,$key));
  2310. &xor ($s3,&DWP(12,$key));
  2311. &sub ($acc,16);
  2312. &jc (&label("slow_dec_partial_x86"));
  2313. &mov ($_len,$acc); # save len
  2314. &mov ($acc,$_out); # load out
  2315. &mov (&DWP(0,$acc),$s0); # write output
  2316. &mov (&DWP(4,$acc),$s1);
  2317. &mov (&DWP(8,$acc),$s2);
  2318. &mov (&DWP(12,$acc),$s3);
  2319. &lea ($acc,&DWP(16,$acc)); # advance out
  2320. &mov ($_out,$acc); # save out
  2321. &lea ($acc,$ivec);
  2322. &mov ($s0,&DWP(0,$acc)); # read temp
  2323. &mov ($s1,&DWP(4,$acc));
  2324. &mov ($s2,&DWP(8,$acc));
  2325. &mov ($s3,&DWP(12,$acc));
  2326. &mov (&DWP(0,$key),$s0); # copy it to iv
  2327. &mov (&DWP(4,$key),$s1);
  2328. &mov (&DWP(8,$key),$s2);
  2329. &mov (&DWP(12,$key),$s3);
  2330. &mov ($acc,$_inp); # load inp
  2331. &lea ($acc,&DWP(16,$acc)); # advance inp
  2332. &mov ($_inp,$acc); # save inp
  2333. &jnz (&label("slow_dec_loop_x86"));
  2334. &mov ("esp",$_esp);
  2335. &popf ();
  2336. &function_end_A();
  2337. &pushf (); # kludge, never executed
  2338. &set_label("slow_dec_partial_x86",16);
  2339. &lea ($acc,$ivec);
  2340. &mov (&DWP(0,$acc),$s0); # save output to temp
  2341. &mov (&DWP(4,$acc),$s1);
  2342. &mov (&DWP(8,$acc),$s2);
  2343. &mov (&DWP(12,$acc),$s3);
  2344. &mov ($acc,$_inp);
  2345. &mov ($s0,&DWP(0,$acc)); # re-read input
  2346. &mov ($s1,&DWP(4,$acc));
  2347. &mov ($s2,&DWP(8,$acc));
  2348. &mov ($s3,&DWP(12,$acc));
  2349. &mov (&DWP(0,$key),$s0); # copy it to iv
  2350. &mov (&DWP(4,$key),$s1);
  2351. &mov (&DWP(8,$key),$s2);
  2352. &mov (&DWP(12,$key),$s3);
  2353. &mov ("ecx",$_len);
  2354. &mov ("edi",$_out);
  2355. &lea ("esi",$ivec);
  2356. &align (4);
  2357. &data_word(0xA4F3F689); # rep movsb # copy partial output
  2358. &mov ("esp",$_esp);
  2359. &popf ();
  2360. &function_end("AES_cbc_encrypt");
  2361. }
  2362. #------------------------------------------------------------------#
  2363. sub enckey()
  2364. {
  2365. &movz ("esi",&LB("edx")); # rk[i]>>0
  2366. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2367. &movz ("esi",&HB("edx")); # rk[i]>>8
  2368. &shl ("ebx",24);
  2369. &xor ("eax","ebx");
  2370. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2371. &shr ("edx",16);
  2372. &movz ("esi",&LB("edx")); # rk[i]>>16
  2373. &xor ("eax","ebx");
  2374. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2375. &movz ("esi",&HB("edx")); # rk[i]>>24
  2376. &shl ("ebx",8);
  2377. &xor ("eax","ebx");
  2378. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2379. &shl ("ebx",16);
  2380. &xor ("eax","ebx");
  2381. &xor ("eax",&DWP(1024-128,$tbl,"ecx",4)); # rcon
  2382. }
  2383. &function_begin("_x86_AES_set_encrypt_key");
  2384. &mov ("esi",&wparam(1)); # user supplied key
  2385. &mov ("edi",&wparam(3)); # private key schedule
  2386. &test ("esi",-1);
  2387. &jz (&label("badpointer"));
  2388. &test ("edi",-1);
  2389. &jz (&label("badpointer"));
  2390. &call (&label("pic_point"));
  2391. &set_label("pic_point");
  2392. &blindpop($tbl);
  2393. &lea ($tbl,&DWP(&label("AES_Te")."-".&label("pic_point"),$tbl));
  2394. &lea ($tbl,&DWP(2048+128,$tbl));
  2395. # prefetch Te4
  2396. &mov ("eax",&DWP(0-128,$tbl));
  2397. &mov ("ebx",&DWP(32-128,$tbl));
  2398. &mov ("ecx",&DWP(64-128,$tbl));
  2399. &mov ("edx",&DWP(96-128,$tbl));
  2400. &mov ("eax",&DWP(128-128,$tbl));
  2401. &mov ("ebx",&DWP(160-128,$tbl));
  2402. &mov ("ecx",&DWP(192-128,$tbl));
  2403. &mov ("edx",&DWP(224-128,$tbl));
  2404. &mov ("ecx",&wparam(2)); # number of bits in key
  2405. &cmp ("ecx",128);
  2406. &je (&label("10rounds"));
  2407. &cmp ("ecx",192);
  2408. &je (&label("12rounds"));
  2409. &cmp ("ecx",256);
  2410. &je (&label("14rounds"));
  2411. &mov ("eax",-2); # invalid number of bits
  2412. &jmp (&label("exit"));
  2413. &set_label("10rounds");
  2414. &mov ("eax",&DWP(0,"esi")); # copy first 4 dwords
  2415. &mov ("ebx",&DWP(4,"esi"));
  2416. &mov ("ecx",&DWP(8,"esi"));
  2417. &mov ("edx",&DWP(12,"esi"));
  2418. &mov (&DWP(0,"edi"),"eax");
  2419. &mov (&DWP(4,"edi"),"ebx");
  2420. &mov (&DWP(8,"edi"),"ecx");
  2421. &mov (&DWP(12,"edi"),"edx");
  2422. &xor ("ecx","ecx");
  2423. &jmp (&label("10shortcut"));
  2424. &align (4);
  2425. &set_label("10loop");
  2426. &mov ("eax",&DWP(0,"edi")); # rk[0]
  2427. &mov ("edx",&DWP(12,"edi")); # rk[3]
  2428. &set_label("10shortcut");
  2429. &enckey ();
  2430. &mov (&DWP(16,"edi"),"eax"); # rk[4]
  2431. &xor ("eax",&DWP(4,"edi"));
  2432. &mov (&DWP(20,"edi"),"eax"); # rk[5]
  2433. &xor ("eax",&DWP(8,"edi"));
  2434. &mov (&DWP(24,"edi"),"eax"); # rk[6]
  2435. &xor ("eax",&DWP(12,"edi"));
  2436. &mov (&DWP(28,"edi"),"eax"); # rk[7]
  2437. &inc ("ecx");
  2438. &add ("edi",16);
  2439. &cmp ("ecx",10);
  2440. &jl (&label("10loop"));
  2441. &mov (&DWP(80,"edi"),10); # setup number of rounds
  2442. &xor ("eax","eax");
  2443. &jmp (&label("exit"));
  2444. &set_label("12rounds");
  2445. &mov ("eax",&DWP(0,"esi")); # copy first 6 dwords
  2446. &mov ("ebx",&DWP(4,"esi"));
  2447. &mov ("ecx",&DWP(8,"esi"));
  2448. &mov ("edx",&DWP(12,"esi"));
  2449. &mov (&DWP(0,"edi"),"eax");
  2450. &mov (&DWP(4,"edi"),"ebx");
  2451. &mov (&DWP(8,"edi"),"ecx");
  2452. &mov (&DWP(12,"edi"),"edx");
  2453. &mov ("ecx",&DWP(16,"esi"));
  2454. &mov ("edx",&DWP(20,"esi"));
  2455. &mov (&DWP(16,"edi"),"ecx");
  2456. &mov (&DWP(20,"edi"),"edx");
  2457. &xor ("ecx","ecx");
  2458. &jmp (&label("12shortcut"));
  2459. &align (4);
  2460. &set_label("12loop");
  2461. &mov ("eax",&DWP(0,"edi")); # rk[0]
  2462. &mov ("edx",&DWP(20,"edi")); # rk[5]
  2463. &set_label("12shortcut");
  2464. &enckey ();
  2465. &mov (&DWP(24,"edi"),"eax"); # rk[6]
  2466. &xor ("eax",&DWP(4,"edi"));
  2467. &mov (&DWP(28,"edi"),"eax"); # rk[7]
  2468. &xor ("eax",&DWP(8,"edi"));
  2469. &mov (&DWP(32,"edi"),"eax"); # rk[8]
  2470. &xor ("eax",&DWP(12,"edi"));
  2471. &mov (&DWP(36,"edi"),"eax"); # rk[9]
  2472. &cmp ("ecx",7);
  2473. &je (&label("12break"));
  2474. &inc ("ecx");
  2475. &xor ("eax",&DWP(16,"edi"));
  2476. &mov (&DWP(40,"edi"),"eax"); # rk[10]
  2477. &xor ("eax",&DWP(20,"edi"));
  2478. &mov (&DWP(44,"edi"),"eax"); # rk[11]
  2479. &add ("edi",24);
  2480. &jmp (&label("12loop"));
  2481. &set_label("12break");
  2482. &mov (&DWP(72,"edi"),12); # setup number of rounds
  2483. &xor ("eax","eax");
  2484. &jmp (&label("exit"));
  2485. &set_label("14rounds");
  2486. &mov ("eax",&DWP(0,"esi")); # copy first 8 dwords
  2487. &mov ("ebx",&DWP(4,"esi"));
  2488. &mov ("ecx",&DWP(8,"esi"));
  2489. &mov ("edx",&DWP(12,"esi"));
  2490. &mov (&DWP(0,"edi"),"eax");
  2491. &mov (&DWP(4,"edi"),"ebx");
  2492. &mov (&DWP(8,"edi"),"ecx");
  2493. &mov (&DWP(12,"edi"),"edx");
  2494. &mov ("eax",&DWP(16,"esi"));
  2495. &mov ("ebx",&DWP(20,"esi"));
  2496. &mov ("ecx",&DWP(24,"esi"));
  2497. &mov ("edx",&DWP(28,"esi"));
  2498. &mov (&DWP(16,"edi"),"eax");
  2499. &mov (&DWP(20,"edi"),"ebx");
  2500. &mov (&DWP(24,"edi"),"ecx");
  2501. &mov (&DWP(28,"edi"),"edx");
  2502. &xor ("ecx","ecx");
  2503. &jmp (&label("14shortcut"));
  2504. &align (4);
  2505. &set_label("14loop");
  2506. &mov ("edx",&DWP(28,"edi")); # rk[7]
  2507. &set_label("14shortcut");
  2508. &mov ("eax",&DWP(0,"edi")); # rk[0]
  2509. &enckey ();
  2510. &mov (&DWP(32,"edi"),"eax"); # rk[8]
  2511. &xor ("eax",&DWP(4,"edi"));
  2512. &mov (&DWP(36,"edi"),"eax"); # rk[9]
  2513. &xor ("eax",&DWP(8,"edi"));
  2514. &mov (&DWP(40,"edi"),"eax"); # rk[10]
  2515. &xor ("eax",&DWP(12,"edi"));
  2516. &mov (&DWP(44,"edi"),"eax"); # rk[11]
  2517. &cmp ("ecx",6);
  2518. &je (&label("14break"));
  2519. &inc ("ecx");
  2520. &mov ("edx","eax");
  2521. &mov ("eax",&DWP(16,"edi")); # rk[4]
  2522. &movz ("esi",&LB("edx")); # rk[11]>>0
  2523. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2524. &movz ("esi",&HB("edx")); # rk[11]>>8
  2525. &xor ("eax","ebx");
  2526. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2527. &shr ("edx",16);
  2528. &shl ("ebx",8);
  2529. &movz ("esi",&LB("edx")); # rk[11]>>16
  2530. &xor ("eax","ebx");
  2531. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2532. &movz ("esi",&HB("edx")); # rk[11]>>24
  2533. &shl ("ebx",16);
  2534. &xor ("eax","ebx");
  2535. &movz ("ebx",&BP(-128,$tbl,"esi",1));
  2536. &shl ("ebx",24);
  2537. &xor ("eax","ebx");
  2538. &mov (&DWP(48,"edi"),"eax"); # rk[12]
  2539. &xor ("eax",&DWP(20,"edi"));
  2540. &mov (&DWP(52,"edi"),"eax"); # rk[13]
  2541. &xor ("eax",&DWP(24,"edi"));
  2542. &mov (&DWP(56,"edi"),"eax"); # rk[14]
  2543. &xor ("eax",&DWP(28,"edi"));
  2544. &mov (&DWP(60,"edi"),"eax"); # rk[15]
  2545. &add ("edi",32);
  2546. &jmp (&label("14loop"));
  2547. &set_label("14break");
  2548. &mov (&DWP(48,"edi"),14); # setup number of rounds
  2549. &xor ("eax","eax");
  2550. &jmp (&label("exit"));
  2551. &set_label("badpointer");
  2552. &mov ("eax",-1);
  2553. &set_label("exit");
  2554. &function_end("_x86_AES_set_encrypt_key");
  2555. # int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
  2556. # AES_KEY *key)
  2557. &function_begin_B("AES_set_encrypt_key");
  2558. &call ("_x86_AES_set_encrypt_key");
  2559. &ret ();
  2560. &function_end_B("AES_set_encrypt_key");
  2561. sub deckey()
  2562. { my ($i,$key,$tp1,$tp2,$tp4,$tp8) = @_;
  2563. my $tmp = $tbl;
  2564. &mov ($tmp,0x80808080);
  2565. &and ($tmp,$tp1);
  2566. &lea ($tp2,&DWP(0,$tp1,$tp1));
  2567. &mov ($acc,$tmp);
  2568. &shr ($tmp,7);
  2569. &sub ($acc,$tmp);
  2570. &and ($tp2,0xfefefefe);
  2571. &and ($acc,0x1b1b1b1b);
  2572. &xor ($tp2,$acc);
  2573. &mov ($tmp,0x80808080);
  2574. &and ($tmp,$tp2);
  2575. &lea ($tp4,&DWP(0,$tp2,$tp2));
  2576. &mov ($acc,$tmp);
  2577. &shr ($tmp,7);
  2578. &sub ($acc,$tmp);
  2579. &and ($tp4,0xfefefefe);
  2580. &and ($acc,0x1b1b1b1b);
  2581. &xor ($tp2,$tp1); # tp2^tp1
  2582. &xor ($tp4,$acc);
  2583. &mov ($tmp,0x80808080);
  2584. &and ($tmp,$tp4);
  2585. &lea ($tp8,&DWP(0,$tp4,$tp4));
  2586. &mov ($acc,$tmp);
  2587. &shr ($tmp,7);
  2588. &xor ($tp4,$tp1); # tp4^tp1
  2589. &sub ($acc,$tmp);
  2590. &and ($tp8,0xfefefefe);
  2591. &and ($acc,0x1b1b1b1b);
  2592. &rotl ($tp1,8); # = ROTATE(tp1,8)
  2593. &xor ($tp8,$acc);
  2594. &mov ($tmp,&DWP(4*($i+1),$key)); # modulo-scheduled load
  2595. &xor ($tp1,$tp2);
  2596. &xor ($tp2,$tp8);
  2597. &xor ($tp1,$tp4);
  2598. &rotl ($tp2,24);
  2599. &xor ($tp4,$tp8);
  2600. &xor ($tp1,$tp8); # ^= tp8^(tp4^tp1)^(tp2^tp1)
  2601. &rotl ($tp4,16);
  2602. &xor ($tp1,$tp2); # ^= ROTATE(tp8^tp2^tp1,24)
  2603. &rotl ($tp8,8);
  2604. &xor ($tp1,$tp4); # ^= ROTATE(tp8^tp4^tp1,16)
  2605. &mov ($tp2,$tmp);
  2606. &xor ($tp1,$tp8); # ^= ROTATE(tp8,8)
  2607. &mov (&DWP(4*$i,$key),$tp1);
  2608. }
  2609. # int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
  2610. # AES_KEY *key)
  2611. &function_begin_B("AES_set_decrypt_key");
  2612. &call ("_x86_AES_set_encrypt_key");
  2613. &cmp ("eax",0);
  2614. &je (&label("proceed"));
  2615. &ret ();
  2616. &set_label("proceed");
  2617. &push ("ebp");
  2618. &push ("ebx");
  2619. &push ("esi");
  2620. &push ("edi");
  2621. &mov ("esi",&wparam(2));
  2622. &mov ("ecx",&DWP(240,"esi")); # pull number of rounds
  2623. &lea ("ecx",&DWP(0,"","ecx",4));
  2624. &lea ("edi",&DWP(0,"esi","ecx",4)); # pointer to last chunk
  2625. &set_label("invert",4); # invert order of chunks
  2626. &mov ("eax",&DWP(0,"esi"));
  2627. &mov ("ebx",&DWP(4,"esi"));
  2628. &mov ("ecx",&DWP(0,"edi"));
  2629. &mov ("edx",&DWP(4,"edi"));
  2630. &mov (&DWP(0,"edi"),"eax");
  2631. &mov (&DWP(4,"edi"),"ebx");
  2632. &mov (&DWP(0,"esi"),"ecx");
  2633. &mov (&DWP(4,"esi"),"edx");
  2634. &mov ("eax",&DWP(8,"esi"));
  2635. &mov ("ebx",&DWP(12,"esi"));
  2636. &mov ("ecx",&DWP(8,"edi"));
  2637. &mov ("edx",&DWP(12,"edi"));
  2638. &mov (&DWP(8,"edi"),"eax");
  2639. &mov (&DWP(12,"edi"),"ebx");
  2640. &mov (&DWP(8,"esi"),"ecx");
  2641. &mov (&DWP(12,"esi"),"edx");
  2642. &add ("esi",16);
  2643. &sub ("edi",16);
  2644. &cmp ("esi","edi");
  2645. &jne (&label("invert"));
  2646. &mov ($key,&wparam(2));
  2647. &mov ($acc,&DWP(240,$key)); # pull number of rounds
  2648. &lea ($acc,&DWP(-2,$acc,$acc));
  2649. &lea ($acc,&DWP(0,$key,$acc,8));
  2650. &mov (&wparam(2),$acc);
  2651. &mov ($s0,&DWP(16,$key)); # modulo-scheduled load
  2652. &set_label("permute",4); # permute the key schedule
  2653. &add ($key,16);
  2654. &deckey (0,$key,$s0,$s1,$s2,$s3);
  2655. &deckey (1,$key,$s1,$s2,$s3,$s0);
  2656. &deckey (2,$key,$s2,$s3,$s0,$s1);
  2657. &deckey (3,$key,$s3,$s0,$s1,$s2);
  2658. &cmp ($key,&wparam(2));
  2659. &jb (&label("permute"));
  2660. &xor ("eax","eax"); # return success
  2661. &function_end("AES_set_decrypt_key");
  2662. &asciz("AES for x86, CRYPTOGAMS by <appro\@openssl.org>");
  2663. &asm_finish();
  2664. close STDOUT or die "error closing STDOUT: $!";