12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609 |
- use cpu::cpu::{
- tlb_data, FLAG_CARRY, FLAG_OVERFLOW, FLAG_SIGN, FLAG_ZERO, OPSIZE_8, OPSIZE_16, OPSIZE_32,
- TLB_GLOBAL, TLB_HAS_CODE, TLB_NO_USER, TLB_READONLY, TLB_VALID,
- };
- use cpu::global_pointers;
- use cpu::memory;
- use jit::{Instruction, InstructionOperand, InstructionOperandDest, JitContext};
- use modrm;
- use modrm::ModrmByte;
- use profiler;
- use regs;
- use wasmgen::wasm_builder::{WasmBuilder, WasmLocal, WasmLocalI64};
- pub fn gen_add_cs_offset(ctx: &mut JitContext) {
- if !ctx.cpu.has_flat_segmentation() {
- ctx.builder
- .load_fixed_i32(global_pointers::get_seg_offset(regs::CS));
- ctx.builder.add_i32();
- }
- }
- pub fn gen_get_eip(builder: &mut WasmBuilder) {
- builder.load_fixed_i32(global_pointers::instruction_pointer as u32);
- }
- pub fn gen_set_eip_to_after_current_instruction(ctx: &mut JitContext) {
- ctx.builder
- .const_i32(global_pointers::instruction_pointer as i32);
- gen_get_eip(ctx.builder);
- ctx.builder.const_i32(!0xFFF);
- ctx.builder.and_i32();
- ctx.builder.const_i32(ctx.cpu.eip as i32 & 0xFFF);
- ctx.builder.or_i32();
- ctx.builder.store_aligned_i32(0);
- }
- pub fn gen_set_previous_eip_offset_from_eip_with_low_bits(
- builder: &mut WasmBuilder,
- low_bits: i32,
- ) {
- // previous_ip = instruction_pointer & ~0xFFF | low_bits;
- dbg_assert!(low_bits & !0xFFF == 0);
- builder.const_i32(global_pointers::previous_ip as i32);
- gen_get_eip(builder);
- builder.const_i32(!0xFFF);
- builder.and_i32();
- builder.const_i32(low_bits);
- builder.or_i32();
- builder.store_aligned_i32(0);
- }
- pub fn gen_set_eip_low_bits(builder: &mut WasmBuilder, low_bits: i32) {
- // instruction_pointer = instruction_pointer & ~0xFFF | low_bits;
- dbg_assert!(low_bits & !0xFFF == 0);
- builder.const_i32(global_pointers::instruction_pointer as i32);
- gen_get_eip(builder);
- builder.const_i32(!0xFFF);
- builder.and_i32();
- builder.const_i32(low_bits);
- builder.or_i32();
- builder.store_aligned_i32(0);
- }
- pub fn gen_set_eip_low_bits_and_jump_rel32(builder: &mut WasmBuilder, low_bits: i32, n: i32) {
- // instruction_pointer = (instruction_pointer & ~0xFFF | low_bits) + n;
- dbg_assert!(low_bits & !0xFFF == 0);
- builder.const_i32(global_pointers::instruction_pointer as i32);
- gen_get_eip(builder);
- builder.const_i32(!0xFFF);
- builder.and_i32();
- builder.const_i32(low_bits);
- builder.or_i32();
- if n != 0 {
- builder.const_i32(n);
- builder.add_i32();
- }
- builder.store_aligned_i32(0);
- }
- pub fn gen_relative_jump(builder: &mut WasmBuilder, n: i32) {
- // add n to instruction_pointer
- if n != 0 {
- builder.const_i32(global_pointers::instruction_pointer as i32);
- gen_get_eip(builder);
- builder.const_i32(n);
- builder.add_i32();
- builder.store_aligned_i32(0);
- }
- }
- pub fn gen_page_switch_check(
- ctx: &mut JitContext,
- next_block_addr: u32,
- last_instruction_addr: u32,
- ) {
- // After switching a page while in jitted code, check if the page mapping still holds
- gen_get_eip(ctx.builder);
- let address_local = ctx.builder.set_new_local();
- gen_get_phys_eip_plus_mem(ctx, &address_local);
- ctx.builder.free_local(address_local);
- ctx.builder
- .const_i32(next_block_addr as i32 + unsafe { memory::mem8 } as i32);
- ctx.builder.ne_i32();
- if cfg!(debug_assertions) {
- ctx.builder.if_void();
- gen_profiler_stat_increment(ctx.builder, profiler::stat::FAILED_PAGE_CHANGE);
- gen_debug_track_jit_exit(ctx.builder, last_instruction_addr);
- ctx.builder.br(ctx.exit_label);
- ctx.builder.block_end();
- }
- else {
- ctx.builder.br_if(ctx.exit_label);
- }
- }
- pub fn gen_update_instruction_counter(ctx: &mut JitContext) {
- ctx.builder
- .const_i32(global_pointers::instruction_counter as i32);
- ctx.builder
- .load_fixed_i32(global_pointers::instruction_counter as u32);
- ctx.builder.get_local(&ctx.instruction_counter);
- ctx.builder.add_i32();
- ctx.builder.store_aligned_i32(0);
- }
- pub fn gen_get_reg8(ctx: &mut JitContext, r: u32) {
- match r {
- regs::AL | regs::CL | regs::DL | regs::BL => {
- ctx.builder.get_local(&ctx.register_locals[r as usize]);
- ctx.builder.const_i32(0xFF);
- ctx.builder.and_i32();
- },
- regs::AH | regs::CH | regs::DH | regs::BH => {
- ctx.builder
- .get_local(&ctx.register_locals[(r - 4) as usize]);
- ctx.builder.const_i32(8);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(0xFF);
- ctx.builder.and_i32();
- },
- _ => assert!(false),
- }
- }
- /// Return a new local referencing one of the 8 bit registers or a direct reference to one of the
- /// register locals. Higher bits might be garbage (suitable for gen_cmp8 etc.). Must be freed with
- /// gen_free_reg8_or_alias.
- pub fn gen_get_reg8_or_alias_to_reg32(ctx: &mut JitContext, r: u32) -> WasmLocal {
- match r {
- regs::AL | regs::CL | regs::DL | regs::BL => ctx.register_locals[r as usize].unsafe_clone(),
- regs::AH | regs::CH | regs::DH | regs::BH => {
- ctx.builder
- .get_local(&ctx.register_locals[(r - 4) as usize]);
- ctx.builder.const_i32(8);
- ctx.builder.shr_u_i32();
- ctx.builder.set_new_local()
- },
- _ => panic!(),
- }
- }
- pub fn gen_free_reg8_or_alias(ctx: &mut JitContext, r: u32, local: WasmLocal) {
- match r {
- regs::AL | regs::CL | regs::DL | regs::BL => {},
- regs::AH | regs::CH | regs::DH | regs::BH => ctx.builder.free_local(local),
- _ => panic!(),
- }
- }
- pub fn gen_get_reg16(ctx: &mut JitContext, r: u32) {
- ctx.builder.get_local(&ctx.register_locals[r as usize]);
- ctx.builder.const_i32(0xFFFF);
- ctx.builder.and_i32();
- }
- pub fn gen_get_reg32(ctx: &mut JitContext, r: u32) {
- ctx.builder.get_local(&ctx.register_locals[r as usize]);
- }
- pub fn gen_set_reg8(ctx: &mut JitContext, r: u32) {
- match r {
- regs::AL | regs::CL | regs::DL | regs::BL => {
- // reg32[r] = stack_value & 0xFF | reg32[r] & ~0xFF
- ctx.builder.const_i32(0xFF);
- ctx.builder.and_i32();
- ctx.builder.get_local(&ctx.register_locals[r as usize]);
- ctx.builder.const_i32(!0xFF);
- ctx.builder.and_i32();
- ctx.builder.or_i32();
- ctx.builder.set_local(&ctx.register_locals[r as usize]);
- },
- regs::AH | regs::CH | regs::DH | regs::BH => {
- // reg32[r] = stack_value << 8 & 0xFF00 | reg32[r] & ~0xFF00
- ctx.builder.const_i32(8);
- ctx.builder.shl_i32();
- ctx.builder.const_i32(0xFF00);
- ctx.builder.and_i32();
- ctx.builder
- .get_local(&ctx.register_locals[(r - 4) as usize]);
- ctx.builder.const_i32(!0xFF00);
- ctx.builder.and_i32();
- ctx.builder.or_i32();
- ctx.builder
- .set_local(&ctx.register_locals[(r - 4) as usize]);
- },
- _ => assert!(false),
- }
- }
- pub fn gen_set_reg8_unmasked(ctx: &mut JitContext, r: u32) {
- if cfg!(debug_assertions) {
- let val = ctx.builder.set_new_local();
- ctx.builder.get_local(&val);
- ctx.builder.const_i32(!0xFF);
- ctx.builder.and_i32();
- ctx.builder.if_void();
- ctx.builder.unreachable();
- ctx.builder.block_end();
- ctx.builder.get_local(&val);
- ctx.builder.free_local(val);
- }
- match r {
- regs::AL | regs::CL | regs::DL | regs::BL => {
- // reg32[r] = stack_value | reg32[r] & ~0xFF
- ctx.builder.get_local(&ctx.register_locals[r as usize]);
- ctx.builder.const_i32(!0xFF);
- ctx.builder.and_i32();
- ctx.builder.or_i32();
- ctx.builder.set_local(&ctx.register_locals[r as usize]);
- },
- regs::AH | regs::CH | regs::DH | regs::BH => {
- // reg32[r] = stack_value << 8 | reg32[r] & ~0xFF00
- ctx.builder.const_i32(8);
- ctx.builder.shl_i32();
- ctx.builder.const_i32(0xFF00);
- ctx.builder.and_i32();
- ctx.builder
- .get_local(&ctx.register_locals[(r - 4) as usize]);
- ctx.builder.const_i32(!0xFF00);
- ctx.builder.and_i32();
- ctx.builder.or_i32();
- ctx.builder
- .set_local(&ctx.register_locals[(r - 4) as usize]);
- },
- _ => assert!(false),
- }
- }
- pub fn gen_set_reg16(ctx: &mut JitContext, r: u32) {
- gen_set_reg16_local(ctx.builder, &ctx.register_locals[r as usize]);
- }
- pub fn gen_set_reg16_unmasked(ctx: &mut JitContext, r: u32) {
- if cfg!(debug_assertions) {
- let val = ctx.builder.set_new_local();
- ctx.builder.get_local(&val);
- ctx.builder.const_i32(!0xFFFF);
- ctx.builder.and_i32();
- ctx.builder.if_void();
- ctx.builder.unreachable();
- ctx.builder.block_end();
- ctx.builder.get_local(&val);
- ctx.builder.free_local(val);
- }
- ctx.builder.get_local(&ctx.reg(r));
- ctx.builder.const_i32(!0xFFFF);
- ctx.builder.and_i32();
- ctx.builder.or_i32();
- ctx.builder.set_local(&ctx.reg(r));
- }
- pub fn gen_set_reg16_local(builder: &mut WasmBuilder, local: &WasmLocal) {
- // reg32[r] = v & 0xFFFF | reg32[r] & ~0xFFFF
- builder.const_i32(0xFFFF);
- builder.and_i32();
- builder.get_local(local);
- builder.const_i32(!0xFFFF);
- builder.and_i32();
- builder.or_i32();
- builder.set_local(local);
- }
- pub fn gen_set_reg32(ctx: &mut JitContext, r: u32) {
- ctx.builder.set_local(&ctx.register_locals[r as usize]);
- }
- pub fn decr_exc_asize(ctx: &mut JitContext) {
- gen_get_reg32(ctx, regs::ECX);
- ctx.builder.const_i32(1);
- ctx.builder.sub_i32();
- if ctx.cpu.asize_32() {
- gen_set_reg32(ctx, regs::ECX);
- }
- else {
- gen_set_reg16(ctx, regs::CX);
- }
- }
- pub fn gen_read_reg_xmm128_into_scratch(ctx: &mut JitContext, r: u32) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- let dest = global_pointers::get_reg_xmm_offset(r);
- ctx.builder.const_i32(dest as i32);
- ctx.builder.load_aligned_i64(0);
- ctx.builder.store_aligned_i64(0);
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32 + 8);
- let dest = global_pointers::get_reg_xmm_offset(r) + 8;
- ctx.builder.const_i32(dest as i32);
- ctx.builder.load_aligned_i64(0);
- ctx.builder.store_aligned_i64(0);
- }
- pub fn gen_get_sreg(ctx: &mut JitContext, r: u32) {
- ctx.builder
- .load_fixed_u16(global_pointers::get_sreg_offset(r))
- }
- pub fn gen_get_ss_offset(ctx: &mut JitContext) {
- ctx.builder
- .load_fixed_i32(global_pointers::get_seg_offset(regs::SS));
- }
- pub fn gen_get_flags(builder: &mut WasmBuilder) {
- builder.load_fixed_i32(global_pointers::flags as u32);
- }
- fn gen_get_flags_changed(builder: &mut WasmBuilder) {
- builder.load_fixed_i32(global_pointers::flags_changed as u32);
- }
- fn gen_get_last_result(builder: &mut WasmBuilder, previous_instruction: &Instruction) {
- match previous_instruction {
- Instruction::Add {
- dest: InstructionOperandDest::WasmLocal(l),
- opsize: OPSIZE_32,
- ..
- }
- | Instruction::AdcSbb {
- dest: InstructionOperandDest::WasmLocal(l),
- opsize: OPSIZE_32,
- ..
- }
- | Instruction::Sub {
- dest: InstructionOperandDest::WasmLocal(l),
- opsize: OPSIZE_32,
- ..
- }
- | Instruction::Bitwise {
- dest: InstructionOperandDest::WasmLocal(l),
- opsize: OPSIZE_32,
- }
- | Instruction::NonZeroShift {
- dest: InstructionOperandDest::WasmLocal(l),
- opsize: OPSIZE_32,
- } => builder.get_local(&l),
- Instruction::Cmp {
- dest: InstructionOperandDest::WasmLocal(l),
- source,
- opsize: OPSIZE_32,
- } => {
- if source.is_zero() {
- builder.get_local(&l)
- }
- else {
- builder.load_fixed_i32(global_pointers::last_result as u32)
- }
- },
- _ => builder.load_fixed_i32(global_pointers::last_result as u32),
- }
- }
- fn gen_get_last_op_size(builder: &mut WasmBuilder) {
- builder.load_fixed_i32(global_pointers::last_op_size as u32);
- }
- fn gen_get_last_op1(builder: &mut WasmBuilder, previous_instruction: &Instruction) {
- match previous_instruction {
- Instruction::Cmp {
- dest: InstructionOperandDest::WasmLocal(l),
- source: _,
- opsize: OPSIZE_32,
- } => builder.get_local(&l),
- _ => builder.load_fixed_i32(global_pointers::last_op1 as u32),
- }
- }
- pub fn gen_get_page_fault(builder: &mut WasmBuilder) {
- builder.load_fixed_u8(global_pointers::page_fault as u32);
- }
- /// sign-extend a byte value on the stack and leave it on the stack
- pub fn sign_extend_i8(builder: &mut WasmBuilder) {
- builder.const_i32(24);
- builder.shl_i32();
- builder.const_i32(24);
- builder.shr_s_i32();
- }
- /// sign-extend a two byte value on the stack and leave it on the stack
- pub fn sign_extend_i16(builder: &mut WasmBuilder) {
- builder.const_i32(16);
- builder.shl_i32();
- builder.const_i32(16);
- builder.shr_s_i32();
- }
- pub fn gen_fn0_const(builder: &mut WasmBuilder, name: &str) { builder.call_fn0(name) }
- pub fn gen_fn1_const(builder: &mut WasmBuilder, name: &str, arg0: u32) {
- builder.const_i32(arg0 as i32);
- builder.call_fn1(name);
- }
- pub fn gen_fn2_const(builder: &mut WasmBuilder, name: &str, arg0: u32, arg1: u32) {
- builder.const_i32(arg0 as i32);
- builder.const_i32(arg1 as i32);
- builder.call_fn2(name);
- }
- pub fn gen_fn3_const(builder: &mut WasmBuilder, name: &str, arg0: u32, arg1: u32, arg2: u32) {
- builder.const_i32(arg0 as i32);
- builder.const_i32(arg1 as i32);
- builder.const_i32(arg2 as i32);
- builder.call_fn3(name);
- }
- // helper functions for gen/generate_jit.js
- pub fn gen_modrm_fn0(builder: &mut WasmBuilder, name: &str) {
- // generates: fn( _ )
- builder.call_fn1(name);
- }
- pub fn gen_modrm_fn1(builder: &mut WasmBuilder, name: &str, arg0: u32) {
- // generates: fn( _, arg0 )
- builder.const_i32(arg0 as i32);
- builder.call_fn2(name);
- }
- pub fn gen_modrm_fn2(builder: &mut WasmBuilder, name: &str, arg0: u32, arg1: u32) {
- // generates: fn( _, arg0, arg1 )
- builder.const_i32(arg0 as i32);
- builder.const_i32(arg1 as i32);
- builder.call_fn3(name);
- }
- pub fn gen_modrm_resolve(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- modrm::gen(ctx, modrm_byte)
- }
- pub fn gen_modrm_resolve_with_local(
- ctx: &mut JitContext,
- modrm_byte: ModrmByte,
- gen: &dyn Fn(&mut JitContext, &WasmLocal),
- ) {
- if let Some(r) = modrm::get_as_reg_index_if_possible(ctx, &modrm_byte) {
- gen(ctx, &ctx.reg(r));
- }
- else {
- gen_modrm_resolve(ctx, modrm_byte);
- let address = ctx.builder.set_new_local();
- gen(ctx, &address);
- ctx.builder.free_local(address);
- }
- }
- pub fn gen_set_reg8_r(ctx: &mut JitContext, dest: u32, src: u32) {
- // generates: reg8[r_dest] = reg8[r_src]
- if src != dest {
- gen_get_reg8(ctx, src);
- gen_set_reg8_unmasked(ctx, dest);
- }
- }
- pub fn gen_set_reg16_r(ctx: &mut JitContext, dest: u32, src: u32) {
- // generates: reg16[r_dest] = reg16[r_src]
- if src != dest {
- gen_get_reg16(ctx, src);
- gen_set_reg16_unmasked(ctx, dest);
- }
- }
- pub fn gen_set_reg32_r(ctx: &mut JitContext, dest: u32, src: u32) {
- // generates: reg32[r_dest] = reg32[r_src]
- if src != dest {
- gen_get_reg32(ctx, src);
- gen_set_reg32(ctx, dest);
- }
- }
- pub fn gen_modrm_resolve_safe_read8(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read8(ctx, addr));
- }
- pub fn gen_modrm_resolve_safe_read16(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read16(ctx, addr));
- }
- pub fn gen_modrm_resolve_safe_read32(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read32(ctx, addr));
- }
- pub fn gen_modrm_resolve_safe_read64(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read64(ctx, addr));
- }
- pub fn gen_modrm_resolve_safe_read128(
- ctx: &mut JitContext,
- modrm_byte: ModrmByte,
- where_to_write: u32,
- ) {
- gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| {
- gen_safe_read128(ctx, addr, where_to_write)
- });
- }
- pub fn gen_safe_read8(ctx: &mut JitContext, address_local: &WasmLocal) {
- gen_safe_read(ctx, BitSize::BYTE, address_local, None);
- }
- pub fn gen_safe_read16(ctx: &mut JitContext, address_local: &WasmLocal) {
- gen_safe_read(ctx, BitSize::WORD, address_local, None);
- }
- pub fn gen_safe_read32(ctx: &mut JitContext, address_local: &WasmLocal) {
- gen_safe_read(ctx, BitSize::DWORD, address_local, None);
- }
- pub fn gen_safe_read64(ctx: &mut JitContext, address_local: &WasmLocal) {
- gen_safe_read(ctx, BitSize::QWORD, &address_local, None);
- }
- pub fn gen_safe_read128(ctx: &mut JitContext, address_local: &WasmLocal, where_to_write: u32) {
- gen_safe_read(ctx, BitSize::DQWORD, &address_local, Some(where_to_write));
- }
- // only used internally for gen_safe_write
- enum GenSafeWriteValue<'a> {
- I32(&'a WasmLocal),
- I64(&'a WasmLocalI64),
- TwoI64s(&'a WasmLocalI64, &'a WasmLocalI64),
- }
- enum GenSafeReadWriteValue {
- I32(WasmLocal),
- I64(WasmLocalI64),
- }
- #[derive(Copy, Clone, Eq, PartialEq)]
- pub enum BitSize {
- BYTE,
- WORD,
- DWORD,
- QWORD,
- DQWORD,
- }
- impl BitSize {
- pub fn bytes(&self) -> u32 {
- match self {
- BitSize::BYTE => 1,
- BitSize::WORD => 2,
- BitSize::DWORD => 4,
- BitSize::QWORD => 8,
- BitSize::DQWORD => 16,
- }
- }
- }
- pub fn gen_safe_write8(ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocal) {
- gen_safe_write(
- ctx,
- BitSize::BYTE,
- address_local,
- GenSafeWriteValue::I32(value_local),
- )
- }
- pub fn gen_safe_write16(ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocal) {
- gen_safe_write(
- ctx,
- BitSize::WORD,
- address_local,
- GenSafeWriteValue::I32(value_local),
- )
- }
- pub fn gen_safe_write32(ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocal) {
- gen_safe_write(
- ctx,
- BitSize::DWORD,
- address_local,
- GenSafeWriteValue::I32(value_local),
- )
- }
- pub fn gen_safe_write64(
- ctx: &mut JitContext,
- address_local: &WasmLocal,
- value_local: &WasmLocalI64,
- ) {
- gen_safe_write(
- ctx,
- BitSize::QWORD,
- address_local,
- GenSafeWriteValue::I64(value_local),
- )
- }
- pub fn gen_safe_write128(
- ctx: &mut JitContext,
- address_local: &WasmLocal,
- value_local_low: &WasmLocalI64,
- value_local_high: &WasmLocalI64,
- ) {
- gen_safe_write(
- ctx,
- BitSize::DQWORD,
- address_local,
- GenSafeWriteValue::TwoI64s(value_local_low, value_local_high),
- )
- }
- fn gen_safe_read(
- ctx: &mut JitContext,
- bits: BitSize,
- address_local: &WasmLocal,
- where_to_write: Option<u32>,
- ) {
- // Execute a virtual memory read. All slow paths (memory-mapped IO, tlb miss, page fault and
- // read across page boundary are handled in safe_read_jit_slow
- // entry <- tlb_data[addr >> 12 << 2]
- // if entry & MASK == TLB_VALID && (addr & 0xFFF) <= 0x1000 - bytes: goto fast
- // entry <- safe_read_jit_slow(addr, instruction_pointer)
- // if page_fault: goto exit-with-pagefault
- // fast: mem[(entry & ~0xFFF) ^ addr]
- let cont = ctx.builder.block_void();
- ctx.builder.get_local(&address_local);
- ctx.builder.const_i32(12);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(2);
- ctx.builder.shl_i32();
- ctx.builder
- .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 });
- let entry_local = ctx.builder.tee_new_local();
- ctx.builder.const_i32(
- (0xFFF
- & !TLB_READONLY
- & !TLB_GLOBAL
- & !TLB_HAS_CODE
- & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32,
- );
- ctx.builder.and_i32();
- ctx.builder.const_i32(TLB_VALID as i32);
- ctx.builder.eq_i32();
- if bits != BitSize::BYTE {
- ctx.builder.get_local(&address_local);
- ctx.builder.const_i32(0xFFF);
- ctx.builder.and_i32();
- ctx.builder.const_i32(0x1000 - bits.bytes() as i32);
- ctx.builder.le_i32();
- ctx.builder.and_i32();
- }
- ctx.builder.br_if(cont);
- if cfg!(feature = "profiler") {
- ctx.builder.get_local(&address_local);
- ctx.builder.get_local(&entry_local);
- ctx.builder.call_fn2("report_safe_read_jit_slow");
- }
- ctx.builder.get_local(&address_local);
- ctx.builder
- .const_i32(ctx.start_of_current_instruction as i32 & 0xFFF);
- match bits {
- BitSize::BYTE => {
- ctx.builder.call_fn2_ret("safe_read8_slow_jit");
- },
- BitSize::WORD => {
- ctx.builder.call_fn2_ret("safe_read16_slow_jit");
- },
- BitSize::DWORD => {
- ctx.builder.call_fn2_ret("safe_read32s_slow_jit");
- },
- BitSize::QWORD => {
- ctx.builder.call_fn2_ret("safe_read64s_slow_jit");
- },
- BitSize::DQWORD => {
- ctx.builder.call_fn2_ret("safe_read128s_slow_jit");
- },
- }
- ctx.builder.tee_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- if cfg!(feature = "profiler") {
- ctx.builder.if_void();
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- ctx.builder.block_end();
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- }
- ctx.builder.br_if(ctx.exit_with_fault_label);
- ctx.builder.block_end();
- gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_READ_FAST); // XXX: Both fast and slow
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(!0xFFF);
- ctx.builder.and_i32();
- ctx.builder.get_local(&address_local);
- ctx.builder.xor_i32();
- // where_to_write is only used by dqword
- dbg_assert!((where_to_write != None) == (bits == BitSize::DQWORD));
- match bits {
- BitSize::BYTE => {
- ctx.builder.load_u8(0);
- },
- BitSize::WORD => {
- ctx.builder.load_unaligned_u16(0);
- },
- BitSize::DWORD => {
- ctx.builder.load_unaligned_i32(0);
- },
- BitSize::QWORD => {
- ctx.builder.load_unaligned_i64(0);
- },
- BitSize::DQWORD => {
- let where_to_write = where_to_write.unwrap();
- let virt_address_local = ctx.builder.set_new_local();
- ctx.builder.const_i32(0);
- ctx.builder.get_local(&virt_address_local);
- ctx.builder.load_unaligned_i64(0);
- ctx.builder.store_unaligned_i64(where_to_write);
- ctx.builder.const_i32(0);
- ctx.builder.get_local(&virt_address_local);
- ctx.builder.load_unaligned_i64(8);
- ctx.builder.store_unaligned_i64(where_to_write + 8);
- ctx.builder.free_local(virt_address_local);
- },
- }
- ctx.builder.free_local(entry_local);
- }
- pub fn gen_get_phys_eip_plus_mem(ctx: &mut JitContext, address_local: &WasmLocal) {
- // Similar to gen_safe_read, but return the physical eip + memory::mem rather than reading from memory
- // In functions that need to use this value we need to fix it by substracting memory::mem
- // this is done in order to remove one instruction from the fast path of memory accesses (no need to add
- // memory::mem anymore ).
- // We need to account for this in gen_page_switch_check and we compare with next_block_addr + memory::mem8
- // We cannot the same while processing an AbsoluteEip flow control change so there we need to fix the value
- // by subscracting memory::mem. Overall, since AbsoluteEip is encountered less often than memory accesses so
- // this ends up improving perf.
- // Does not (need to) handle mapped memory
- // XXX: Currently does not use ctx.start_of_current_instruction, but rather assumes that eip is
- // already correct (pointing at the current instruction)
- let cont = ctx.builder.block_void();
- ctx.builder.get_local(&address_local);
- ctx.builder.const_i32(12);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(2);
- ctx.builder.shl_i32();
- ctx.builder
- .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 });
- let entry_local = ctx.builder.tee_new_local();
- ctx.builder.const_i32(
- (0xFFF
- & !TLB_READONLY
- & !TLB_GLOBAL
- & !TLB_HAS_CODE
- & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32,
- );
- ctx.builder.and_i32();
- ctx.builder.const_i32(TLB_VALID as i32);
- ctx.builder.eq_i32();
- ctx.builder.br_if(cont);
- if cfg!(feature = "profiler") {
- ctx.builder.get_local(&address_local);
- ctx.builder.get_local(&entry_local);
- ctx.builder.call_fn2("report_safe_read_jit_slow");
- }
- ctx.builder.get_local(&address_local);
- ctx.builder.call_fn1_ret("get_phys_eip_slow_jit");
- ctx.builder.tee_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- if cfg!(feature = "profiler") {
- ctx.builder.if_void();
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); // XXX
- ctx.builder.block_end();
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- }
- ctx.builder.br_if(ctx.exit_with_fault_label);
- ctx.builder.block_end();
- gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_READ_FAST); // XXX: Both fast and slow
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(!0xFFF);
- ctx.builder.and_i32();
- ctx.builder.get_local(&address_local);
- ctx.builder.xor_i32();
- ctx.builder.free_local(entry_local);
- }
- fn gen_safe_write(
- ctx: &mut JitContext,
- bits: BitSize,
- address_local: &WasmLocal,
- value_local: GenSafeWriteValue,
- ) {
- // Execute a virtual memory write. All slow paths (memory-mapped IO, tlb miss, page fault,
- // write across page boundary and page containing jitted code are handled in safe_write_jit_slow
- // entry <- tlb_data[addr >> 12 << 2]
- // if entry & MASK == TLB_VALID && (addr & 0xFFF) <= 0x1000 - bytes: goto fast
- // entry <- safe_write_jit_slow(addr, value, instruction_pointer)
- // if page_fault: goto exit-with-pagefault
- // fast: mem[(entry & ~0xFFF) ^ addr] <- value
- let cont = ctx.builder.block_void();
- ctx.builder.get_local(&address_local);
- ctx.builder.const_i32(12);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(2);
- ctx.builder.shl_i32();
- ctx.builder
- .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 });
- let entry_local = ctx.builder.tee_new_local();
- ctx.builder
- .const_i32((0xFFF & !TLB_GLOBAL & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32);
- ctx.builder.and_i32();
- ctx.builder.const_i32(TLB_VALID as i32);
- ctx.builder.eq_i32();
- if bits != BitSize::BYTE {
- ctx.builder.get_local(&address_local);
- ctx.builder.const_i32(0xFFF);
- ctx.builder.and_i32();
- ctx.builder.const_i32(0x1000 - bits.bytes() as i32);
- ctx.builder.le_i32();
- ctx.builder.and_i32();
- }
- ctx.builder.br_if(cont);
- if cfg!(feature = "profiler") {
- ctx.builder.get_local(&address_local);
- ctx.builder.get_local(&entry_local);
- ctx.builder.call_fn2("report_safe_write_jit_slow");
- }
- ctx.builder.get_local(&address_local);
- match value_local {
- GenSafeWriteValue::I32(local) => ctx.builder.get_local(local),
- GenSafeWriteValue::I64(local) => ctx.builder.get_local_i64(local),
- GenSafeWriteValue::TwoI64s(local1, local2) => {
- ctx.builder.get_local_i64(local1);
- ctx.builder.get_local_i64(local2)
- },
- }
- ctx.builder
- .const_i32(ctx.start_of_current_instruction as i32 & 0xFFF);
- match bits {
- BitSize::BYTE => {
- ctx.builder.call_fn3_ret("safe_write8_slow_jit");
- },
- BitSize::WORD => {
- ctx.builder.call_fn3_ret("safe_write16_slow_jit");
- },
- BitSize::DWORD => {
- ctx.builder.call_fn3_ret("safe_write32_slow_jit");
- },
- BitSize::QWORD => {
- ctx.builder
- .call_fn3_i32_i64_i32_ret("safe_write64_slow_jit");
- },
- BitSize::DQWORD => {
- ctx.builder
- .call_fn4_i32_i64_i64_i32_ret("safe_write128_slow_jit");
- },
- }
- ctx.builder.tee_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- if cfg!(feature = "profiler") {
- ctx.builder.if_void();
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- ctx.builder.block_end();
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- }
- ctx.builder.br_if(ctx.exit_with_fault_label);
- ctx.builder.block_end();
- gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_WRITE_FAST); // XXX: Both fast and slow
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(!0xFFF);
- ctx.builder.and_i32();
- ctx.builder.get_local(&address_local);
- ctx.builder.xor_i32();
- match value_local {
- GenSafeWriteValue::I32(local) => ctx.builder.get_local(local),
- GenSafeWriteValue::I64(local) => ctx.builder.get_local_i64(local),
- GenSafeWriteValue::TwoI64s(local1, local2) => {
- assert!(bits == BitSize::DQWORD);
- let virt_address_local = ctx.builder.tee_new_local();
- ctx.builder.get_local_i64(local1);
- ctx.builder.store_unaligned_i64(0);
- ctx.builder.get_local(&virt_address_local);
- ctx.builder.get_local_i64(local2);
- ctx.builder.store_unaligned_i64(8);
- ctx.builder.free_local(virt_address_local);
- },
- }
- match bits {
- BitSize::BYTE => {
- ctx.builder.store_u8(0);
- },
- BitSize::WORD => {
- ctx.builder.store_unaligned_u16(0);
- },
- BitSize::DWORD => {
- ctx.builder.store_unaligned_i32(0);
- },
- BitSize::QWORD => {
- ctx.builder.store_unaligned_i64(0);
- },
- BitSize::DQWORD => {}, // handled above
- }
- ctx.builder.free_local(entry_local);
- }
- pub fn gen_safe_read_write(
- ctx: &mut JitContext,
- bits: BitSize,
- address_local: &WasmLocal,
- f: &dyn Fn(&mut JitContext),
- ) {
- // Execute a virtual memory read+write. All slow paths (memory-mapped IO, tlb miss, page fault,
- // write across page boundary and page containing jitted code are handled in
- // safe_read_write_jit_slow
- // entry <- tlb_data[addr >> 12 << 2]
- // can_use_fast_path <- entry & MASK == TLB_VALID && (addr & 0xFFF) <= 0x1000 - bytes
- // if can_use_fast_path: goto fast
- // entry <- safe_read_write_jit_slow(addr, instruction_pointer)
- // if page_fault: goto exit-with-pagefault
- // fast: value <- f(mem[(entry & ~0xFFF) ^ addr])
- // if !can_use_fast_path { safe_write_jit_slow(addr, value, instruction_pointer) }
- // mem[(entry & ~0xFFF) ^ addr] <- value
- let cont = ctx.builder.block_void();
- ctx.builder.get_local(address_local);
- ctx.builder.const_i32(12);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(2);
- ctx.builder.shl_i32();
- ctx.builder
- .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 });
- let entry_local = ctx.builder.tee_new_local();
- ctx.builder
- .const_i32((0xFFF & !TLB_GLOBAL & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32);
- ctx.builder.and_i32();
- ctx.builder.const_i32(TLB_VALID as i32);
- ctx.builder.eq_i32();
- if bits != BitSize::BYTE {
- ctx.builder.get_local(&address_local);
- ctx.builder.const_i32(0xFFF);
- ctx.builder.and_i32();
- ctx.builder.const_i32(0x1000 - bits.bytes() as i32);
- ctx.builder.le_i32();
- ctx.builder.and_i32();
- }
- let can_use_fast_path_local = ctx.builder.tee_new_local();
- ctx.builder.br_if(cont);
- if cfg!(feature = "profiler") {
- ctx.builder.get_local(&address_local);
- ctx.builder.get_local(&entry_local);
- ctx.builder.call_fn2("report_safe_read_write_jit_slow");
- }
- ctx.builder.get_local(&address_local);
- ctx.builder
- .const_i32(ctx.start_of_current_instruction as i32 & 0xFFF);
- match bits {
- BitSize::BYTE => {
- ctx.builder.call_fn2_ret("safe_read_write8_slow_jit");
- },
- BitSize::WORD => {
- ctx.builder.call_fn2_ret("safe_read_write16_slow_jit");
- },
- BitSize::DWORD => {
- ctx.builder.call_fn2_ret("safe_read_write32s_slow_jit");
- },
- BitSize::QWORD => {
- ctx.builder.call_fn2_ret("safe_read_write64_slow_jit");
- },
- BitSize::DQWORD => {
- dbg_assert!(false);
- },
- }
- ctx.builder.tee_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- if cfg!(feature = "profiler") {
- ctx.builder.if_void();
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- ctx.builder.block_end();
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- }
- ctx.builder.br_if(ctx.exit_with_fault_label);
- ctx.builder.block_end();
- gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_READ_WRITE_FAST); // XXX: Also slow
- ctx.builder.get_local(&entry_local);
- ctx.builder.const_i32(!0xFFF);
- ctx.builder.and_i32();
- ctx.builder.get_local(&address_local);
- ctx.builder.xor_i32();
- ctx.builder.free_local(entry_local);
- let phys_addr_local = ctx.builder.tee_new_local();
- match bits {
- BitSize::BYTE => {
- ctx.builder.load_u8(0);
- },
- BitSize::WORD => {
- ctx.builder.load_unaligned_u16(0);
- },
- BitSize::DWORD => {
- ctx.builder.load_unaligned_i32(0);
- },
- BitSize::QWORD => {
- ctx.builder.load_unaligned_i64(0);
- },
- BitSize::DQWORD => assert!(false), // not used
- }
- // value is now on stack
- f(ctx);
- // TODO: Could get rid of this local by returning one from f
- let value_local = if bits == BitSize::QWORD {
- GenSafeReadWriteValue::I64(ctx.builder.set_new_local_i64())
- }
- else {
- GenSafeReadWriteValue::I32(ctx.builder.set_new_local())
- };
- ctx.builder.get_local(&can_use_fast_path_local);
- ctx.builder.eqz_i32();
- ctx.builder.if_void();
- {
- ctx.builder.get_local(&address_local);
- match &value_local {
- GenSafeReadWriteValue::I32(l) => ctx.builder.get_local(l),
- GenSafeReadWriteValue::I64(l) => ctx.builder.get_local_i64(l),
- }
- ctx.builder
- .const_i32(ctx.start_of_current_instruction as i32);
- match bits {
- BitSize::BYTE => {
- ctx.builder.call_fn3_ret("safe_write8_slow_jit");
- },
- BitSize::WORD => {
- ctx.builder.call_fn3_ret("safe_write16_slow_jit");
- },
- BitSize::DWORD => {
- ctx.builder.call_fn3_ret("safe_write32_slow_jit");
- },
- BitSize::QWORD => {
- ctx.builder
- .call_fn3_i32_i64_i32_ret("safe_write64_slow_jit");
- },
- BitSize::DQWORD => {
- dbg_assert!(false);
- },
- }
- if cfg!(debug_assertions) {
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- ctx.builder.if_void();
- {
- // handled above
- ctx.builder.const_i32(match bits {
- BitSize::BYTE => 8,
- BitSize::WORD => 16,
- BitSize::DWORD => 32,
- BitSize::QWORD => 64,
- _ => {
- dbg_assert!(false);
- 0
- },
- });
- ctx.builder.get_local(&address_local);
- ctx.builder.call_fn2("bug_gen_safe_read_write_page_fault");
- }
- ctx.builder.block_end();
- }
- else {
- ctx.builder.drop_();
- }
- }
- ctx.builder.block_end();
- ctx.builder.get_local(&phys_addr_local);
- match &value_local {
- GenSafeReadWriteValue::I32(l) => ctx.builder.get_local(l),
- GenSafeReadWriteValue::I64(l) => ctx.builder.get_local_i64(l),
- }
- match bits {
- BitSize::BYTE => {
- ctx.builder.store_u8(0);
- },
- BitSize::WORD => {
- ctx.builder.store_unaligned_u16(0);
- },
- BitSize::DWORD => {
- ctx.builder.store_unaligned_i32(0);
- },
- BitSize::QWORD => {
- ctx.builder.store_unaligned_i64(0);
- },
- BitSize::DQWORD => {
- dbg_assert!(false);
- },
- }
- match value_local {
- GenSafeReadWriteValue::I32(l) => ctx.builder.free_local(l),
- GenSafeReadWriteValue::I64(l) => ctx.builder.free_local_i64(l),
- }
- ctx.builder.free_local(can_use_fast_path_local);
- ctx.builder.free_local(phys_addr_local);
- }
- #[cfg(debug_assertions)]
- #[no_mangle]
- pub fn bug_gen_safe_read_write_page_fault(bits: i32, addr: u32) {
- dbg_log!("bug: gen_safe_read_write_page_fault {} {:x}", bits, addr);
- dbg_assert!(false);
- }
- pub fn gen_jmp_rel16(builder: &mut WasmBuilder, rel16: u16) {
- let cs_offset_addr = global_pointers::get_seg_offset(regs::CS);
- builder.load_fixed_i32(cs_offset_addr);
- let local = builder.set_new_local();
- // generate:
- // *instruction_pointer = cs_offset + ((*instruction_pointer - cs_offset + rel16) & 0xFFFF);
- {
- builder.const_i32(global_pointers::instruction_pointer as i32);
- gen_get_eip(builder);
- builder.get_local(&local);
- builder.sub_i32();
- builder.const_i32(rel16 as i32);
- builder.add_i32();
- builder.const_i32(0xFFFF);
- builder.and_i32();
- builder.get_local(&local);
- builder.add_i32();
- builder.store_aligned_i32(0);
- }
- builder.free_local(local);
- }
- pub fn gen_pop16_ss16(ctx: &mut JitContext) {
- // sp = segment_offsets[SS] + reg16[SP] (or just reg16[SP] if has_flat_segmentation)
- gen_get_reg16(ctx, regs::SP);
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- }
- // result = safe_read16(sp)
- let address_local = ctx.builder.set_new_local();
- gen_safe_read16(ctx, &address_local);
- ctx.builder.free_local(address_local);
- // reg16[SP] += 2;
- gen_get_reg16(ctx, regs::SP);
- ctx.builder.const_i32(2);
- ctx.builder.add_i32();
- gen_set_reg16(ctx, regs::SP);
- // return value is already on stack
- }
- pub fn gen_pop16_ss32(ctx: &mut JitContext) {
- // esp = segment_offsets[SS] + reg32[ESP] (or just reg32[ESP] if has_flat_segmentation)
- gen_get_reg32(ctx, regs::ESP);
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- }
- // result = safe_read16(esp)
- let address_local = ctx.builder.set_new_local();
- gen_safe_read16(ctx, &address_local);
- ctx.builder.free_local(address_local);
- // reg32[ESP] += 2;
- gen_get_reg32(ctx, regs::ESP);
- ctx.builder.const_i32(2);
- ctx.builder.add_i32();
- gen_set_reg32(ctx, regs::ESP);
- // return value is already on stack
- }
- pub fn gen_pop16(ctx: &mut JitContext) {
- if ctx.cpu.ssize_32() {
- gen_pop16_ss32(ctx);
- }
- else {
- gen_pop16_ss16(ctx);
- }
- }
- pub fn gen_pop32s_ss16(ctx: &mut JitContext) {
- // sp = reg16[SP]
- gen_get_reg16(ctx, regs::SP);
- // result = safe_read32s(segment_offsets[SS] + sp) (or just sp if has_flat_segmentation)
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- }
- let address_local = ctx.builder.set_new_local();
- gen_safe_read32(ctx, &address_local);
- ctx.builder.free_local(address_local);
- // reg16[SP] = sp + 4;
- gen_get_reg16(ctx, regs::SP);
- ctx.builder.const_i32(4);
- ctx.builder.add_i32();
- gen_set_reg16(ctx, regs::SP);
- // return value is already on stack
- }
- pub fn gen_pop32s_ss32(ctx: &mut JitContext) {
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_reg32(ctx, regs::ESP);
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- let address_local = ctx.builder.set_new_local();
- gen_safe_read32(ctx, &address_local);
- ctx.builder.free_local(address_local);
- }
- else {
- let reg = ctx.register_locals[regs::ESP as usize].unsafe_clone();
- gen_safe_read32(ctx, ®);
- }
- gen_get_reg32(ctx, regs::ESP);
- ctx.builder.const_i32(4);
- ctx.builder.add_i32();
- gen_set_reg32(ctx, regs::ESP);
- // return value is already on stack
- }
- pub fn gen_pop32s(ctx: &mut JitContext) {
- if ctx.cpu.ssize_32() {
- gen_pop32s_ss32(ctx);
- }
- else {
- gen_pop32s_ss16(ctx);
- }
- }
- pub fn gen_adjust_stack_reg(ctx: &mut JitContext, offset: u32) {
- if ctx.cpu.ssize_32() {
- gen_get_reg32(ctx, regs::ESP);
- ctx.builder.const_i32(offset as i32);
- ctx.builder.add_i32();
- gen_set_reg32(ctx, regs::ESP);
- }
- else {
- gen_get_reg16(ctx, regs::SP);
- ctx.builder.const_i32(offset as i32);
- ctx.builder.add_i32();
- gen_set_reg16(ctx, regs::SP);
- }
- }
- pub fn gen_leave(ctx: &mut JitContext, os32: bool) {
- // [e]bp = safe_read{16,32}([e]bp)
- if ctx.cpu.ssize_32() {
- gen_get_reg32(ctx, regs::EBP);
- }
- else {
- gen_get_reg16(ctx, regs::BP);
- }
- let old_vbp = ctx.builder.tee_new_local();
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- }
- if os32 {
- let address_local = ctx.builder.set_new_local();
- gen_safe_read32(ctx, &address_local);
- ctx.builder.free_local(address_local);
- gen_set_reg32(ctx, regs::EBP);
- }
- else {
- let address_local = ctx.builder.set_new_local();
- gen_safe_read16(ctx, &address_local);
- ctx.builder.free_local(address_local);
- gen_set_reg16(ctx, regs::BP);
- }
- // [e]sp = [e]bp + (os32 ? 4 : 2)
- if ctx.cpu.ssize_32() {
- ctx.builder.get_local(&old_vbp);
- ctx.builder.const_i32(if os32 { 4 } else { 2 });
- ctx.builder.add_i32();
- gen_set_reg32(ctx, regs::ESP);
- }
- else {
- ctx.builder.get_local(&old_vbp);
- ctx.builder.const_i32(if os32 { 4 } else { 2 });
- ctx.builder.add_i32();
- gen_set_reg16(ctx, regs::SP);
- }
- ctx.builder.free_local(old_vbp);
- }
- pub fn gen_task_switch_test(ctx: &mut JitContext) {
- // generate if(cr[0] & (CR0_EM | CR0_TS)) { task_switch_test_jit(); goto exit_with_fault; }
- let cr0_offset = global_pointers::get_creg_offset(0);
- dbg_assert!(regs::CR0_EM | regs::CR0_TS <= 0xFF);
- ctx.builder.load_fixed_u8(cr0_offset);
- ctx.builder.const_i32((regs::CR0_EM | regs::CR0_TS) as i32);
- ctx.builder.and_i32();
- ctx.builder.if_void();
- {
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- gen_fn1_const(
- ctx.builder,
- "task_switch_test_jit",
- ctx.start_of_current_instruction,
- );
- ctx.builder.br(ctx.exit_with_fault_label);
- }
- ctx.builder.block_end();
- }
- pub fn gen_task_switch_test_mmx(ctx: &mut JitContext) {
- // generate if(cr[0] & (CR0_EM | CR0_TS)) { task_switch_test_mmx_jit(); goto exit_with_fault; }
- let cr0_offset = global_pointers::get_creg_offset(0);
- dbg_assert!(regs::CR0_EM | regs::CR0_TS <= 0xFF);
- ctx.builder.load_fixed_u8(cr0_offset);
- ctx.builder.const_i32((regs::CR0_EM | regs::CR0_TS) as i32);
- ctx.builder.and_i32();
- ctx.builder.if_void();
- {
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- gen_fn1_const(
- ctx.builder,
- "task_switch_test_mmx_jit",
- ctx.start_of_current_instruction,
- );
- ctx.builder.br(ctx.exit_with_fault_label);
- }
- ctx.builder.block_end();
- }
- pub fn gen_push16(ctx: &mut JitContext, value_local: &WasmLocal) {
- if ctx.cpu.ssize_32() {
- gen_get_reg32(ctx, regs::ESP);
- }
- else {
- gen_get_reg16(ctx, regs::SP);
- };
- ctx.builder.const_i32(2);
- ctx.builder.sub_i32();
- let reg_updated_local = if !ctx.cpu.ssize_32() || !ctx.cpu.has_flat_segmentation() {
- let reg_updated_local = ctx.builder.tee_new_local();
- if !ctx.cpu.ssize_32() {
- ctx.builder.const_i32(0xFFFF);
- ctx.builder.and_i32();
- }
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- }
- let sp_local = ctx.builder.set_new_local();
- gen_safe_write16(ctx, &sp_local, &value_local);
- ctx.builder.free_local(sp_local);
- ctx.builder.get_local(®_updated_local);
- reg_updated_local
- }
- else {
- // short path: The address written to is equal to ESP/SP minus two
- let reg_updated_local = ctx.builder.tee_new_local();
- gen_safe_write16(ctx, ®_updated_local, &value_local);
- reg_updated_local
- };
- if ctx.cpu.ssize_32() {
- gen_set_reg32(ctx, regs::ESP);
- }
- else {
- gen_set_reg16(ctx, regs::SP);
- };
- ctx.builder.free_local(reg_updated_local);
- }
- pub fn gen_push32(ctx: &mut JitContext, value_local: &WasmLocal) {
- if ctx.cpu.ssize_32() {
- gen_get_reg32(ctx, regs::ESP);
- }
- else {
- gen_get_reg16(ctx, regs::SP);
- };
- ctx.builder.const_i32(4);
- ctx.builder.sub_i32();
- let new_sp_local = if !ctx.cpu.ssize_32() || !ctx.cpu.has_flat_segmentation() {
- let new_sp_local = ctx.builder.tee_new_local();
- if !ctx.cpu.ssize_32() {
- ctx.builder.const_i32(0xFFFF);
- ctx.builder.and_i32();
- }
- if !ctx.cpu.has_flat_segmentation() {
- gen_get_ss_offset(ctx);
- ctx.builder.add_i32();
- }
- let sp_local = ctx.builder.set_new_local();
- gen_safe_write32(ctx, &sp_local, &value_local);
- ctx.builder.free_local(sp_local);
- ctx.builder.get_local(&new_sp_local);
- new_sp_local
- }
- else {
- // short path: The address written to is equal to ESP/SP minus four
- let new_sp_local = ctx.builder.tee_new_local();
- gen_safe_write32(ctx, &new_sp_local, &value_local);
- new_sp_local
- };
- if ctx.cpu.ssize_32() {
- gen_set_reg32(ctx, regs::ESP);
- }
- else {
- gen_set_reg16(ctx, regs::SP);
- };
- ctx.builder.free_local(new_sp_local);
- }
- pub fn gen_get_real_eip(ctx: &mut JitContext) {
- gen_get_eip(ctx.builder);
- ctx.builder.const_i32(!0xFFF);
- ctx.builder.and_i32();
- ctx.builder.const_i32(ctx.cpu.eip as i32 & 0xFFF);
- ctx.builder.or_i32();
- if !ctx.cpu.has_flat_segmentation() {
- ctx.builder
- .load_fixed_i32(global_pointers::get_seg_offset(regs::CS));
- ctx.builder.sub_i32();
- }
- }
- pub fn gen_set_last_op1(builder: &mut WasmBuilder, source: &WasmLocal) {
- builder.const_i32(global_pointers::last_op1 as i32);
- builder.get_local(&source);
- builder.store_aligned_i32(0);
- }
- pub fn gen_set_last_result(builder: &mut WasmBuilder, source: &WasmLocal) {
- builder.const_i32(global_pointers::last_result as i32);
- builder.get_local(&source);
- builder.store_aligned_i32(0);
- }
- pub fn gen_clear_flags_changed_bits(builder: &mut WasmBuilder, bits_to_clear: i32) {
- builder.const_i32(global_pointers::flags_changed as i32);
- gen_get_flags_changed(builder);
- builder.const_i32(!bits_to_clear);
- builder.and_i32();
- builder.store_aligned_i32(0);
- }
- pub fn gen_set_last_op_size_and_flags_changed(
- builder: &mut WasmBuilder,
- last_op_size: i32,
- flags_changed: i32,
- ) {
- dbg_assert!(last_op_size == OPSIZE_8 || last_op_size == OPSIZE_16 || last_op_size == OPSIZE_32);
- dbg_assert!(global_pointers::last_op_size as i32 % 8 == 0);
- dbg_assert!(global_pointers::last_op_size as i32 + 4 == global_pointers::flags_changed as i32);
- builder.const_i32(global_pointers::last_op_size as i32);
- builder.const_i64(last_op_size as u32 as i64 | (flags_changed as u32 as i64) << 32);
- builder.store_aligned_i64(0);
- }
- pub fn gen_set_flags_bits(builder: &mut WasmBuilder, bits_to_set: i32) {
- builder.const_i32(global_pointers::flags as i32);
- gen_get_flags(builder);
- builder.const_i32(bits_to_set);
- builder.or_i32();
- builder.store_aligned_i32(0);
- }
- pub fn gen_clear_flags_bits(builder: &mut WasmBuilder, bits_to_clear: i32) {
- builder.const_i32(global_pointers::flags as i32);
- gen_get_flags(builder);
- builder.const_i32(!bits_to_clear);
- builder.and_i32();
- builder.store_aligned_i32(0);
- }
- #[derive(PartialEq)]
- pub enum ConditionNegate {
- True,
- False,
- }
- pub fn gen_getzf(ctx: &mut JitContext, negate: ConditionNegate) {
- match &ctx.previous_instruction {
- Instruction::Cmp {
- dest: InstructionOperandDest::WasmLocal(dest),
- source: InstructionOperand::WasmLocal(source),
- opsize: OPSIZE_32,
- } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- ctx.builder.get_local(dest);
- ctx.builder.get_local(source);
- if negate == ConditionNegate::False {
- ctx.builder.eq_i32();
- }
- else {
- ctx.builder.ne_i32();
- }
- },
- Instruction::Cmp {
- dest: InstructionOperandDest::WasmLocal(dest),
- source: InstructionOperand::Immediate(i),
- opsize: OPSIZE_32,
- } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- ctx.builder.get_local(dest);
- if *i != 0 {
- ctx.builder.const_i32(*i);
- if negate == ConditionNegate::False {
- ctx.builder.eq_i32();
- }
- else {
- ctx.builder.ne_i32();
- }
- }
- else {
- if negate == ConditionNegate::False {
- ctx.builder.eqz_i32();
- }
- }
- },
- Instruction::Cmp { .. }
- | Instruction::Sub { .. }
- | Instruction::Add { .. }
- | Instruction::AdcSbb { .. }
- | Instruction::NonZeroShift { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- if negate == ConditionNegate::False {
- ctx.builder.eqz_i32();
- }
- },
- Instruction::Bitwise { opsize, .. } => {
- let &opsize = opsize;
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- // Note: Necessary because test{8,16} don't mask their neither last_result nor any of their operands
- // TODO: Use local instead of last_result for 8-bit/16-bit
- if opsize == OPSIZE_32 {
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- }
- else if opsize == OPSIZE_16 {
- ctx.builder
- .load_fixed_u16(global_pointers::last_result as u32);
- }
- else if opsize == OPSIZE_8 {
- ctx.builder
- .load_fixed_u8(global_pointers::last_result as u32);
- }
- if negate == ConditionNegate::False {
- ctx.builder.eqz_i32();
- }
- },
- &Instruction::Other => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_get_flags_changed(ctx.builder);
- ctx.builder.const_i32(FLAG_ZERO);
- ctx.builder.and_i32();
- ctx.builder.if_i32();
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- let last_result = ctx.builder.tee_new_local();
- ctx.builder.const_i32(-1);
- ctx.builder.xor_i32();
- ctx.builder.get_local(&last_result);
- ctx.builder.free_local(last_result);
- ctx.builder.const_i32(1);
- ctx.builder.sub_i32();
- ctx.builder.and_i32();
- gen_get_last_op_size(ctx.builder);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- ctx.builder.else_();
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_ZERO);
- ctx.builder.and_i32();
- ctx.builder.block_end();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- }
- }
- pub fn gen_getcf(ctx: &mut JitContext, negate: ConditionNegate) {
- match &ctx.previous_instruction {
- Instruction::Cmp { source, opsize, .. }
- | Instruction::Sub {
- source,
- opsize,
- is_dec: false,
- ..
- } => {
- // Note: x < y and x < x - y can be used interchangeably (see getcf)
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- match (opsize, source) {
- (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l),
- (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(i),
- _ => gen_get_last_result(ctx.builder, &ctx.previous_instruction),
- }
- if negate == ConditionNegate::True {
- ctx.builder.geu_i32();
- }
- else {
- ctx.builder.ltu_i32();
- }
- },
- Instruction::Add {
- source,
- opsize,
- is_inc: false,
- ..
- } => {
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- match (opsize, source) {
- (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l),
- (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(i),
- _ => gen_get_last_op1(ctx.builder, &ctx.previous_instruction),
- }
- if negate == ConditionNegate::True {
- ctx.builder.geu_i32();
- }
- else {
- ctx.builder.ltu_i32();
- }
- },
- Instruction::Add { is_inc: true, .. } | Instruction::Sub { is_dec: true, .. } => {
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_CARRY);
- ctx.builder.and_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- Instruction::Bitwise { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- ctx.builder
- .const_i32(if negate == ConditionNegate::True { 1 } else { 0 });
- },
- Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_CARRY);
- ctx.builder.and_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- &Instruction::Other => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_get_flags_changed(ctx.builder);
- let flags_changed = ctx.builder.tee_new_local();
- ctx.builder.const_i32(FLAG_CARRY);
- ctx.builder.and_i32();
- ctx.builder.if_i32();
- ctx.builder.get_local(&flags_changed);
- ctx.builder.const_i32(31);
- ctx.builder.shr_s_i32();
- ctx.builder.free_local(flags_changed);
- let sub_mask = ctx.builder.set_new_local();
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.get_local(&sub_mask);
- ctx.builder.xor_i32();
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- ctx.builder.get_local(&sub_mask);
- ctx.builder.xor_i32();
- ctx.builder.ltu_i32();
- ctx.builder.else_();
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_CARRY);
- ctx.builder.and_i32();
- ctx.builder.block_end();
- ctx.builder.free_local(sub_mask);
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- }
- }
- pub fn gen_getsf(ctx: &mut JitContext, negate: ConditionNegate) {
- match &ctx.previous_instruction {
- Instruction::Cmp { opsize, .. }
- | Instruction::Sub { opsize, .. }
- | Instruction::Add { opsize, .. }
- | Instruction::AdcSbb { opsize, .. }
- | Instruction::Bitwise { opsize, .. }
- | Instruction::NonZeroShift { opsize, .. } => {
- let &opsize = opsize;
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- if opsize == OPSIZE_32 {
- ctx.builder.const_i32(0);
- if negate == ConditionNegate::True {
- ctx.builder.ge_i32();
- }
- else {
- ctx.builder.lt_i32();
- }
- }
- else {
- // TODO: use register (see get_last_result)
- ctx.builder
- .const_i32(if opsize == OPSIZE_16 { 0x8000 } else { 0x80 });
- ctx.builder.and_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- }
- },
- &Instruction::Other => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_get_flags_changed(ctx.builder);
- ctx.builder.const_i32(FLAG_SIGN);
- ctx.builder.and_i32();
- ctx.builder.if_i32();
- {
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- gen_get_last_op_size(ctx.builder);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- }
- ctx.builder.else_();
- {
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_SIGN);
- ctx.builder.and_i32();
- }
- ctx.builder.block_end();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- }
- }
- pub fn gen_getof(ctx: &mut JitContext) {
- match &ctx.previous_instruction {
- Instruction::Cmp { opsize, .. } | Instruction::Sub { opsize, .. } => {
- // TODO: a better formula might be possible
- let &opsize = opsize;
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.xor_i32();
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- ctx.builder.xor_i32();
- ctx.builder.and_i32();
- ctx.builder.const_i32(if opsize == OPSIZE_32 {
- 0x8000_0000u32 as i32
- }
- else if opsize == OPSIZE_16 {
- 0x8000
- }
- else {
- 0x80
- });
- ctx.builder.and_i32();
- },
- Instruction::Add { opsize, .. } => {
- // TODO: a better formula might be possible
- let &opsize = opsize;
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.xor_i32();
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- ctx.builder.xor_i32();
- ctx.builder.and_i32();
- ctx.builder.const_i32(if opsize == OPSIZE_32 {
- 0x8000_0000u32 as i32
- }
- else if opsize == OPSIZE_16 {
- 0x8000
- }
- else {
- 0x80
- });
- ctx.builder.and_i32();
- },
- Instruction::Bitwise { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- ctx.builder.const_i32(0);
- },
- Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_OVERFLOW);
- ctx.builder.and_i32();
- },
- &Instruction::Other => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_get_flags_changed(ctx.builder);
- let flags_changed = ctx.builder.tee_new_local();
- ctx.builder.const_i32(FLAG_OVERFLOW);
- ctx.builder.and_i32();
- ctx.builder.if_i32();
- {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- let last_op1 = ctx.builder.tee_new_local();
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- let last_result = ctx.builder.tee_new_local();
- ctx.builder.xor_i32();
- ctx.builder.get_local(&last_result);
- ctx.builder.get_local(&last_op1);
- ctx.builder.sub_i32();
- gen_get_flags_changed(ctx.builder);
- ctx.builder.const_i32(31);
- ctx.builder.shr_u_i32();
- ctx.builder.sub_i32();
- ctx.builder.get_local(&last_result);
- ctx.builder.xor_i32();
- ctx.builder.and_i32();
- gen_get_last_op_size(ctx.builder);
- ctx.builder.shr_u_i32();
- ctx.builder.const_i32(1);
- ctx.builder.and_i32();
- ctx.builder.free_local(last_op1);
- ctx.builder.free_local(last_result);
- }
- ctx.builder.else_();
- {
- gen_get_flags(ctx.builder);
- ctx.builder.const_i32(FLAG_OVERFLOW);
- ctx.builder.and_i32();
- }
- ctx.builder.block_end();
- ctx.builder.free_local(flags_changed);
- },
- }
- }
- pub fn gen_test_be(ctx: &mut JitContext, negate: ConditionNegate) {
- match &ctx.previous_instruction {
- Instruction::Cmp {
- dest,
- source,
- opsize,
- } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- match dest {
- InstructionOperandDest::WasmLocal(l) => {
- ctx.builder.get_local(l);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF });
- ctx.builder.and_i32();
- }
- },
- InstructionOperandDest::Other => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- },
- }
- match source {
- InstructionOperand::WasmLocal(l) => {
- ctx.builder.get_local(l);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF });
- ctx.builder.and_i32();
- }
- },
- InstructionOperand::Other => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF });
- ctx.builder.and_i32();
- }
- },
- &InstructionOperand::Immediate(i) => {
- dbg_assert!(*opsize != OPSIZE_8 || i >= 0 && i < 0x100);
- dbg_assert!(*opsize != OPSIZE_16 || i >= 0 && i < 0x10000);
- ctx.builder.const_i32(i);
- },
- }
- if negate == ConditionNegate::True {
- ctx.builder.gtu_i32();
- }
- else {
- ctx.builder.leu_i32();
- }
- },
- Instruction::Sub {
- opsize,
- source,
- is_dec: false,
- ..
- } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- match (opsize, source) {
- (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l),
- (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(i),
- _ => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF });
- ctx.builder.and_i32();
- }
- },
- }
- if negate == ConditionNegate::True {
- ctx.builder.gtu_i32();
- }
- else {
- ctx.builder.leu_i32();
- }
- },
- &Instruction::Bitwise { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_getzf(ctx, negate);
- },
- &Instruction::Add { .. } | &Instruction::Sub { is_dec: true, .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- // not the best code generation, but reasonable for this fairly uncommon case
- gen_getcf(ctx, ConditionNegate::False);
- gen_getzf(ctx, ConditionNegate::False);
- ctx.builder.or_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- Instruction::Other | Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_getcf(ctx, ConditionNegate::False);
- gen_getzf(ctx, ConditionNegate::False);
- ctx.builder.or_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- }
- }
- pub fn gen_test_l(ctx: &mut JitContext, negate: ConditionNegate) {
- match &ctx.previous_instruction {
- Instruction::Cmp {
- dest,
- source,
- opsize,
- } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- match dest {
- InstructionOperandDest::WasmLocal(l) => {
- ctx.builder.get_local(l);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- InstructionOperandDest::Other => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- }
- match source {
- InstructionOperand::WasmLocal(l) => {
- ctx.builder.get_local(l);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- InstructionOperand::Other => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- &InstructionOperand::Immediate(i) => {
- ctx.builder.const_i32(i);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- }
- if negate == ConditionNegate::True {
- ctx.builder.ge_i32();
- }
- else {
- ctx.builder.lt_i32();
- }
- },
- Instruction::Sub { opsize, source, .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- match (opsize, source) {
- (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l),
- (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(
- i << if *opsize == OPSIZE_32 {
- 0
- }
- else if *opsize == OPSIZE_16 {
- 16
- }
- else {
- 24
- },
- ),
- _ => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- }
- if negate == ConditionNegate::True {
- ctx.builder.ge_i32();
- }
- else {
- ctx.builder.lt_i32();
- }
- },
- &Instruction::Bitwise { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_getsf(ctx, negate);
- },
- &Instruction::Other
- | Instruction::Add { .. }
- | Instruction::NonZeroShift { .. }
- | Instruction::AdcSbb { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- if let Instruction::Add { .. } = ctx.previous_instruction {
- gen_profiler_stat_increment(
- ctx.builder,
- profiler::stat::CONDITION_UNOPTIMISED_UNHANDLED_L,
- );
- }
- gen_getsf(ctx, ConditionNegate::False);
- ctx.builder.eqz_i32();
- gen_getof(ctx);
- ctx.builder.eqz_i32();
- ctx.builder.xor_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- }
- }
- pub fn gen_test_le(ctx: &mut JitContext, negate: ConditionNegate) {
- match &ctx.previous_instruction {
- Instruction::Cmp {
- dest,
- source,
- opsize,
- } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- match dest {
- InstructionOperandDest::WasmLocal(l) => {
- ctx.builder.get_local(l);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- InstructionOperandDest::Other => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- }
- match source {
- InstructionOperand::WasmLocal(l) => {
- ctx.builder.get_local(l);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- InstructionOperand::Other => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- &InstructionOperand::Immediate(i) => {
- ctx.builder.const_i32(i);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- }
- if negate == ConditionNegate::True {
- ctx.builder.gt_i32();
- }
- else {
- ctx.builder.le_i32();
- }
- },
- Instruction::Sub { opsize, source, .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- match (opsize, source) {
- (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l),
- (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(
- i << if *opsize == OPSIZE_32 {
- 0
- }
- else if *opsize == OPSIZE_16 {
- 16
- }
- else {
- 24
- },
- ),
- _ => {
- gen_get_last_op1(ctx.builder, &ctx.previous_instruction);
- gen_get_last_result(ctx.builder, &ctx.previous_instruction);
- ctx.builder.sub_i32();
- if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 {
- ctx.builder
- .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 });
- ctx.builder.shl_i32();
- }
- },
- }
- if negate == ConditionNegate::True {
- ctx.builder.gt_i32();
- }
- else {
- ctx.builder.le_i32();
- }
- },
- &Instruction::Bitwise { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED);
- // TODO: Could probably be improved (<= 0)
- gen_test_l(ctx, ConditionNegate::False);
- gen_getzf(ctx, ConditionNegate::False);
- ctx.builder.or_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- Instruction::Other
- | Instruction::Add { .. }
- | Instruction::NonZeroShift { .. }
- | Instruction::AdcSbb { .. } => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- if let Instruction::Add { .. } = ctx.previous_instruction {
- gen_profiler_stat_increment(
- ctx.builder,
- profiler::stat::CONDITION_UNOPTIMISED_UNHANDLED_LE,
- );
- }
- gen_test_l(ctx, ConditionNegate::False);
- gen_getzf(ctx, ConditionNegate::False);
- ctx.builder.or_i32();
- if negate == ConditionNegate::True {
- ctx.builder.eqz_i32();
- }
- },
- }
- }
- pub fn gen_test_loopnz(ctx: &mut JitContext, is_asize_32: bool) {
- gen_test_loop(ctx, is_asize_32);
- ctx.builder.eqz_i32();
- gen_getzf(ctx, ConditionNegate::False);
- ctx.builder.or_i32();
- ctx.builder.eqz_i32();
- }
- pub fn gen_test_loopz(ctx: &mut JitContext, is_asize_32: bool) {
- gen_test_loop(ctx, is_asize_32);
- ctx.builder.eqz_i32();
- gen_getzf(ctx, ConditionNegate::False);
- ctx.builder.eqz_i32();
- ctx.builder.or_i32();
- ctx.builder.eqz_i32();
- }
- pub fn gen_test_loop(ctx: &mut JitContext, is_asize_32: bool) {
- if is_asize_32 {
- gen_get_reg32(ctx, regs::ECX);
- }
- else {
- gen_get_reg16(ctx, regs::CX);
- }
- }
- pub fn gen_test_jcxz(ctx: &mut JitContext, is_asize_32: bool) {
- if is_asize_32 {
- gen_get_reg32(ctx, regs::ECX);
- }
- else {
- gen_get_reg16(ctx, regs::CX);
- }
- ctx.builder.eqz_i32();
- }
- pub fn gen_fpu_get_sti(ctx: &mut JitContext, i: u32) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- ctx.builder.const_i32(i as i32);
- ctx.builder.call_fn2("fpu_get_sti");
- ctx.builder
- .load_fixed_i64(global_pointers::sse_scratch_register as u32);
- ctx.builder
- .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8);
- }
- pub fn gen_fpu_load_m32(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- gen_modrm_resolve_safe_read32(ctx, modrm_byte);
- ctx.builder.call_fn2("f32_to_f80");
- ctx.builder
- .load_fixed_i64(global_pointers::sse_scratch_register as u32);
- ctx.builder
- .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8);
- }
- pub fn gen_fpu_load_m64(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- gen_modrm_resolve_safe_read64(ctx, modrm_byte);
- ctx.builder.call_fn2_i32_i64("f64_to_f80");
- ctx.builder
- .load_fixed_i64(global_pointers::sse_scratch_register as u32);
- ctx.builder
- .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8);
- }
- pub fn gen_fpu_load_i16(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- gen_modrm_resolve_safe_read16(ctx, modrm_byte);
- sign_extend_i16(ctx.builder);
- ctx.builder.call_fn2("i32_to_f80");
- ctx.builder
- .load_fixed_i64(global_pointers::sse_scratch_register as u32);
- ctx.builder
- .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8);
- }
- pub fn gen_fpu_load_i32(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- gen_modrm_resolve_safe_read32(ctx, modrm_byte);
- ctx.builder.call_fn2("i32_to_f80");
- ctx.builder
- .load_fixed_i64(global_pointers::sse_scratch_register as u32);
- ctx.builder
- .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8);
- }
- pub fn gen_fpu_load_i64(ctx: &mut JitContext, modrm_byte: ModrmByte) {
- ctx.builder
- .const_i32(global_pointers::sse_scratch_register as i32);
- gen_modrm_resolve_safe_read64(ctx, modrm_byte);
- ctx.builder.call_fn2_i32_i64("i64_to_f80");
- ctx.builder
- .load_fixed_i64(global_pointers::sse_scratch_register as u32);
- ctx.builder
- .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8);
- }
- pub fn gen_trigger_de(ctx: &mut JitContext) {
- gen_fn1_const(
- ctx.builder,
- "trigger_de_jit",
- ctx.start_of_current_instruction,
- );
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- ctx.builder.br(ctx.exit_with_fault_label);
- }
- pub fn gen_trigger_ud(ctx: &mut JitContext) {
- gen_fn1_const(
- ctx.builder,
- "trigger_ud_jit",
- ctx.start_of_current_instruction,
- );
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- ctx.builder.br(ctx.exit_with_fault_label);
- }
- pub fn gen_trigger_gp(ctx: &mut JitContext, error_code: u32) {
- gen_fn2_const(
- ctx.builder,
- "trigger_gp_jit",
- error_code,
- ctx.start_of_current_instruction,
- );
- gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction);
- ctx.builder.br(ctx.exit_with_fault_label);
- }
- pub fn gen_condition_fn_negated(ctx: &mut JitContext, condition: u8) {
- gen_condition_fn(ctx, condition ^ 1)
- }
- pub fn gen_condition_fn(ctx: &mut JitContext, condition: u8) {
- if condition & 0xF0 == 0x00 || condition & 0xF0 == 0x70 || condition & 0xF0 == 0x80 {
- match condition & 0xF {
- 0x0 => {
- gen_getof(ctx);
- },
- 0x1 => {
- gen_getof(ctx);
- ctx.builder.eqz_i32();
- },
- 0x2 => {
- gen_getcf(ctx, ConditionNegate::False);
- },
- 0x3 => {
- gen_getcf(ctx, ConditionNegate::True);
- },
- 0x4 => {
- gen_getzf(ctx, ConditionNegate::False);
- },
- 0x5 => {
- gen_getzf(ctx, ConditionNegate::True);
- },
- 0x6 => {
- gen_test_be(ctx, ConditionNegate::False);
- },
- 0x7 => {
- gen_test_be(ctx, ConditionNegate::True);
- },
- 0x8 => {
- gen_getsf(ctx, ConditionNegate::False);
- },
- 0x9 => {
- gen_getsf(ctx, ConditionNegate::True);
- },
- 0xA => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED_PF);
- ctx.builder.call_fn0_ret("test_p");
- },
- 0xB => {
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED);
- gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED_PF);
- ctx.builder.call_fn0_ret("test_np");
- },
- 0xC => {
- gen_test_l(ctx, ConditionNegate::False);
- },
- 0xD => {
- gen_test_l(ctx, ConditionNegate::True);
- },
- 0xE => {
- gen_test_le(ctx, ConditionNegate::False);
- },
- 0xF => {
- gen_test_le(ctx, ConditionNegate::True);
- },
- _ => {
- dbg_assert!(false);
- },
- }
- }
- else {
- // loop, loopnz, loopz, jcxz
- dbg_assert!(condition & !0x3 == 0xE0);
- if condition == 0xE0 {
- gen_test_loopnz(ctx, ctx.cpu.asize_32());
- }
- else if condition == 0xE1 {
- gen_test_loopz(ctx, ctx.cpu.asize_32());
- }
- else if condition == 0xE2 {
- gen_test_loop(ctx, ctx.cpu.asize_32());
- }
- else if condition == 0xE3 {
- gen_test_jcxz(ctx, ctx.cpu.asize_32());
- }
- }
- }
- pub fn gen_move_registers_from_locals_to_memory(ctx: &mut JitContext) {
- if cfg!(feature = "profiler") {
- let instruction = memory::read32s(ctx.start_of_current_instruction) as u32;
- ::opstats::gen_opstat_unguarded_register(ctx.builder, instruction);
- }
- for i in 0..8 {
- ctx.builder
- .const_i32(global_pointers::get_reg32_offset(i as u32) as i32);
- ctx.builder.get_local(&ctx.register_locals[i]);
- ctx.builder.store_aligned_i32(0);
- }
- }
- pub fn gen_move_registers_from_memory_to_locals(ctx: &mut JitContext) {
- if cfg!(feature = "profiler") {
- let instruction = memory::read32s(ctx.start_of_current_instruction) as u32;
- ::opstats::gen_opstat_unguarded_register(ctx.builder, instruction);
- }
- for i in 0..8 {
- ctx.builder
- .const_i32(global_pointers::get_reg32_offset(i as u32) as i32);
- ctx.builder.load_aligned_i32(0);
- ctx.builder.set_local(&ctx.register_locals[i]);
- }
- }
- pub fn gen_profiler_stat_increment(builder: &mut WasmBuilder, stat: profiler::stat) {
- if !cfg!(feature = "profiler") {
- return;
- }
- let addr = unsafe { profiler::stat_array.as_mut_ptr().offset(stat as isize) } as u32;
- builder.increment_fixed_i64(addr, 1)
- }
- pub fn gen_debug_track_jit_exit(builder: &mut WasmBuilder, address: u32) {
- if cfg!(feature = "profiler") {
- gen_fn1_const(builder, "track_jit_exit", address);
- }
- }
|