use cpu::cpu::{ tlb_data, FLAG_CARRY, FLAG_OVERFLOW, FLAG_SIGN, FLAG_ZERO, OPSIZE_8, OPSIZE_16, OPSIZE_32, TLB_GLOBAL, TLB_HAS_CODE, TLB_NO_USER, TLB_READONLY, TLB_VALID, }; use cpu::global_pointers; use cpu::memory; use jit::{Instruction, InstructionOperand, InstructionOperandDest, JitContext}; use modrm; use modrm::ModrmByte; use profiler; use regs; use wasmgen::wasm_builder::{WasmBuilder, WasmLocal, WasmLocalI64}; pub fn gen_add_cs_offset(ctx: &mut JitContext) { if !ctx.cpu.has_flat_segmentation() { ctx.builder .load_fixed_i32(global_pointers::get_seg_offset(regs::CS)); ctx.builder.add_i32(); } } pub fn gen_get_eip(builder: &mut WasmBuilder) { builder.load_fixed_i32(global_pointers::instruction_pointer as u32); } pub fn gen_set_eip_to_after_current_instruction(ctx: &mut JitContext) { ctx.builder .const_i32(global_pointers::instruction_pointer as i32); gen_get_eip(ctx.builder); ctx.builder.const_i32(!0xFFF); ctx.builder.and_i32(); ctx.builder.const_i32(ctx.cpu.eip as i32 & 0xFFF); ctx.builder.or_i32(); ctx.builder.store_aligned_i32(0); } pub fn gen_set_previous_eip_offset_from_eip_with_low_bits( builder: &mut WasmBuilder, low_bits: i32, ) { // previous_ip = instruction_pointer & ~0xFFF | low_bits; dbg_assert!(low_bits & !0xFFF == 0); builder.const_i32(global_pointers::previous_ip as i32); gen_get_eip(builder); builder.const_i32(!0xFFF); builder.and_i32(); builder.const_i32(low_bits); builder.or_i32(); builder.store_aligned_i32(0); } pub fn gen_set_eip_low_bits(builder: &mut WasmBuilder, low_bits: i32) { // instruction_pointer = instruction_pointer & ~0xFFF | low_bits; dbg_assert!(low_bits & !0xFFF == 0); builder.const_i32(global_pointers::instruction_pointer as i32); gen_get_eip(builder); builder.const_i32(!0xFFF); builder.and_i32(); builder.const_i32(low_bits); builder.or_i32(); builder.store_aligned_i32(0); } pub fn gen_set_eip_low_bits_and_jump_rel32(builder: &mut WasmBuilder, low_bits: i32, n: i32) { // instruction_pointer = (instruction_pointer & ~0xFFF | low_bits) + n; dbg_assert!(low_bits & !0xFFF == 0); builder.const_i32(global_pointers::instruction_pointer as i32); gen_get_eip(builder); builder.const_i32(!0xFFF); builder.and_i32(); builder.const_i32(low_bits); builder.or_i32(); if n != 0 { builder.const_i32(n); builder.add_i32(); } builder.store_aligned_i32(0); } pub fn gen_relative_jump(builder: &mut WasmBuilder, n: i32) { // add n to instruction_pointer if n != 0 { builder.const_i32(global_pointers::instruction_pointer as i32); gen_get_eip(builder); builder.const_i32(n); builder.add_i32(); builder.store_aligned_i32(0); } } pub fn gen_page_switch_check( ctx: &mut JitContext, next_block_addr: u32, last_instruction_addr: u32, ) { // After switching a page while in jitted code, check if the page mapping still holds gen_get_eip(ctx.builder); let address_local = ctx.builder.set_new_local(); gen_get_phys_eip_plus_mem(ctx, &address_local); ctx.builder.free_local(address_local); ctx.builder .const_i32(next_block_addr as i32 + unsafe { memory::mem8 } as i32); ctx.builder.ne_i32(); if cfg!(debug_assertions) { ctx.builder.if_void(); gen_profiler_stat_increment(ctx.builder, profiler::stat::FAILED_PAGE_CHANGE); gen_debug_track_jit_exit(ctx.builder, last_instruction_addr); ctx.builder.br(ctx.exit_label); ctx.builder.block_end(); } else { ctx.builder.br_if(ctx.exit_label); } } pub fn gen_update_instruction_counter(ctx: &mut JitContext) { ctx.builder .const_i32(global_pointers::instruction_counter as i32); ctx.builder .load_fixed_i32(global_pointers::instruction_counter as u32); ctx.builder.get_local(&ctx.instruction_counter); ctx.builder.add_i32(); ctx.builder.store_aligned_i32(0); } pub fn gen_get_reg8(ctx: &mut JitContext, r: u32) { match r { regs::AL | regs::CL | regs::DL | regs::BL => { ctx.builder.get_local(&ctx.register_locals[r as usize]); ctx.builder.const_i32(0xFF); ctx.builder.and_i32(); }, regs::AH | regs::CH | regs::DH | regs::BH => { ctx.builder .get_local(&ctx.register_locals[(r - 4) as usize]); ctx.builder.const_i32(8); ctx.builder.shr_u_i32(); ctx.builder.const_i32(0xFF); ctx.builder.and_i32(); }, _ => assert!(false), } } /// Return a new local referencing one of the 8 bit registers or a direct reference to one of the /// register locals. Higher bits might be garbage (suitable for gen_cmp8 etc.). Must be freed with /// gen_free_reg8_or_alias. pub fn gen_get_reg8_or_alias_to_reg32(ctx: &mut JitContext, r: u32) -> WasmLocal { match r { regs::AL | regs::CL | regs::DL | regs::BL => ctx.register_locals[r as usize].unsafe_clone(), regs::AH | regs::CH | regs::DH | regs::BH => { ctx.builder .get_local(&ctx.register_locals[(r - 4) as usize]); ctx.builder.const_i32(8); ctx.builder.shr_u_i32(); ctx.builder.set_new_local() }, _ => panic!(), } } pub fn gen_free_reg8_or_alias(ctx: &mut JitContext, r: u32, local: WasmLocal) { match r { regs::AL | regs::CL | regs::DL | regs::BL => {}, regs::AH | regs::CH | regs::DH | regs::BH => ctx.builder.free_local(local), _ => panic!(), } } pub fn gen_get_reg16(ctx: &mut JitContext, r: u32) { ctx.builder.get_local(&ctx.register_locals[r as usize]); ctx.builder.const_i32(0xFFFF); ctx.builder.and_i32(); } pub fn gen_get_reg32(ctx: &mut JitContext, r: u32) { ctx.builder.get_local(&ctx.register_locals[r as usize]); } pub fn gen_set_reg8(ctx: &mut JitContext, r: u32) { match r { regs::AL | regs::CL | regs::DL | regs::BL => { // reg32[r] = stack_value & 0xFF | reg32[r] & ~0xFF ctx.builder.const_i32(0xFF); ctx.builder.and_i32(); ctx.builder.get_local(&ctx.register_locals[r as usize]); ctx.builder.const_i32(!0xFF); ctx.builder.and_i32(); ctx.builder.or_i32(); ctx.builder.set_local(&ctx.register_locals[r as usize]); }, regs::AH | regs::CH | regs::DH | regs::BH => { // reg32[r] = stack_value << 8 & 0xFF00 | reg32[r] & ~0xFF00 ctx.builder.const_i32(8); ctx.builder.shl_i32(); ctx.builder.const_i32(0xFF00); ctx.builder.and_i32(); ctx.builder .get_local(&ctx.register_locals[(r - 4) as usize]); ctx.builder.const_i32(!0xFF00); ctx.builder.and_i32(); ctx.builder.or_i32(); ctx.builder .set_local(&ctx.register_locals[(r - 4) as usize]); }, _ => assert!(false), } } pub fn gen_set_reg8_unmasked(ctx: &mut JitContext, r: u32) { if cfg!(debug_assertions) { let val = ctx.builder.set_new_local(); ctx.builder.get_local(&val); ctx.builder.const_i32(!0xFF); ctx.builder.and_i32(); ctx.builder.if_void(); ctx.builder.unreachable(); ctx.builder.block_end(); ctx.builder.get_local(&val); ctx.builder.free_local(val); } match r { regs::AL | regs::CL | regs::DL | regs::BL => { // reg32[r] = stack_value | reg32[r] & ~0xFF ctx.builder.get_local(&ctx.register_locals[r as usize]); ctx.builder.const_i32(!0xFF); ctx.builder.and_i32(); ctx.builder.or_i32(); ctx.builder.set_local(&ctx.register_locals[r as usize]); }, regs::AH | regs::CH | regs::DH | regs::BH => { // reg32[r] = stack_value << 8 | reg32[r] & ~0xFF00 ctx.builder.const_i32(8); ctx.builder.shl_i32(); ctx.builder.const_i32(0xFF00); ctx.builder.and_i32(); ctx.builder .get_local(&ctx.register_locals[(r - 4) as usize]); ctx.builder.const_i32(!0xFF00); ctx.builder.and_i32(); ctx.builder.or_i32(); ctx.builder .set_local(&ctx.register_locals[(r - 4) as usize]); }, _ => assert!(false), } } pub fn gen_set_reg16(ctx: &mut JitContext, r: u32) { gen_set_reg16_local(ctx.builder, &ctx.register_locals[r as usize]); } pub fn gen_set_reg16_unmasked(ctx: &mut JitContext, r: u32) { if cfg!(debug_assertions) { let val = ctx.builder.set_new_local(); ctx.builder.get_local(&val); ctx.builder.const_i32(!0xFFFF); ctx.builder.and_i32(); ctx.builder.if_void(); ctx.builder.unreachable(); ctx.builder.block_end(); ctx.builder.get_local(&val); ctx.builder.free_local(val); } ctx.builder.get_local(&ctx.reg(r)); ctx.builder.const_i32(!0xFFFF); ctx.builder.and_i32(); ctx.builder.or_i32(); ctx.builder.set_local(&ctx.reg(r)); } pub fn gen_set_reg16_local(builder: &mut WasmBuilder, local: &WasmLocal) { // reg32[r] = v & 0xFFFF | reg32[r] & ~0xFFFF builder.const_i32(0xFFFF); builder.and_i32(); builder.get_local(local); builder.const_i32(!0xFFFF); builder.and_i32(); builder.or_i32(); builder.set_local(local); } pub fn gen_set_reg32(ctx: &mut JitContext, r: u32) { ctx.builder.set_local(&ctx.register_locals[r as usize]); } pub fn decr_exc_asize(ctx: &mut JitContext) { gen_get_reg32(ctx, regs::ECX); ctx.builder.const_i32(1); ctx.builder.sub_i32(); if ctx.cpu.asize_32() { gen_set_reg32(ctx, regs::ECX); } else { gen_set_reg16(ctx, regs::CX); } } pub fn gen_read_reg_xmm128_into_scratch(ctx: &mut JitContext, r: u32) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); let dest = global_pointers::get_reg_xmm_offset(r); ctx.builder.const_i32(dest as i32); ctx.builder.load_aligned_i64(0); ctx.builder.store_aligned_i64(0); ctx.builder .const_i32(global_pointers::sse_scratch_register as i32 + 8); let dest = global_pointers::get_reg_xmm_offset(r) + 8; ctx.builder.const_i32(dest as i32); ctx.builder.load_aligned_i64(0); ctx.builder.store_aligned_i64(0); } pub fn gen_get_sreg(ctx: &mut JitContext, r: u32) { ctx.builder .load_fixed_u16(global_pointers::get_sreg_offset(r)) } pub fn gen_get_ss_offset(ctx: &mut JitContext) { ctx.builder .load_fixed_i32(global_pointers::get_seg_offset(regs::SS)); } pub fn gen_get_flags(builder: &mut WasmBuilder) { builder.load_fixed_i32(global_pointers::flags as u32); } fn gen_get_flags_changed(builder: &mut WasmBuilder) { builder.load_fixed_i32(global_pointers::flags_changed as u32); } fn gen_get_last_result(builder: &mut WasmBuilder, previous_instruction: &Instruction) { match previous_instruction { Instruction::Add { dest: InstructionOperandDest::WasmLocal(l), opsize: OPSIZE_32, .. } | Instruction::AdcSbb { dest: InstructionOperandDest::WasmLocal(l), opsize: OPSIZE_32, .. } | Instruction::Sub { dest: InstructionOperandDest::WasmLocal(l), opsize: OPSIZE_32, .. } | Instruction::Bitwise { dest: InstructionOperandDest::WasmLocal(l), opsize: OPSIZE_32, } | Instruction::NonZeroShift { dest: InstructionOperandDest::WasmLocal(l), opsize: OPSIZE_32, } => builder.get_local(&l), Instruction::Cmp { dest: InstructionOperandDest::WasmLocal(l), source, opsize: OPSIZE_32, } => { if source.is_zero() { builder.get_local(&l) } else { builder.load_fixed_i32(global_pointers::last_result as u32) } }, _ => builder.load_fixed_i32(global_pointers::last_result as u32), } } fn gen_get_last_op_size(builder: &mut WasmBuilder) { builder.load_fixed_i32(global_pointers::last_op_size as u32); } fn gen_get_last_op1(builder: &mut WasmBuilder, previous_instruction: &Instruction) { match previous_instruction { Instruction::Cmp { dest: InstructionOperandDest::WasmLocal(l), source: _, opsize: OPSIZE_32, } => builder.get_local(&l), _ => builder.load_fixed_i32(global_pointers::last_op1 as u32), } } pub fn gen_get_page_fault(builder: &mut WasmBuilder) { builder.load_fixed_u8(global_pointers::page_fault as u32); } /// sign-extend a byte value on the stack and leave it on the stack pub fn sign_extend_i8(builder: &mut WasmBuilder) { builder.const_i32(24); builder.shl_i32(); builder.const_i32(24); builder.shr_s_i32(); } /// sign-extend a two byte value on the stack and leave it on the stack pub fn sign_extend_i16(builder: &mut WasmBuilder) { builder.const_i32(16); builder.shl_i32(); builder.const_i32(16); builder.shr_s_i32(); } pub fn gen_fn0_const(builder: &mut WasmBuilder, name: &str) { builder.call_fn0(name) } pub fn gen_fn1_const(builder: &mut WasmBuilder, name: &str, arg0: u32) { builder.const_i32(arg0 as i32); builder.call_fn1(name); } pub fn gen_fn2_const(builder: &mut WasmBuilder, name: &str, arg0: u32, arg1: u32) { builder.const_i32(arg0 as i32); builder.const_i32(arg1 as i32); builder.call_fn2(name); } // helper functions for gen/generate_jit.js pub fn gen_modrm_fn0(builder: &mut WasmBuilder, name: &str) { // generates: fn( _ ) builder.call_fn1(name); } pub fn gen_modrm_fn1(builder: &mut WasmBuilder, name: &str, arg0: u32) { // generates: fn( _, arg0 ) builder.const_i32(arg0 as i32); builder.call_fn2(name); } pub fn gen_modrm_resolve(ctx: &mut JitContext, modrm_byte: ModrmByte) { modrm::gen(ctx, modrm_byte) } pub fn gen_modrm_resolve_with_local( ctx: &mut JitContext, modrm_byte: ModrmByte, gen: &dyn Fn(&mut JitContext, &WasmLocal), ) { if let Some(r) = modrm::get_as_reg_index_if_possible(ctx, &modrm_byte) { gen(ctx, &ctx.reg(r)); } else { gen_modrm_resolve(ctx, modrm_byte); let address = ctx.builder.set_new_local(); gen(ctx, &address); ctx.builder.free_local(address); } } pub fn gen_set_reg8_r(ctx: &mut JitContext, dest: u32, src: u32) { // generates: reg8[r_dest] = reg8[r_src] if src != dest { gen_get_reg8(ctx, src); gen_set_reg8_unmasked(ctx, dest); } } pub fn gen_set_reg16_r(ctx: &mut JitContext, dest: u32, src: u32) { // generates: reg16[r_dest] = reg16[r_src] if src != dest { gen_get_reg16(ctx, src); gen_set_reg16_unmasked(ctx, dest); } } pub fn gen_set_reg32_r(ctx: &mut JitContext, dest: u32, src: u32) { // generates: reg32[r_dest] = reg32[r_src] if src != dest { gen_get_reg32(ctx, src); gen_set_reg32(ctx, dest); } } pub fn gen_modrm_resolve_safe_read8(ctx: &mut JitContext, modrm_byte: ModrmByte) { gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read8(ctx, addr)); } pub fn gen_modrm_resolve_safe_read16(ctx: &mut JitContext, modrm_byte: ModrmByte) { gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read16(ctx, addr)); } pub fn gen_modrm_resolve_safe_read32(ctx: &mut JitContext, modrm_byte: ModrmByte) { gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read32(ctx, addr)); } pub fn gen_modrm_resolve_safe_read64(ctx: &mut JitContext, modrm_byte: ModrmByte) { gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| gen_safe_read64(ctx, addr)); } pub fn gen_modrm_resolve_safe_read128( ctx: &mut JitContext, modrm_byte: ModrmByte, where_to_write: u32, ) { gen_modrm_resolve_with_local(ctx, modrm_byte, &|ctx, addr| { gen_safe_read128(ctx, addr, where_to_write) }); } pub fn gen_safe_read8(ctx: &mut JitContext, address_local: &WasmLocal) { gen_safe_read(ctx, BitSize::BYTE, address_local, None); } pub fn gen_safe_read16(ctx: &mut JitContext, address_local: &WasmLocal) { gen_safe_read(ctx, BitSize::WORD, address_local, None); } pub fn gen_safe_read32(ctx: &mut JitContext, address_local: &WasmLocal) { gen_safe_read(ctx, BitSize::DWORD, address_local, None); } pub fn gen_safe_read64(ctx: &mut JitContext, address_local: &WasmLocal) { gen_safe_read(ctx, BitSize::QWORD, &address_local, None); } pub fn gen_safe_read128(ctx: &mut JitContext, address_local: &WasmLocal, where_to_write: u32) { gen_safe_read(ctx, BitSize::DQWORD, &address_local, Some(where_to_write)); } // only used internally for gen_safe_write enum GenSafeWriteValue<'a> { I32(&'a WasmLocal), I64(&'a WasmLocalI64), TwoI64s(&'a WasmLocalI64, &'a WasmLocalI64), } enum GenSafeReadWriteValue { I32(WasmLocal), I64(WasmLocalI64), } #[derive(Copy, Clone, Eq, PartialEq)] pub enum BitSize { BYTE, WORD, DWORD, QWORD, DQWORD, } impl BitSize { pub fn bytes(&self) -> u32 { match self { BitSize::BYTE => 1, BitSize::WORD => 2, BitSize::DWORD => 4, BitSize::QWORD => 8, BitSize::DQWORD => 16, } } } pub fn gen_safe_write8(ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocal) { gen_safe_write( ctx, BitSize::BYTE, address_local, GenSafeWriteValue::I32(value_local), ) } pub fn gen_safe_write16(ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocal) { gen_safe_write( ctx, BitSize::WORD, address_local, GenSafeWriteValue::I32(value_local), ) } pub fn gen_safe_write32(ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocal) { gen_safe_write( ctx, BitSize::DWORD, address_local, GenSafeWriteValue::I32(value_local), ) } pub fn gen_safe_write64( ctx: &mut JitContext, address_local: &WasmLocal, value_local: &WasmLocalI64, ) { gen_safe_write( ctx, BitSize::QWORD, address_local, GenSafeWriteValue::I64(value_local), ) } pub fn gen_safe_write128( ctx: &mut JitContext, address_local: &WasmLocal, value_local_low: &WasmLocalI64, value_local_high: &WasmLocalI64, ) { gen_safe_write( ctx, BitSize::DQWORD, address_local, GenSafeWriteValue::TwoI64s(value_local_low, value_local_high), ) } fn gen_safe_read( ctx: &mut JitContext, bits: BitSize, address_local: &WasmLocal, where_to_write: Option, ) { // Execute a virtual memory read. All slow paths (memory-mapped IO, tlb miss, page fault and // read across page boundary are handled in safe_read_jit_slow // entry <- tlb_data[addr >> 12 << 2] // if entry & MASK == TLB_VALID && (addr & 0xFFF) <= 0x1000 - bytes: goto fast // entry <- safe_read_jit_slow(addr, instruction_pointer) // if page_fault: goto exit-with-pagefault // fast: mem[(entry & ~0xFFF) ^ addr] let cont = ctx.builder.block_void(); ctx.builder.get_local(&address_local); ctx.builder.const_i32(12); ctx.builder.shr_u_i32(); ctx.builder.const_i32(2); ctx.builder.shl_i32(); ctx.builder .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 }); let entry_local = ctx.builder.tee_new_local(); ctx.builder.const_i32( (0xFFF & !TLB_READONLY & !TLB_GLOBAL & !TLB_HAS_CODE & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32, ); ctx.builder.and_i32(); ctx.builder.const_i32(TLB_VALID as i32); ctx.builder.eq_i32(); if bits != BitSize::BYTE { ctx.builder.get_local(&address_local); ctx.builder.const_i32(0xFFF); ctx.builder.and_i32(); ctx.builder.const_i32(0x1000 - bits.bytes() as i32); ctx.builder.le_i32(); ctx.builder.and_i32(); } ctx.builder.br_if(cont); if cfg!(feature = "profiler") { ctx.builder.get_local(&address_local); ctx.builder.get_local(&entry_local); ctx.builder.call_fn2("report_safe_read_jit_slow"); } ctx.builder.get_local(&address_local); ctx.builder .const_i32(ctx.start_of_current_instruction as i32 & 0xFFF); match bits { BitSize::BYTE => { ctx.builder.call_fn2_ret("safe_read8_slow_jit"); }, BitSize::WORD => { ctx.builder.call_fn2_ret("safe_read16_slow_jit"); }, BitSize::DWORD => { ctx.builder.call_fn2_ret("safe_read32s_slow_jit"); }, BitSize::QWORD => { ctx.builder.call_fn2_ret("safe_read64s_slow_jit"); }, BitSize::DQWORD => { ctx.builder.call_fn2_ret("safe_read128s_slow_jit"); }, } ctx.builder.tee_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); if cfg!(feature = "profiler") { ctx.builder.if_void(); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); ctx.builder.block_end(); ctx.builder.get_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); } ctx.builder.br_if(ctx.exit_with_fault_label); ctx.builder.block_end(); gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_READ_FAST); // XXX: Both fast and slow ctx.builder.get_local(&entry_local); ctx.builder.const_i32(!0xFFF); ctx.builder.and_i32(); ctx.builder.get_local(&address_local); ctx.builder.xor_i32(); // where_to_write is only used by dqword dbg_assert!((where_to_write != None) == (bits == BitSize::DQWORD)); match bits { BitSize::BYTE => { ctx.builder.load_u8(0); }, BitSize::WORD => { ctx.builder.load_unaligned_u16(0); }, BitSize::DWORD => { ctx.builder.load_unaligned_i32(0); }, BitSize::QWORD => { ctx.builder.load_unaligned_i64(0); }, BitSize::DQWORD => { let where_to_write = where_to_write.unwrap(); let virt_address_local = ctx.builder.set_new_local(); ctx.builder.const_i32(0); ctx.builder.get_local(&virt_address_local); ctx.builder.load_unaligned_i64(0); ctx.builder.store_unaligned_i64(where_to_write); ctx.builder.const_i32(0); ctx.builder.get_local(&virt_address_local); ctx.builder.load_unaligned_i64(8); ctx.builder.store_unaligned_i64(where_to_write + 8); ctx.builder.free_local(virt_address_local); }, } ctx.builder.free_local(entry_local); } pub fn gen_get_phys_eip_plus_mem(ctx: &mut JitContext, address_local: &WasmLocal) { // Similar to gen_safe_read, but return the physical eip + memory::mem rather than reading from memory // In functions that need to use this value we need to fix it by substracting memory::mem // this is done in order to remove one instruction from the fast path of memory accesses (no need to add // memory::mem anymore ). // We need to account for this in gen_page_switch_check and we compare with next_block_addr + memory::mem8 // We cannot the same while processing an AbsoluteEip flow control change so there we need to fix the value // by subscracting memory::mem. Overall, since AbsoluteEip is encountered less often than memory accesses so // this ends up improving perf. // Does not (need to) handle mapped memory // XXX: Currently does not use ctx.start_of_current_instruction, but rather assumes that eip is // already correct (pointing at the current instruction) let cont = ctx.builder.block_void(); ctx.builder.get_local(&address_local); ctx.builder.const_i32(12); ctx.builder.shr_u_i32(); ctx.builder.const_i32(2); ctx.builder.shl_i32(); ctx.builder .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 }); let entry_local = ctx.builder.tee_new_local(); ctx.builder.const_i32( (0xFFF & !TLB_READONLY & !TLB_GLOBAL & !TLB_HAS_CODE & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32, ); ctx.builder.and_i32(); ctx.builder.const_i32(TLB_VALID as i32); ctx.builder.eq_i32(); ctx.builder.br_if(cont); if cfg!(feature = "profiler") { ctx.builder.get_local(&address_local); ctx.builder.get_local(&entry_local); ctx.builder.call_fn2("report_safe_read_jit_slow"); } ctx.builder.get_local(&address_local); ctx.builder.call_fn1_ret("get_phys_eip_slow_jit"); ctx.builder.tee_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); if cfg!(feature = "profiler") { ctx.builder.if_void(); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); // XXX ctx.builder.block_end(); ctx.builder.get_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); } ctx.builder.br_if(ctx.exit_with_fault_label); ctx.builder.block_end(); gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_READ_FAST); // XXX: Both fast and slow ctx.builder.get_local(&entry_local); ctx.builder.const_i32(!0xFFF); ctx.builder.and_i32(); ctx.builder.get_local(&address_local); ctx.builder.xor_i32(); ctx.builder.free_local(entry_local); } fn gen_safe_write( ctx: &mut JitContext, bits: BitSize, address_local: &WasmLocal, value_local: GenSafeWriteValue, ) { // Execute a virtual memory write. All slow paths (memory-mapped IO, tlb miss, page fault, // write across page boundary and page containing jitted code are handled in safe_write_jit_slow // entry <- tlb_data[addr >> 12 << 2] // if entry & MASK == TLB_VALID && (addr & 0xFFF) <= 0x1000 - bytes: goto fast // entry <- safe_write_jit_slow(addr, value, instruction_pointer) // if page_fault: goto exit-with-pagefault // fast: mem[(entry & ~0xFFF) ^ addr] <- value let cont = ctx.builder.block_void(); ctx.builder.get_local(&address_local); ctx.builder.const_i32(12); ctx.builder.shr_u_i32(); ctx.builder.const_i32(2); ctx.builder.shl_i32(); ctx.builder .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 }); let entry_local = ctx.builder.tee_new_local(); ctx.builder .const_i32((0xFFF & !TLB_GLOBAL & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32); ctx.builder.and_i32(); ctx.builder.const_i32(TLB_VALID as i32); ctx.builder.eq_i32(); if bits != BitSize::BYTE { ctx.builder.get_local(&address_local); ctx.builder.const_i32(0xFFF); ctx.builder.and_i32(); ctx.builder.const_i32(0x1000 - bits.bytes() as i32); ctx.builder.le_i32(); ctx.builder.and_i32(); } ctx.builder.br_if(cont); if cfg!(feature = "profiler") { ctx.builder.get_local(&address_local); ctx.builder.get_local(&entry_local); ctx.builder.call_fn2("report_safe_write_jit_slow"); } ctx.builder.get_local(&address_local); match value_local { GenSafeWriteValue::I32(local) => ctx.builder.get_local(local), GenSafeWriteValue::I64(local) => ctx.builder.get_local_i64(local), GenSafeWriteValue::TwoI64s(local1, local2) => { ctx.builder.get_local_i64(local1); ctx.builder.get_local_i64(local2) }, } ctx.builder .const_i32(ctx.start_of_current_instruction as i32 & 0xFFF); match bits { BitSize::BYTE => { ctx.builder.call_fn3_ret("safe_write8_slow_jit"); }, BitSize::WORD => { ctx.builder.call_fn3_ret("safe_write16_slow_jit"); }, BitSize::DWORD => { ctx.builder.call_fn3_ret("safe_write32_slow_jit"); }, BitSize::QWORD => { ctx.builder .call_fn3_i32_i64_i32_ret("safe_write64_slow_jit"); }, BitSize::DQWORD => { ctx.builder .call_fn4_i32_i64_i64_i32_ret("safe_write128_slow_jit"); }, } ctx.builder.tee_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); if cfg!(feature = "profiler") { ctx.builder.if_void(); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); ctx.builder.block_end(); ctx.builder.get_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); } ctx.builder.br_if(ctx.exit_with_fault_label); ctx.builder.block_end(); gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_WRITE_FAST); // XXX: Both fast and slow ctx.builder.get_local(&entry_local); ctx.builder.const_i32(!0xFFF); ctx.builder.and_i32(); ctx.builder.get_local(&address_local); ctx.builder.xor_i32(); match value_local { GenSafeWriteValue::I32(local) => ctx.builder.get_local(local), GenSafeWriteValue::I64(local) => ctx.builder.get_local_i64(local), GenSafeWriteValue::TwoI64s(local1, local2) => { assert!(bits == BitSize::DQWORD); let virt_address_local = ctx.builder.tee_new_local(); ctx.builder.get_local_i64(local1); ctx.builder.store_unaligned_i64(0); ctx.builder.get_local(&virt_address_local); ctx.builder.get_local_i64(local2); ctx.builder.store_unaligned_i64(8); ctx.builder.free_local(virt_address_local); }, } match bits { BitSize::BYTE => { ctx.builder.store_u8(0); }, BitSize::WORD => { ctx.builder.store_unaligned_u16(0); }, BitSize::DWORD => { ctx.builder.store_unaligned_i32(0); }, BitSize::QWORD => { ctx.builder.store_unaligned_i64(0); }, BitSize::DQWORD => {}, // handled above } ctx.builder.free_local(entry_local); } pub fn gen_safe_read_write( ctx: &mut JitContext, bits: BitSize, address_local: &WasmLocal, f: &dyn Fn(&mut JitContext), ) { // Execute a virtual memory read+write. All slow paths (memory-mapped IO, tlb miss, page fault, // write across page boundary and page containing jitted code are handled in // safe_read_write_jit_slow // entry <- tlb_data[addr >> 12 << 2] // can_use_fast_path <- entry & MASK == TLB_VALID && (addr & 0xFFF) <= 0x1000 - bytes // if can_use_fast_path: goto fast // entry <- safe_read_write_jit_slow(addr, instruction_pointer) // if page_fault: goto exit-with-pagefault // fast: value <- f(mem[(entry & ~0xFFF) ^ addr]) // if !can_use_fast_path { safe_write_jit_slow(addr, value, instruction_pointer) } // mem[(entry & ~0xFFF) ^ addr] <- value let cont = ctx.builder.block_void(); ctx.builder.get_local(address_local); ctx.builder.const_i32(12); ctx.builder.shr_u_i32(); ctx.builder.const_i32(2); ctx.builder.shl_i32(); ctx.builder .load_aligned_i32(unsafe { &tlb_data[0] as *const i32 as u32 }); let entry_local = ctx.builder.tee_new_local(); ctx.builder .const_i32((0xFFF & !TLB_GLOBAL & !(if ctx.cpu.cpl3() { 0 } else { TLB_NO_USER })) as i32); ctx.builder.and_i32(); ctx.builder.const_i32(TLB_VALID as i32); ctx.builder.eq_i32(); if bits != BitSize::BYTE { ctx.builder.get_local(&address_local); ctx.builder.const_i32(0xFFF); ctx.builder.and_i32(); ctx.builder.const_i32(0x1000 - bits.bytes() as i32); ctx.builder.le_i32(); ctx.builder.and_i32(); } let can_use_fast_path_local = ctx.builder.tee_new_local(); ctx.builder.br_if(cont); if cfg!(feature = "profiler") { ctx.builder.get_local(&address_local); ctx.builder.get_local(&entry_local); ctx.builder.call_fn2("report_safe_read_write_jit_slow"); } ctx.builder.get_local(&address_local); ctx.builder .const_i32(ctx.start_of_current_instruction as i32 & 0xFFF); match bits { BitSize::BYTE => { ctx.builder.call_fn2_ret("safe_read_write8_slow_jit"); }, BitSize::WORD => { ctx.builder.call_fn2_ret("safe_read_write16_slow_jit"); }, BitSize::DWORD => { ctx.builder.call_fn2_ret("safe_read_write32s_slow_jit"); }, BitSize::QWORD => { ctx.builder.call_fn2_ret("safe_read_write64_slow_jit"); }, BitSize::DQWORD => { dbg_assert!(false); }, } ctx.builder.tee_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); if cfg!(feature = "profiler") { ctx.builder.if_void(); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); ctx.builder.block_end(); ctx.builder.get_local(&entry_local); ctx.builder.const_i32(1); ctx.builder.and_i32(); } ctx.builder.br_if(ctx.exit_with_fault_label); ctx.builder.block_end(); gen_profiler_stat_increment(ctx.builder, profiler::stat::SAFE_READ_WRITE_FAST); // XXX: Also slow ctx.builder.get_local(&entry_local); ctx.builder.const_i32(!0xFFF); ctx.builder.and_i32(); ctx.builder.get_local(&address_local); ctx.builder.xor_i32(); ctx.builder.free_local(entry_local); let phys_addr_local = ctx.builder.tee_new_local(); match bits { BitSize::BYTE => { ctx.builder.load_u8(0); }, BitSize::WORD => { ctx.builder.load_unaligned_u16(0); }, BitSize::DWORD => { ctx.builder.load_unaligned_i32(0); }, BitSize::QWORD => { ctx.builder.load_unaligned_i64(0); }, BitSize::DQWORD => assert!(false), // not used } // value is now on stack f(ctx); // TODO: Could get rid of this local by returning one from f let value_local = if bits == BitSize::QWORD { GenSafeReadWriteValue::I64(ctx.builder.set_new_local_i64()) } else { GenSafeReadWriteValue::I32(ctx.builder.set_new_local()) }; ctx.builder.get_local(&can_use_fast_path_local); ctx.builder.eqz_i32(); ctx.builder.if_void(); { ctx.builder.get_local(&address_local); match &value_local { GenSafeReadWriteValue::I32(l) => ctx.builder.get_local(l), GenSafeReadWriteValue::I64(l) => ctx.builder.get_local_i64(l), } ctx.builder .const_i32(ctx.start_of_current_instruction as i32); match bits { BitSize::BYTE => { ctx.builder.call_fn3_ret("safe_write8_slow_jit"); }, BitSize::WORD => { ctx.builder.call_fn3_ret("safe_write16_slow_jit"); }, BitSize::DWORD => { ctx.builder.call_fn3_ret("safe_write32_slow_jit"); }, BitSize::QWORD => { ctx.builder .call_fn3_i32_i64_i32_ret("safe_write64_slow_jit"); }, BitSize::DQWORD => { dbg_assert!(false); }, } if cfg!(debug_assertions) { ctx.builder.const_i32(1); ctx.builder.and_i32(); ctx.builder.if_void(); { // handled above ctx.builder.const_i32(match bits { BitSize::BYTE => 8, BitSize::WORD => 16, BitSize::DWORD => 32, BitSize::QWORD => 64, _ => { dbg_assert!(false); 0 }, }); ctx.builder.get_local(&address_local); ctx.builder.call_fn2("bug_gen_safe_read_write_page_fault"); } ctx.builder.block_end(); } else { ctx.builder.drop_(); } } ctx.builder.block_end(); ctx.builder.get_local(&phys_addr_local); match &value_local { GenSafeReadWriteValue::I32(l) => ctx.builder.get_local(l), GenSafeReadWriteValue::I64(l) => ctx.builder.get_local_i64(l), } match bits { BitSize::BYTE => { ctx.builder.store_u8(0); }, BitSize::WORD => { ctx.builder.store_unaligned_u16(0); }, BitSize::DWORD => { ctx.builder.store_unaligned_i32(0); }, BitSize::QWORD => { ctx.builder.store_unaligned_i64(0); }, BitSize::DQWORD => { dbg_assert!(false); }, } match value_local { GenSafeReadWriteValue::I32(l) => ctx.builder.free_local(l), GenSafeReadWriteValue::I64(l) => ctx.builder.free_local_i64(l), } ctx.builder.free_local(can_use_fast_path_local); ctx.builder.free_local(phys_addr_local); } #[cfg(debug_assertions)] #[no_mangle] pub fn bug_gen_safe_read_write_page_fault(bits: i32, addr: u32) { dbg_log!("bug: gen_safe_read_write_page_fault {} {:x}", bits, addr); dbg_assert!(false); } pub fn gen_jmp_rel16(builder: &mut WasmBuilder, rel16: u16) { let cs_offset_addr = global_pointers::get_seg_offset(regs::CS); builder.load_fixed_i32(cs_offset_addr); let local = builder.set_new_local(); // generate: // *instruction_pointer = cs_offset + ((*instruction_pointer - cs_offset + rel16) & 0xFFFF); { builder.const_i32(global_pointers::instruction_pointer as i32); gen_get_eip(builder); builder.get_local(&local); builder.sub_i32(); builder.const_i32(rel16 as i32); builder.add_i32(); builder.const_i32(0xFFFF); builder.and_i32(); builder.get_local(&local); builder.add_i32(); builder.store_aligned_i32(0); } builder.free_local(local); } pub fn gen_pop16_ss16(ctx: &mut JitContext) { // sp = segment_offsets[SS] + reg16[SP] (or just reg16[SP] if has_flat_segmentation) gen_get_reg16(ctx, regs::SP); if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } // result = safe_read16(sp) let address_local = ctx.builder.set_new_local(); gen_safe_read16(ctx, &address_local); ctx.builder.free_local(address_local); // reg16[SP] += 2; gen_get_reg16(ctx, regs::SP); ctx.builder.const_i32(2); ctx.builder.add_i32(); gen_set_reg16(ctx, regs::SP); // return value is already on stack } pub fn gen_pop16_ss32(ctx: &mut JitContext) { // esp = segment_offsets[SS] + reg32[ESP] (or just reg32[ESP] if has_flat_segmentation) gen_get_reg32(ctx, regs::ESP); if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } // result = safe_read16(esp) let address_local = ctx.builder.set_new_local(); gen_safe_read16(ctx, &address_local); ctx.builder.free_local(address_local); // reg32[ESP] += 2; gen_get_reg32(ctx, regs::ESP); ctx.builder.const_i32(2); ctx.builder.add_i32(); gen_set_reg32(ctx, regs::ESP); // return value is already on stack } pub fn gen_pop16(ctx: &mut JitContext) { if ctx.cpu.ssize_32() { gen_pop16_ss32(ctx); } else { gen_pop16_ss16(ctx); } } pub fn gen_pop32s_ss16(ctx: &mut JitContext) { // sp = reg16[SP] gen_get_reg16(ctx, regs::SP); // result = safe_read32s(segment_offsets[SS] + sp) (or just sp if has_flat_segmentation) if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } let address_local = ctx.builder.set_new_local(); gen_safe_read32(ctx, &address_local); ctx.builder.free_local(address_local); // reg16[SP] = sp + 4; gen_get_reg16(ctx, regs::SP); ctx.builder.const_i32(4); ctx.builder.add_i32(); gen_set_reg16(ctx, regs::SP); // return value is already on stack } pub fn gen_pop32s_ss32(ctx: &mut JitContext) { if !ctx.cpu.has_flat_segmentation() { gen_get_reg32(ctx, regs::ESP); gen_get_ss_offset(ctx); ctx.builder.add_i32(); let address_local = ctx.builder.set_new_local(); gen_safe_read32(ctx, &address_local); ctx.builder.free_local(address_local); } else { let reg = ctx.register_locals[regs::ESP as usize].unsafe_clone(); gen_safe_read32(ctx, ®); } gen_get_reg32(ctx, regs::ESP); ctx.builder.const_i32(4); ctx.builder.add_i32(); gen_set_reg32(ctx, regs::ESP); // return value is already on stack } pub fn gen_pop32s(ctx: &mut JitContext) { if ctx.cpu.ssize_32() { gen_pop32s_ss32(ctx); } else { gen_pop32s_ss16(ctx); } } pub fn gen_adjust_stack_reg(ctx: &mut JitContext, offset: u32) { if ctx.cpu.ssize_32() { gen_get_reg32(ctx, regs::ESP); ctx.builder.const_i32(offset as i32); ctx.builder.add_i32(); gen_set_reg32(ctx, regs::ESP); } else { gen_get_reg16(ctx, regs::SP); ctx.builder.const_i32(offset as i32); ctx.builder.add_i32(); gen_set_reg16(ctx, regs::SP); } } pub fn gen_leave(ctx: &mut JitContext, os32: bool) { // [e]bp = safe_read{16,32}([e]bp) if ctx.cpu.ssize_32() { gen_get_reg32(ctx, regs::EBP); } else { gen_get_reg16(ctx, regs::BP); } let old_vbp = ctx.builder.tee_new_local(); if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } if os32 { let address_local = ctx.builder.set_new_local(); gen_safe_read32(ctx, &address_local); ctx.builder.free_local(address_local); gen_set_reg32(ctx, regs::EBP); } else { let address_local = ctx.builder.set_new_local(); gen_safe_read16(ctx, &address_local); ctx.builder.free_local(address_local); gen_set_reg16(ctx, regs::BP); } // [e]sp = [e]bp + (os32 ? 4 : 2) if ctx.cpu.ssize_32() { ctx.builder.get_local(&old_vbp); ctx.builder.const_i32(if os32 { 4 } else { 2 }); ctx.builder.add_i32(); gen_set_reg32(ctx, regs::ESP); } else { ctx.builder.get_local(&old_vbp); ctx.builder.const_i32(if os32 { 4 } else { 2 }); ctx.builder.add_i32(); gen_set_reg16(ctx, regs::SP); } ctx.builder.free_local(old_vbp); } pub fn gen_task_switch_test(ctx: &mut JitContext) { // generate if(cr[0] & (CR0_EM | CR0_TS)) { task_switch_test_jit(); goto exit_with_fault; } let cr0_offset = global_pointers::get_creg_offset(0); dbg_assert!(regs::CR0_EM | regs::CR0_TS <= 0xFF); ctx.builder.load_fixed_u8(cr0_offset); ctx.builder.const_i32((regs::CR0_EM | regs::CR0_TS) as i32); ctx.builder.and_i32(); ctx.builder.if_void(); { gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); gen_fn1_const( ctx.builder, "task_switch_test_jit", ctx.start_of_current_instruction, ); ctx.builder.br(ctx.exit_with_fault_label); } ctx.builder.block_end(); } pub fn gen_task_switch_test_mmx(ctx: &mut JitContext) { // generate if(cr[0] & (CR0_EM | CR0_TS)) { task_switch_test_mmx_jit(); goto exit_with_fault; } let cr0_offset = global_pointers::get_creg_offset(0); dbg_assert!(regs::CR0_EM | regs::CR0_TS <= 0xFF); ctx.builder.load_fixed_u8(cr0_offset); ctx.builder.const_i32((regs::CR0_EM | regs::CR0_TS) as i32); ctx.builder.and_i32(); ctx.builder.if_void(); { gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); gen_fn1_const( ctx.builder, "task_switch_test_mmx_jit", ctx.start_of_current_instruction, ); ctx.builder.br(ctx.exit_with_fault_label); } ctx.builder.block_end(); } pub fn gen_push16(ctx: &mut JitContext, value_local: &WasmLocal) { if ctx.cpu.ssize_32() { gen_get_reg32(ctx, regs::ESP); } else { gen_get_reg16(ctx, regs::SP); }; ctx.builder.const_i32(2); ctx.builder.sub_i32(); let reg_updated_local = if !ctx.cpu.ssize_32() || !ctx.cpu.has_flat_segmentation() { let reg_updated_local = ctx.builder.tee_new_local(); if !ctx.cpu.ssize_32() { ctx.builder.const_i32(0xFFFF); ctx.builder.and_i32(); } if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } let sp_local = ctx.builder.set_new_local(); gen_safe_write16(ctx, &sp_local, &value_local); ctx.builder.free_local(sp_local); ctx.builder.get_local(®_updated_local); reg_updated_local } else { // short path: The address written to is equal to ESP/SP minus two let reg_updated_local = ctx.builder.tee_new_local(); gen_safe_write16(ctx, ®_updated_local, &value_local); reg_updated_local }; if ctx.cpu.ssize_32() { gen_set_reg32(ctx, regs::ESP); } else { gen_set_reg16(ctx, regs::SP); }; ctx.builder.free_local(reg_updated_local); } pub fn gen_push32(ctx: &mut JitContext, value_local: &WasmLocal) { if ctx.cpu.ssize_32() { gen_get_reg32(ctx, regs::ESP); } else { gen_get_reg16(ctx, regs::SP); }; ctx.builder.const_i32(4); ctx.builder.sub_i32(); let new_sp_local = if !ctx.cpu.ssize_32() || !ctx.cpu.has_flat_segmentation() { let new_sp_local = ctx.builder.tee_new_local(); if !ctx.cpu.ssize_32() { ctx.builder.const_i32(0xFFFF); ctx.builder.and_i32(); } if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } let sp_local = ctx.builder.set_new_local(); gen_safe_write32(ctx, &sp_local, &value_local); ctx.builder.free_local(sp_local); ctx.builder.get_local(&new_sp_local); new_sp_local } else { // short path: The address written to is equal to ESP/SP minus four let new_sp_local = ctx.builder.tee_new_local(); gen_safe_write32(ctx, &new_sp_local, &value_local); new_sp_local }; if ctx.cpu.ssize_32() { gen_set_reg32(ctx, regs::ESP); } else { gen_set_reg16(ctx, regs::SP); }; ctx.builder.free_local(new_sp_local); } pub fn gen_push32_sreg(ctx: &mut JitContext, reg: u32) { gen_get_sreg(ctx, reg); let value_local = ctx.builder.set_new_local(); if ctx.cpu.ssize_32() { gen_get_reg32(ctx, regs::ESP); } else { gen_get_reg16(ctx, regs::SP); }; ctx.builder.const_i32(4); ctx.builder.sub_i32(); let new_sp_local = if !ctx.cpu.ssize_32() || !ctx.cpu.has_flat_segmentation() { let new_sp_local = ctx.builder.tee_new_local(); if !ctx.cpu.ssize_32() { ctx.builder.const_i32(0xFFFF); ctx.builder.and_i32(); } if !ctx.cpu.has_flat_segmentation() { gen_get_ss_offset(ctx); ctx.builder.add_i32(); } let sp_local = ctx.builder.set_new_local(); gen_safe_write16(ctx, &sp_local, &value_local); ctx.builder.free_local(sp_local); ctx.builder.get_local(&new_sp_local); new_sp_local } else { // short path: The address written to is equal to ESP/SP minus four let new_sp_local = ctx.builder.tee_new_local(); gen_safe_write16(ctx, &new_sp_local, &value_local); new_sp_local }; if ctx.cpu.ssize_32() { gen_set_reg32(ctx, regs::ESP); } else { gen_set_reg16(ctx, regs::SP); }; ctx.builder.free_local(new_sp_local); ctx.builder.free_local(value_local); } pub fn gen_get_real_eip(ctx: &mut JitContext) { gen_get_eip(ctx.builder); ctx.builder.const_i32(!0xFFF); ctx.builder.and_i32(); ctx.builder.const_i32(ctx.cpu.eip as i32 & 0xFFF); ctx.builder.or_i32(); if !ctx.cpu.has_flat_segmentation() { ctx.builder .load_fixed_i32(global_pointers::get_seg_offset(regs::CS)); ctx.builder.sub_i32(); } } pub fn gen_set_last_op1(builder: &mut WasmBuilder, source: &WasmLocal) { builder.const_i32(global_pointers::last_op1 as i32); builder.get_local(&source); builder.store_aligned_i32(0); } pub fn gen_set_last_result(builder: &mut WasmBuilder, source: &WasmLocal) { builder.const_i32(global_pointers::last_result as i32); builder.get_local(&source); builder.store_aligned_i32(0); } pub fn gen_clear_flags_changed_bits(builder: &mut WasmBuilder, bits_to_clear: i32) { builder.const_i32(global_pointers::flags_changed as i32); gen_get_flags_changed(builder); builder.const_i32(!bits_to_clear); builder.and_i32(); builder.store_aligned_i32(0); } pub fn gen_set_last_op_size_and_flags_changed( builder: &mut WasmBuilder, last_op_size: i32, flags_changed: i32, ) { dbg_assert!(last_op_size == OPSIZE_8 || last_op_size == OPSIZE_16 || last_op_size == OPSIZE_32); dbg_assert!(global_pointers::last_op_size as i32 % 8 == 0); dbg_assert!(global_pointers::last_op_size as i32 + 4 == global_pointers::flags_changed as i32); builder.const_i32(global_pointers::last_op_size as i32); builder.const_i64(last_op_size as u32 as i64 | (flags_changed as u32 as i64) << 32); builder.store_aligned_i64(0); } pub fn gen_set_flags_bits(builder: &mut WasmBuilder, bits_to_set: i32) { builder.const_i32(global_pointers::flags as i32); gen_get_flags(builder); builder.const_i32(bits_to_set); builder.or_i32(); builder.store_aligned_i32(0); } pub fn gen_clear_flags_bits(builder: &mut WasmBuilder, bits_to_clear: i32) { builder.const_i32(global_pointers::flags as i32); gen_get_flags(builder); builder.const_i32(!bits_to_clear); builder.and_i32(); builder.store_aligned_i32(0); } #[derive(PartialEq)] pub enum ConditionNegate { True, False, } pub fn gen_getzf(ctx: &mut JitContext, negate: ConditionNegate) { match &ctx.previous_instruction { Instruction::Cmp { dest: InstructionOperandDest::WasmLocal(dest), source: InstructionOperand::WasmLocal(source), opsize: OPSIZE_32, } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); ctx.builder.get_local(dest); ctx.builder.get_local(source); if negate == ConditionNegate::False { ctx.builder.eq_i32(); } else { ctx.builder.ne_i32(); } }, Instruction::Cmp { dest: InstructionOperandDest::WasmLocal(dest), source: InstructionOperand::Immediate(i), opsize: OPSIZE_32, } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); ctx.builder.get_local(dest); if *i != 0 { ctx.builder.const_i32(*i); if negate == ConditionNegate::False { ctx.builder.eq_i32(); } else { ctx.builder.ne_i32(); } } else { if negate == ConditionNegate::False { ctx.builder.eqz_i32(); } } }, Instruction::Cmp { .. } | Instruction::Sub { .. } | Instruction::Add { .. } | Instruction::AdcSbb { .. } | Instruction::NonZeroShift { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_result(ctx.builder, &ctx.previous_instruction); if negate == ConditionNegate::False { ctx.builder.eqz_i32(); } }, Instruction::Bitwise { opsize, .. } => { let &opsize = opsize; gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); // Note: Necessary because test{8,16} don't mask their neither last_result nor any of their operands // TODO: Use local instead of last_result for 8-bit/16-bit if opsize == OPSIZE_32 { gen_get_last_result(ctx.builder, &ctx.previous_instruction); } else if opsize == OPSIZE_16 { ctx.builder .load_fixed_u16(global_pointers::last_result as u32); } else if opsize == OPSIZE_8 { ctx.builder .load_fixed_u8(global_pointers::last_result as u32); } if negate == ConditionNegate::False { ctx.builder.eqz_i32(); } }, &Instruction::Other => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_get_flags_changed(ctx.builder); ctx.builder.const_i32(FLAG_ZERO); ctx.builder.and_i32(); ctx.builder.if_i32(); gen_get_last_result(ctx.builder, &ctx.previous_instruction); let last_result = ctx.builder.tee_new_local(); ctx.builder.const_i32(-1); ctx.builder.xor_i32(); ctx.builder.get_local(&last_result); ctx.builder.free_local(last_result); ctx.builder.const_i32(1); ctx.builder.sub_i32(); ctx.builder.and_i32(); gen_get_last_op_size(ctx.builder); ctx.builder.shr_u_i32(); ctx.builder.const_i32(1); ctx.builder.and_i32(); ctx.builder.else_(); gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_ZERO); ctx.builder.and_i32(); ctx.builder.block_end(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, } } pub fn gen_getcf(ctx: &mut JitContext, negate: ConditionNegate) { match &ctx.previous_instruction { Instruction::Cmp { source, opsize, .. } | Instruction::Sub { source, opsize, is_dec: false, .. } => { // Note: x < y and x < x - y can be used interchangeably (see getcf) gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); match (opsize, source) { (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l), (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(i), _ => gen_get_last_result(ctx.builder, &ctx.previous_instruction), } if negate == ConditionNegate::True { ctx.builder.geu_i32(); } else { ctx.builder.ltu_i32(); } }, Instruction::Add { source, opsize, is_inc: false, .. } => { gen_get_last_result(ctx.builder, &ctx.previous_instruction); match (opsize, source) { (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l), (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(i), _ => gen_get_last_op1(ctx.builder, &ctx.previous_instruction), } if negate == ConditionNegate::True { ctx.builder.geu_i32(); } else { ctx.builder.ltu_i32(); } }, Instruction::Add { is_inc: true, .. } | Instruction::Sub { is_dec: true, .. } => { gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_CARRY); ctx.builder.and_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, Instruction::Bitwise { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); ctx.builder .const_i32(if negate == ConditionNegate::True { 1 } else { 0 }); }, Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_CARRY); ctx.builder.and_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, &Instruction::Other => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_get_flags_changed(ctx.builder); let flags_changed = ctx.builder.tee_new_local(); ctx.builder.const_i32(FLAG_CARRY); ctx.builder.and_i32(); ctx.builder.if_i32(); ctx.builder.get_local(&flags_changed); ctx.builder.const_i32(31); ctx.builder.shr_s_i32(); ctx.builder.free_local(flags_changed); let sub_mask = ctx.builder.set_new_local(); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.get_local(&sub_mask); ctx.builder.xor_i32(); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); ctx.builder.get_local(&sub_mask); ctx.builder.xor_i32(); ctx.builder.ltu_i32(); ctx.builder.else_(); gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_CARRY); ctx.builder.and_i32(); ctx.builder.block_end(); ctx.builder.free_local(sub_mask); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, } } pub fn gen_getsf(ctx: &mut JitContext, negate: ConditionNegate) { match &ctx.previous_instruction { Instruction::Cmp { opsize, .. } | Instruction::Sub { opsize, .. } | Instruction::Add { opsize, .. } | Instruction::AdcSbb { opsize, .. } | Instruction::Bitwise { opsize, .. } | Instruction::NonZeroShift { opsize, .. } => { let &opsize = opsize; gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_result(ctx.builder, &ctx.previous_instruction); if opsize == OPSIZE_32 { ctx.builder.const_i32(0); if negate == ConditionNegate::True { ctx.builder.ge_i32(); } else { ctx.builder.lt_i32(); } } else { // TODO: use register (see get_last_result) ctx.builder .const_i32(if opsize == OPSIZE_16 { 0x8000 } else { 0x80 }); ctx.builder.and_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } } }, &Instruction::Other => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_get_flags_changed(ctx.builder); ctx.builder.const_i32(FLAG_SIGN); ctx.builder.and_i32(); ctx.builder.if_i32(); { gen_get_last_result(ctx.builder, &ctx.previous_instruction); gen_get_last_op_size(ctx.builder); ctx.builder.shr_u_i32(); ctx.builder.const_i32(1); ctx.builder.and_i32(); } ctx.builder.else_(); { gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_SIGN); ctx.builder.and_i32(); } ctx.builder.block_end(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, } } pub fn gen_getof(ctx: &mut JitContext) { match &ctx.previous_instruction { Instruction::Cmp { opsize, .. } | Instruction::Sub { opsize, .. } => { // TODO: a better formula might be possible let &opsize = opsize; gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.xor_i32(); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); ctx.builder.xor_i32(); ctx.builder.and_i32(); ctx.builder.const_i32(if opsize == OPSIZE_32 { 0x8000_0000u32 as i32 } else if opsize == OPSIZE_16 { 0x8000 } else { 0x80 }); ctx.builder.and_i32(); }, Instruction::Add { opsize, .. } => { // TODO: a better formula might be possible let &opsize = opsize; gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.xor_i32(); gen_get_last_result(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); ctx.builder.xor_i32(); ctx.builder.and_i32(); ctx.builder.const_i32(if opsize == OPSIZE_32 { 0x8000_0000u32 as i32 } else if opsize == OPSIZE_16 { 0x8000 } else { 0x80 }); ctx.builder.and_i32(); }, Instruction::Bitwise { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); ctx.builder.const_i32(0); }, Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_OVERFLOW); ctx.builder.and_i32(); }, &Instruction::Other => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_get_flags_changed(ctx.builder); let flags_changed = ctx.builder.tee_new_local(); ctx.builder.const_i32(FLAG_OVERFLOW); ctx.builder.and_i32(); ctx.builder.if_i32(); { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); let last_op1 = ctx.builder.tee_new_local(); gen_get_last_result(ctx.builder, &ctx.previous_instruction); let last_result = ctx.builder.tee_new_local(); ctx.builder.xor_i32(); ctx.builder.get_local(&last_result); ctx.builder.get_local(&last_op1); ctx.builder.sub_i32(); gen_get_flags_changed(ctx.builder); ctx.builder.const_i32(31); ctx.builder.shr_u_i32(); ctx.builder.sub_i32(); ctx.builder.get_local(&last_result); ctx.builder.xor_i32(); ctx.builder.and_i32(); gen_get_last_op_size(ctx.builder); ctx.builder.shr_u_i32(); ctx.builder.const_i32(1); ctx.builder.and_i32(); ctx.builder.free_local(last_op1); ctx.builder.free_local(last_result); } ctx.builder.else_(); { gen_get_flags(ctx.builder); ctx.builder.const_i32(FLAG_OVERFLOW); ctx.builder.and_i32(); } ctx.builder.block_end(); ctx.builder.free_local(flags_changed); }, } } pub fn gen_test_be(ctx: &mut JitContext, negate: ConditionNegate) { match &ctx.previous_instruction { Instruction::Cmp { dest, source, opsize, } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); match dest { InstructionOperandDest::WasmLocal(l) => { ctx.builder.get_local(l); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF }); ctx.builder.and_i32(); } }, InstructionOperandDest::Other => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); }, } match source { InstructionOperand::WasmLocal(l) => { ctx.builder.get_local(l); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF }); ctx.builder.and_i32(); } }, InstructionOperand::Other => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF }); ctx.builder.and_i32(); } }, &InstructionOperand::Immediate(i) => { dbg_assert!(*opsize != OPSIZE_8 || i >= 0 && i < 0x100); dbg_assert!(*opsize != OPSIZE_16 || i >= 0 && i < 0x10000); ctx.builder.const_i32(i); }, } if negate == ConditionNegate::True { ctx.builder.gtu_i32(); } else { ctx.builder.leu_i32(); } }, Instruction::Sub { opsize, source, is_dec: false, .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); match (opsize, source) { (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l), (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32(i), _ => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 0xFF } else { 0xFFFF }); ctx.builder.and_i32(); } }, } if negate == ConditionNegate::True { ctx.builder.gtu_i32(); } else { ctx.builder.leu_i32(); } }, &Instruction::Bitwise { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_getzf(ctx, negate); }, &Instruction::Add { .. } | &Instruction::Sub { is_dec: true, .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); // not the best code generation, but reasonable for this fairly uncommon case gen_getcf(ctx, ConditionNegate::False); gen_getzf(ctx, ConditionNegate::False); ctx.builder.or_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, Instruction::Other | Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_getcf(ctx, ConditionNegate::False); gen_getzf(ctx, ConditionNegate::False); ctx.builder.or_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, } } pub fn gen_test_l(ctx: &mut JitContext, negate: ConditionNegate) { match &ctx.previous_instruction { Instruction::Cmp { dest, source, opsize, } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); match dest { InstructionOperandDest::WasmLocal(l) => { ctx.builder.get_local(l); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, InstructionOperandDest::Other => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, } match source { InstructionOperand::WasmLocal(l) => { ctx.builder.get_local(l); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, InstructionOperand::Other => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, &InstructionOperand::Immediate(i) => { ctx.builder.const_i32(i); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, } if negate == ConditionNegate::True { ctx.builder.ge_i32(); } else { ctx.builder.lt_i32(); } }, Instruction::Sub { opsize, source, .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } match (opsize, source) { (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l), (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32( i << if *opsize == OPSIZE_32 { 0 } else if *opsize == OPSIZE_16 { 16 } else { 24 }, ), _ => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, } if negate == ConditionNegate::True { ctx.builder.ge_i32(); } else { ctx.builder.lt_i32(); } }, &Instruction::Bitwise { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_getsf(ctx, negate); }, &Instruction::Other | Instruction::Add { .. } | Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); if let Instruction::Add { .. } = ctx.previous_instruction { gen_profiler_stat_increment( ctx.builder, profiler::stat::CONDITION_UNOPTIMISED_UNHANDLED_L, ); } gen_getsf(ctx, ConditionNegate::False); ctx.builder.eqz_i32(); gen_getof(ctx); ctx.builder.eqz_i32(); ctx.builder.xor_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, } } pub fn gen_test_le(ctx: &mut JitContext, negate: ConditionNegate) { match &ctx.previous_instruction { Instruction::Cmp { dest, source, opsize, } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); match dest { InstructionOperandDest::WasmLocal(l) => { ctx.builder.get_local(l); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, InstructionOperandDest::Other => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, } match source { InstructionOperand::WasmLocal(l) => { ctx.builder.get_local(l); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, InstructionOperand::Other => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, &InstructionOperand::Immediate(i) => { ctx.builder.const_i32(i); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, } if negate == ConditionNegate::True { ctx.builder.gt_i32(); } else { ctx.builder.le_i32(); } }, Instruction::Sub { opsize, source, .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); gen_get_last_op1(ctx.builder, &ctx.previous_instruction); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } match (opsize, source) { (&OPSIZE_32, InstructionOperand::WasmLocal(l)) => ctx.builder.get_local(l), (_, &InstructionOperand::Immediate(i)) => ctx.builder.const_i32( i << if *opsize == OPSIZE_32 { 0 } else if *opsize == OPSIZE_16 { 16 } else { 24 }, ), _ => { gen_get_last_op1(ctx.builder, &ctx.previous_instruction); gen_get_last_result(ctx.builder, &ctx.previous_instruction); ctx.builder.sub_i32(); if *opsize == OPSIZE_8 || *opsize == OPSIZE_16 { ctx.builder .const_i32(if *opsize == OPSIZE_8 { 24 } else { 16 }); ctx.builder.shl_i32(); } }, } if negate == ConditionNegate::True { ctx.builder.gt_i32(); } else { ctx.builder.le_i32(); } }, &Instruction::Bitwise { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_OPTIMISED); // TODO: Could probably be improved (<= 0) gen_test_l(ctx, ConditionNegate::False); gen_getzf(ctx, ConditionNegate::False); ctx.builder.or_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, Instruction::Other | Instruction::Add { .. } | Instruction::NonZeroShift { .. } | Instruction::AdcSbb { .. } => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); if let Instruction::Add { .. } = ctx.previous_instruction { gen_profiler_stat_increment( ctx.builder, profiler::stat::CONDITION_UNOPTIMISED_UNHANDLED_LE, ); } gen_test_l(ctx, ConditionNegate::False); gen_getzf(ctx, ConditionNegate::False); ctx.builder.or_i32(); if negate == ConditionNegate::True { ctx.builder.eqz_i32(); } }, } } pub fn gen_test_loopnz(ctx: &mut JitContext, is_asize_32: bool) { gen_test_loop(ctx, is_asize_32); ctx.builder.eqz_i32(); gen_getzf(ctx, ConditionNegate::False); ctx.builder.or_i32(); ctx.builder.eqz_i32(); } pub fn gen_test_loopz(ctx: &mut JitContext, is_asize_32: bool) { gen_test_loop(ctx, is_asize_32); ctx.builder.eqz_i32(); gen_getzf(ctx, ConditionNegate::False); ctx.builder.eqz_i32(); ctx.builder.or_i32(); ctx.builder.eqz_i32(); } pub fn gen_test_loop(ctx: &mut JitContext, is_asize_32: bool) { if is_asize_32 { gen_get_reg32(ctx, regs::ECX); } else { gen_get_reg16(ctx, regs::CX); } } pub fn gen_test_jcxz(ctx: &mut JitContext, is_asize_32: bool) { if is_asize_32 { gen_get_reg32(ctx, regs::ECX); } else { gen_get_reg16(ctx, regs::CX); } ctx.builder.eqz_i32(); } pub fn gen_fpu_get_sti(ctx: &mut JitContext, i: u32) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); ctx.builder.const_i32(i as i32); ctx.builder.call_fn2("fpu_get_sti"); ctx.builder .load_fixed_i64(global_pointers::sse_scratch_register as u32); ctx.builder .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8); } pub fn gen_fpu_load_m32(ctx: &mut JitContext, modrm_byte: ModrmByte) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); gen_modrm_resolve_safe_read32(ctx, modrm_byte); ctx.builder.call_fn2("f32_to_f80"); ctx.builder .load_fixed_i64(global_pointers::sse_scratch_register as u32); ctx.builder .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8); } pub fn gen_fpu_load_m64(ctx: &mut JitContext, modrm_byte: ModrmByte) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); gen_modrm_resolve_safe_read64(ctx, modrm_byte); ctx.builder.call_fn2_i32_i64("f64_to_f80"); ctx.builder .load_fixed_i64(global_pointers::sse_scratch_register as u32); ctx.builder .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8); } pub fn gen_fpu_load_i16(ctx: &mut JitContext, modrm_byte: ModrmByte) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); gen_modrm_resolve_safe_read16(ctx, modrm_byte); sign_extend_i16(ctx.builder); ctx.builder.call_fn2("i32_to_f80"); ctx.builder .load_fixed_i64(global_pointers::sse_scratch_register as u32); ctx.builder .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8); } pub fn gen_fpu_load_i32(ctx: &mut JitContext, modrm_byte: ModrmByte) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); gen_modrm_resolve_safe_read32(ctx, modrm_byte); ctx.builder.call_fn2("i32_to_f80"); ctx.builder .load_fixed_i64(global_pointers::sse_scratch_register as u32); ctx.builder .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8); } pub fn gen_fpu_load_i64(ctx: &mut JitContext, modrm_byte: ModrmByte) { ctx.builder .const_i32(global_pointers::sse_scratch_register as i32); gen_modrm_resolve_safe_read64(ctx, modrm_byte); ctx.builder.call_fn2_i32_i64("i64_to_f80"); ctx.builder .load_fixed_i64(global_pointers::sse_scratch_register as u32); ctx.builder .load_fixed_u16(global_pointers::sse_scratch_register as u32 + 8); } pub fn gen_trigger_de(ctx: &mut JitContext) { gen_fn1_const( ctx.builder, "trigger_de_jit", ctx.start_of_current_instruction, ); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); ctx.builder.br(ctx.exit_with_fault_label); } pub fn gen_trigger_ud(ctx: &mut JitContext) { gen_fn1_const( ctx.builder, "trigger_ud_jit", ctx.start_of_current_instruction, ); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); ctx.builder.br(ctx.exit_with_fault_label); } pub fn gen_trigger_gp(ctx: &mut JitContext, error_code: u32) { gen_fn2_const( ctx.builder, "trigger_gp_jit", error_code, ctx.start_of_current_instruction, ); gen_debug_track_jit_exit(ctx.builder, ctx.start_of_current_instruction); ctx.builder.br(ctx.exit_with_fault_label); } pub fn gen_condition_fn_negated(ctx: &mut JitContext, condition: u8) { gen_condition_fn(ctx, condition ^ 1) } pub fn gen_condition_fn(ctx: &mut JitContext, condition: u8) { if condition & 0xF0 == 0x00 || condition & 0xF0 == 0x70 || condition & 0xF0 == 0x80 { match condition & 0xF { 0x0 => { gen_getof(ctx); }, 0x1 => { gen_getof(ctx); ctx.builder.eqz_i32(); }, 0x2 => { gen_getcf(ctx, ConditionNegate::False); }, 0x3 => { gen_getcf(ctx, ConditionNegate::True); }, 0x4 => { gen_getzf(ctx, ConditionNegate::False); }, 0x5 => { gen_getzf(ctx, ConditionNegate::True); }, 0x6 => { gen_test_be(ctx, ConditionNegate::False); }, 0x7 => { gen_test_be(ctx, ConditionNegate::True); }, 0x8 => { gen_getsf(ctx, ConditionNegate::False); }, 0x9 => { gen_getsf(ctx, ConditionNegate::True); }, 0xA => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED_PF); ctx.builder.call_fn0_ret("test_p"); }, 0xB => { gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED); gen_profiler_stat_increment(ctx.builder, profiler::stat::CONDITION_UNOPTIMISED_PF); ctx.builder.call_fn0_ret("test_np"); }, 0xC => { gen_test_l(ctx, ConditionNegate::False); }, 0xD => { gen_test_l(ctx, ConditionNegate::True); }, 0xE => { gen_test_le(ctx, ConditionNegate::False); }, 0xF => { gen_test_le(ctx, ConditionNegate::True); }, _ => { dbg_assert!(false); }, } } else { // loop, loopnz, loopz, jcxz dbg_assert!(condition & !0x3 == 0xE0); if condition == 0xE0 { gen_test_loopnz(ctx, ctx.cpu.asize_32()); } else if condition == 0xE1 { gen_test_loopz(ctx, ctx.cpu.asize_32()); } else if condition == 0xE2 { gen_test_loop(ctx, ctx.cpu.asize_32()); } else if condition == 0xE3 { gen_test_jcxz(ctx, ctx.cpu.asize_32()); } } } pub fn gen_move_registers_from_locals_to_memory(ctx: &mut JitContext) { if cfg!(feature = "profiler") { let instruction = memory::read32s(ctx.start_of_current_instruction) as u32; ::opstats::gen_opstat_unguarded_register(ctx.builder, instruction); } for i in 0..8 { ctx.builder .const_i32(global_pointers::get_reg32_offset(i as u32) as i32); ctx.builder.get_local(&ctx.register_locals[i]); ctx.builder.store_aligned_i32(0); } } pub fn gen_move_registers_from_memory_to_locals(ctx: &mut JitContext) { if cfg!(feature = "profiler") { let instruction = memory::read32s(ctx.start_of_current_instruction) as u32; ::opstats::gen_opstat_unguarded_register(ctx.builder, instruction); } for i in 0..8 { ctx.builder .const_i32(global_pointers::get_reg32_offset(i as u32) as i32); ctx.builder.load_aligned_i32(0); ctx.builder.set_local(&ctx.register_locals[i]); } } pub fn gen_profiler_stat_increment(builder: &mut WasmBuilder, stat: profiler::stat) { if !cfg!(feature = "profiler") { return; } let addr = unsafe { profiler::stat_array.as_mut_ptr().offset(stat as isize) } as u32; builder.increment_fixed_i64(addr, 1) } pub fn gen_debug_track_jit_exit(builder: &mut WasmBuilder, address: u32) { if cfg!(feature = "profiler") { gen_fn1_const(builder, "track_jit_exit", address); } }