123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379 |
- use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
- use std::iter::FromIterator;
- use std::mem;
- use std::ptr::NonNull;
- use analysis::AnalysisType;
- use codegen;
- use control_flow;
- use control_flow::WasmStructure;
- use cpu::cpu;
- use cpu::global_pointers;
- use cpu::memory;
- use cpu_context::CpuContext;
- use jit_instructions;
- use opstats;
- use page::Page;
- use profiler;
- use profiler::stat;
- use state_flags::CachedStateFlags;
- use util::SafeToU16;
- use wasmgen::wasm_builder::{Label, WasmBuilder, WasmLocal};
- #[derive(Copy, Clone, Eq, Hash, PartialEq)]
- #[repr(transparent)]
- pub struct WasmTableIndex(u16);
- impl WasmTableIndex {
- pub fn to_u16(self) -> u16 { self.0 }
- }
- mod unsafe_jit {
- use jit::{CachedStateFlags, WasmTableIndex};
- extern "C" {
- pub fn codegen_finalize(
- wasm_table_index: WasmTableIndex,
- phys_addr: u32,
- state_flags: CachedStateFlags,
- ptr: u32,
- len: u32,
- );
- pub fn jit_clear_func(wasm_table_index: WasmTableIndex);
- }
- }
- fn codegen_finalize(
- wasm_table_index: WasmTableIndex,
- phys_addr: u32,
- state_flags: CachedStateFlags,
- ptr: u32,
- len: u32,
- ) {
- unsafe { unsafe_jit::codegen_finalize(wasm_table_index, phys_addr, state_flags, ptr, len) }
- }
- pub fn jit_clear_func(wasm_table_index: WasmTableIndex) {
- unsafe { unsafe_jit::jit_clear_func(wasm_table_index) }
- }
- // less branches will generate if-else, more will generate brtable
- pub const BRTABLE_CUTOFF: usize = 10;
- pub const WASM_TABLE_SIZE: u32 = 900;
- pub const CHECK_JIT_STATE_INVARIANTS: bool = false;
- pub const JIT_USE_LOOP_SAFETY: bool = true;
- pub const JIT_THRESHOLD: u32 = 200 * 1000;
- pub const MAX_EXTRA_BASIC_BLOCKS: usize = 250;
- const MAX_INSTRUCTION_LENGTH: u32 = 16;
- #[allow(non_upper_case_globals)]
- static mut jit_state: NonNull<JitState> =
- unsafe { NonNull::new_unchecked(mem::align_of::<JitState>() as *mut _) };
- pub fn get_jit_state() -> &'static mut JitState { unsafe { jit_state.as_mut() } }
- #[no_mangle]
- pub fn rust_init() {
- dbg_assert!(std::mem::size_of::<[Option<NonNull<cpu::Code>>; 0x100000]>() == 0x100000 * 4);
- let x = Box::new(JitState::create_and_initialise());
- unsafe {
- jit_state = NonNull::new(Box::into_raw(x)).unwrap()
- }
- use std::panic;
- panic::set_hook(Box::new(|panic_info| {
- console_log!("{}", panic_info.to_string());
- }));
- }
- struct PageInfo {
- wasm_table_index: WasmTableIndex,
- hidden_wasm_table_indices: Vec<WasmTableIndex>,
- entry_points: Vec<(u16, u16)>,
- state_flags: CachedStateFlags,
- }
- enum CompilingPageState {
- Compiling { pages: HashMap<Page, PageInfo> },
- CompilingWritten,
- }
- pub struct JitState {
- wasm_builder: WasmBuilder,
- // as an alternative to HashSet, we could use a bitmap of 4096 bits here
- // (faster, but uses much more memory)
- // or a compressed bitmap (likely faster)
- // or HashSet<u32> rather than nested
- entry_points: HashMap<Page, (u32, HashSet<u16>)>,
- pages: HashMap<Page, PageInfo>,
- wasm_table_index_free_list: Vec<WasmTableIndex>,
- compiling: Option<(WasmTableIndex, CompilingPageState)>,
- }
- pub fn check_jit_state_invariants(ctx: &mut JitState) {
- if !CHECK_JIT_STATE_INVARIANTS {
- return;
- }
- match &ctx.compiling {
- Some((_, CompilingPageState::Compiling { pages })) => {
- dbg_assert!(pages.keys().all(|page| ctx.entry_points.contains_key(page)));
- },
- _ => {},
- }
- let free: HashSet<WasmTableIndex> =
- HashSet::from_iter(ctx.wasm_table_index_free_list.iter().cloned());
- let used = HashSet::from_iter(ctx.pages.values().map(|info| info.wasm_table_index));
- let compiling = HashSet::from_iter(ctx.compiling.as_ref().map(|&(index, _)| index));
- dbg_assert!(free.intersection(&used).next().is_none());
- dbg_assert!(used.intersection(&compiling).next().is_none());
- dbg_assert!(free.len() + used.len() + compiling.len() == (WASM_TABLE_SIZE - 1) as usize);
- match &ctx.compiling {
- Some((_, CompilingPageState::Compiling { pages })) => {
- dbg_assert!(pages.keys().all(|page| ctx.entry_points.contains_key(page)));
- },
- _ => {},
- }
- for i in 0..unsafe { cpu::valid_tlb_entries_count } {
- let page = unsafe { cpu::valid_tlb_entries[i as usize] };
- let entry = unsafe { cpu::tlb_data[page as usize] };
- if 0 != entry {
- let tlb_physical_page = Page::of_u32(
- (entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12),
- );
- let w = match unsafe { cpu::tlb_code[page as usize] } {
- None => None,
- Some(c) => unsafe {
- Some(c.as_ref().wasm_table_index)
- },
- };
- let tlb_has_code = entry & cpu::TLB_HAS_CODE == cpu::TLB_HAS_CODE;
- let infos = ctx.pages.get(&tlb_physical_page);
- let entry_points = ctx.entry_points.get(&tlb_physical_page);
- dbg_assert!(tlb_has_code || !w.is_some());
- dbg_assert!(tlb_has_code || !infos.is_some());
- dbg_assert!(tlb_has_code || !entry_points.is_some());
- //dbg_assert!((w.is_some() || page.is_some() || entry_points.is_some()) == tlb_has_code); // XXX: check this
- }
- }
- }
- impl JitState {
- pub fn create_and_initialise() -> JitState {
- // don't assign 0 (XXX: Check)
- let wasm_table_indices = (1..=(WASM_TABLE_SIZE - 1) as u16).map(|x| WasmTableIndex(x));
- JitState {
- wasm_builder: WasmBuilder::new(),
- entry_points: HashMap::new(),
- pages: HashMap::new(),
- wasm_table_index_free_list: Vec::from_iter(wasm_table_indices),
- compiling: None,
- }
- }
- }
- #[derive(PartialEq, Eq)]
- pub enum BasicBlockType {
- Normal {
- next_block_addr: Option<u32>,
- jump_offset: i32,
- jump_offset_is_32: bool,
- },
- ConditionalJump {
- next_block_addr: Option<u32>,
- next_block_branch_taken_addr: Option<u32>,
- condition: u8,
- jump_offset: i32,
- jump_offset_is_32: bool,
- },
- // Set eip to an absolute value (ret, jmp r/m, call r/m)
- AbsoluteEip,
- Exit,
- }
- pub struct BasicBlock {
- pub addr: u32,
- pub virt_addr: i32,
- pub last_instruction_addr: u32,
- pub end_addr: u32,
- pub is_entry_block: bool,
- pub ty: BasicBlockType,
- pub has_sti: bool,
- pub number_of_instructions: u32,
- }
- #[derive(Copy, Clone, PartialEq)]
- pub struct CachedCode {
- pub wasm_table_index: WasmTableIndex,
- pub initial_state: u16,
- }
- impl CachedCode {
- pub const NONE: CachedCode = CachedCode {
- wasm_table_index: WasmTableIndex(0),
- initial_state: 0,
- };
- }
- #[derive(PartialEq)]
- pub enum InstructionOperandDest {
- WasmLocal(WasmLocal),
- Other,
- }
- #[derive(PartialEq)]
- pub enum InstructionOperand {
- WasmLocal(WasmLocal),
- Immediate(i32),
- Other,
- }
- impl InstructionOperand {
- pub fn is_zero(&self) -> bool {
- match self {
- InstructionOperand::Immediate(0) => true,
- _ => false,
- }
- }
- }
- impl Into<InstructionOperand> for InstructionOperandDest {
- fn into(self: InstructionOperandDest) -> InstructionOperand {
- match self {
- InstructionOperandDest::WasmLocal(l) => InstructionOperand::WasmLocal(l),
- InstructionOperandDest::Other => InstructionOperand::Other,
- }
- }
- }
- pub enum Instruction {
- Cmp {
- dest: InstructionOperandDest,
- source: InstructionOperand,
- opsize: i32,
- },
- Sub {
- dest: InstructionOperandDest,
- source: InstructionOperand,
- opsize: i32,
- is_dec: bool,
- },
- Add {
- dest: InstructionOperandDest,
- source: InstructionOperand,
- opsize: i32,
- is_inc: bool,
- },
- AdcSbb {
- dest: InstructionOperandDest,
- source: InstructionOperand,
- opsize: i32,
- },
- NonZeroShift {
- dest: InstructionOperandDest,
- opsize: i32,
- },
- Bitwise {
- dest: InstructionOperandDest,
- opsize: i32,
- },
- Other,
- }
- pub struct JitContext<'a> {
- pub cpu: &'a mut CpuContext,
- pub builder: &'a mut WasmBuilder,
- pub register_locals: &'a mut Vec<WasmLocal>,
- pub start_of_current_instruction: u32,
- pub exit_with_fault_label: Label,
- pub exit_label: Label,
- pub current_instruction: Instruction,
- pub previous_instruction: Instruction,
- pub instruction_counter: WasmLocal,
- }
- impl<'a> JitContext<'a> {
- pub fn reg(&self, i: u32) -> WasmLocal { self.register_locals[i as usize].unsafe_clone() }
- }
- pub const JIT_INSTR_BLOCK_BOUNDARY_FLAG: u32 = 1 << 0;
- pub fn is_near_end_of_page(address: u32) -> bool {
- address & 0xFFF >= 0x1000 - MAX_INSTRUCTION_LENGTH
- }
- pub fn jit_find_cache_entry(phys_address: u32, state_flags: CachedStateFlags) -> CachedCode {
- // TODO: dedup with jit_find_cache_entry_in_page?
- // NOTE: This is currently only used for invariant/missed-entry-point checking
- let ctx = get_jit_state();
- match ctx.pages.get(&Page::page_of(phys_address)) {
- Some(PageInfo {
- wasm_table_index,
- state_flags: s,
- entry_points,
- hidden_wasm_table_indices: _,
- }) => {
- if *s == state_flags {
- let page_offset = phys_address as u16 & 0xFFF;
- if let Some(&(_, initial_state)) =
- entry_points.iter().find(|(p, _)| p == &page_offset)
- {
- return CachedCode {
- wasm_table_index: *wasm_table_index,
- initial_state,
- };
- }
- }
- },
- None => {},
- }
- return CachedCode::NONE;
- }
- #[no_mangle]
- pub fn jit_find_cache_entry_in_page(
- virt_address: u32,
- wasm_table_index: WasmTableIndex,
- state_flags: u32,
- ) -> i32 {
- // TODO: generate code for this
- profiler::stat_increment(stat::INDIRECT_JUMP);
- let state_flags = CachedStateFlags::of_u32(state_flags);
- unsafe {
- match cpu::tlb_code[(virt_address >> 12) as usize] {
- None => {},
- Some(c) => {
- let c = c.as_ref();
- if state_flags == c.state_flags && wasm_table_index == c.wasm_table_index {
- let state = c.state_table[virt_address as usize & 0xFFF];
- if state != u16::MAX {
- return state.into();
- }
- }
- },
- }
- }
- profiler::stat_increment(stat::INDIRECT_JUMP_NO_ENTRY);
- return -1;
- }
- // Maximum number of pages per wasm module. Necessary for the following reasons:
- // - There is an upper limit on the size of a single function in wasm (currently ~7MB in all browsers)
- // See https://github.com/WebAssembly/design/issues/1138
- // - v8 poorly handles large br_table elements and OOMs on modules much smaller than the above limit
- // See https://bugs.chromium.org/p/v8/issues/detail?id=9697 and https://bugs.chromium.org/p/v8/issues/detail?id=9141
- // Will hopefully be fixed in the near future by generating direct control flow
- const MAX_PAGES: usize = 3;
- fn jit_find_basic_blocks(
- ctx: &mut JitState,
- entry_points: HashSet<i32>,
- cpu: CpuContext,
- ) -> Vec<BasicBlock> {
- fn follow_jump(
- virt_target: i32,
- ctx: &mut JitState,
- pages: &mut HashSet<Page>,
- page_blacklist: &mut HashSet<Page>,
- max_pages: usize,
- marked_as_entry: &mut HashSet<i32>,
- to_visit_stack: &mut Vec<i32>,
- ) -> Option<u32> {
- if is_near_end_of_page(virt_target as u32) {
- return None;
- }
- let phys_target = match cpu::translate_address_read_no_side_effects(virt_target) {
- Err(()) => {
- dbg_log!("Not analysing {:x} (page not mapped)", virt_target);
- return None;
- },
- Ok(t) => t,
- };
- let phys_page = Page::page_of(phys_target);
- if !pages.contains(&phys_page) && pages.len() == max_pages
- || page_blacklist.contains(&phys_page)
- {
- return None;
- }
- if !pages.contains(&phys_page) {
- // page seen for the first time, handle entry points
- if let Some((hotness, entry_points)) = ctx.entry_points.get_mut(&phys_page) {
- let existing_entry_points = match ctx.pages.get(&phys_page) {
- Some(PageInfo { entry_points, .. }) => {
- HashSet::from_iter(entry_points.iter().map(|x| x.0))
- },
- None => HashSet::new(),
- };
- if entry_points
- .iter()
- .all(|entry_point| existing_entry_points.contains(entry_point))
- {
- page_blacklist.insert(phys_page);
- return None;
- }
- // XXX: Remove this paragraph
- //let old_length = entry_points.len();
- //entry_points.extend(existing_entry_points);
- //dbg_assert!(
- // entry_points.union(&existing_entry_points).count() == entry_points.len()
- //);
- *hotness = 0;
- for &addr_low in entry_points.iter() {
- let addr = virt_target & !0xFFF | addr_low as i32;
- to_visit_stack.push(addr);
- marked_as_entry.insert(addr);
- }
- }
- else {
- // no entry points: ignore this page?
- page_blacklist.insert(phys_page);
- return None;
- }
- pages.insert(phys_page);
- dbg_assert!(pages.len() <= max_pages);
- }
- to_visit_stack.push(virt_target);
- Some(phys_target)
- }
- let mut to_visit_stack: Vec<i32> = Vec::new();
- let mut marked_as_entry: HashSet<i32> = HashSet::new();
- let mut basic_blocks: BTreeMap<u32, BasicBlock> = BTreeMap::new();
- let mut pages: HashSet<Page> = HashSet::new();
- let mut page_blacklist = HashSet::new();
- // 16-bit doesn't not work correctly, most likely due to instruction pointer wrap-around
- let max_pages = if cpu.state_flags.is_32() { MAX_PAGES } else { 1 };
- for virt_addr in entry_points {
- let ok = follow_jump(
- virt_addr,
- ctx,
- &mut pages,
- &mut page_blacklist,
- max_pages,
- &mut marked_as_entry,
- &mut to_visit_stack,
- );
- dbg_assert!(ok.is_some());
- dbg_assert!(marked_as_entry.contains(&virt_addr));
- }
- while let Some(to_visit) = to_visit_stack.pop() {
- let phys_addr = match cpu::translate_address_read_no_side_effects(to_visit) {
- Err(()) => {
- dbg_log!("Not analysing {:x} (page not mapped)", to_visit);
- continue;
- },
- Ok(phys_addr) => phys_addr,
- };
- if basic_blocks.contains_key(&phys_addr) {
- continue;
- }
- if is_near_end_of_page(phys_addr) {
- // Empty basic block, don't insert
- profiler::stat_increment(stat::COMPILE_CUT_OFF_AT_END_OF_PAGE);
- continue;
- }
- let mut current_address = phys_addr;
- let mut current_block = BasicBlock {
- addr: current_address,
- virt_addr: to_visit,
- last_instruction_addr: 0,
- end_addr: 0,
- ty: BasicBlockType::Exit,
- is_entry_block: false,
- has_sti: false,
- number_of_instructions: 0,
- };
- loop {
- let addr_before_instruction = current_address;
- let mut cpu = &mut CpuContext {
- eip: current_address,
- ..cpu
- };
- let analysis = ::analysis::analyze_step(&mut cpu);
- current_block.number_of_instructions += 1;
- let has_next_instruction = !analysis.no_next_instruction;
- current_address = cpu.eip;
- dbg_assert!(Page::page_of(current_address) == Page::page_of(addr_before_instruction));
- let current_virt_addr = to_visit & !0xFFF | current_address as i32 & 0xFFF;
- match analysis.ty {
- AnalysisType::Normal | AnalysisType::STI => {
- dbg_assert!(has_next_instruction);
- dbg_assert!(!analysis.absolute_jump);
- if current_block.has_sti {
- // Convert next instruction after STI (i.e., the current instruction) into block boundary
- marked_as_entry.insert(current_virt_addr);
- to_visit_stack.push(current_virt_addr);
- current_block.last_instruction_addr = addr_before_instruction;
- current_block.end_addr = current_address;
- break;
- }
- if analysis.ty == AnalysisType::STI {
- current_block.has_sti = true;
- dbg_assert!(
- !is_near_end_of_page(current_address),
- "TODO: Handle STI instruction near end of page"
- );
- }
- else {
- // Only split non-STI blocks (one instruction needs to run after STI before
- // handle_irqs may be called)
- if basic_blocks.contains_key(¤t_address) {
- current_block.last_instruction_addr = addr_before_instruction;
- current_block.end_addr = current_address;
- dbg_assert!(!is_near_end_of_page(current_address));
- current_block.ty = BasicBlockType::Normal {
- next_block_addr: Some(current_address),
- jump_offset: 0,
- jump_offset_is_32: true,
- };
- break;
- }
- }
- },
- AnalysisType::Jump {
- offset,
- is_32,
- condition: Some(condition),
- } => {
- dbg_assert!(!analysis.absolute_jump);
- // conditional jump: continue at next and continue at jump target
- let jump_target = if is_32 {
- current_virt_addr + offset
- }
- else {
- cpu.cs_offset as i32
- + (current_virt_addr - cpu.cs_offset as i32 + offset & 0xFFFF)
- };
- dbg_assert!(has_next_instruction);
- to_visit_stack.push(current_virt_addr);
- let next_block_addr = if is_near_end_of_page(current_address) {
- None
- }
- else {
- Some(current_address)
- };
- current_block.ty = BasicBlockType::ConditionalJump {
- next_block_addr,
- next_block_branch_taken_addr: follow_jump(
- jump_target,
- ctx,
- &mut pages,
- &mut page_blacklist,
- max_pages,
- &mut marked_as_entry,
- &mut to_visit_stack,
- ),
- condition,
- jump_offset: offset,
- jump_offset_is_32: is_32,
- };
- current_block.last_instruction_addr = addr_before_instruction;
- current_block.end_addr = current_address;
- break;
- },
- AnalysisType::Jump {
- offset,
- is_32,
- condition: None,
- } => {
- dbg_assert!(!analysis.absolute_jump);
- // non-conditional jump: continue at jump target
- let jump_target = if is_32 {
- current_virt_addr + offset
- }
- else {
- cpu.cs_offset as i32
- + (current_virt_addr - cpu.cs_offset as i32 + offset & 0xFFFF)
- };
- if has_next_instruction {
- // Execution will eventually come back to the next instruction (CALL)
- marked_as_entry.insert(current_virt_addr);
- to_visit_stack.push(current_virt_addr);
- }
- current_block.ty = BasicBlockType::Normal {
- next_block_addr: follow_jump(
- jump_target,
- ctx,
- &mut pages,
- &mut page_blacklist,
- max_pages,
- &mut marked_as_entry,
- &mut to_visit_stack,
- ),
- jump_offset: offset,
- jump_offset_is_32: is_32,
- };
- current_block.last_instruction_addr = addr_before_instruction;
- current_block.end_addr = current_address;
- break;
- },
- AnalysisType::BlockBoundary => {
- // a block boundary but not a jump, get out
- if has_next_instruction {
- // block boundary, but execution will eventually come back
- // to the next instruction. Create a new basic block
- // starting at the next instruction and register it as an
- // entry point
- marked_as_entry.insert(current_virt_addr);
- to_visit_stack.push(current_virt_addr);
- }
- if analysis.absolute_jump {
- current_block.ty = BasicBlockType::AbsoluteEip;
- }
- current_block.last_instruction_addr = addr_before_instruction;
- current_block.end_addr = current_address;
- break;
- },
- }
- if is_near_end_of_page(current_address) {
- current_block.last_instruction_addr = addr_before_instruction;
- current_block.end_addr = current_address;
- profiler::stat_increment(stat::COMPILE_CUT_OFF_AT_END_OF_PAGE);
- break;
- }
- }
- let previous_block = basic_blocks
- .range(..current_block.addr)
- .next_back()
- .filter(|(_, previous_block)| (!previous_block.has_sti))
- .map(|(_, previous_block)| previous_block.clone());
- if let Some(previous_block) = previous_block {
- if current_block.addr < previous_block.end_addr {
- // If this block overlaps with the previous block, re-analyze the previous block
- to_visit_stack.push(previous_block.virt_addr);
- let addr = previous_block.addr;
- let old_block = basic_blocks.remove(&addr);
- dbg_assert!(old_block.is_some());
- // Note that this does not ensure the invariant that two consecutive blocks don't
- // overlay. For that, we also need to check the following block.
- }
- }
- dbg_assert!(current_block.addr < current_block.end_addr);
- dbg_assert!(current_block.addr <= current_block.last_instruction_addr);
- dbg_assert!(current_block.last_instruction_addr < current_block.end_addr);
- basic_blocks.insert(current_block.addr, current_block);
- }
- dbg_assert!(pages.len() <= max_pages);
- for block in basic_blocks.values_mut() {
- if marked_as_entry.contains(&block.virt_addr) {
- block.is_entry_block = true;
- }
- }
- let basic_blocks: Vec<BasicBlock> = basic_blocks.into_iter().map(|(_, block)| block).collect();
- for i in 0..basic_blocks.len() - 1 {
- let next_block_addr = basic_blocks[i + 1].addr;
- let next_block_end_addr = basic_blocks[i + 1].end_addr;
- let next_block_is_entry = basic_blocks[i + 1].is_entry_block;
- let block = &basic_blocks[i];
- dbg_assert!(block.addr < next_block_addr);
- if next_block_addr < block.end_addr {
- dbg_log!(
- "Overlapping first=[from={:x} to={:x} is_entry={}] second=[from={:x} to={:x} is_entry={}]",
- block.addr,
- block.end_addr,
- block.is_entry_block as u8,
- next_block_addr,
- next_block_end_addr,
- next_block_is_entry as u8
- );
- }
- }
- basic_blocks
- }
- #[no_mangle]
- #[cfg(debug_assertions)]
- pub fn jit_force_generate_unsafe(virt_addr: i32) {
- dbg_assert!(
- !is_near_end_of_page(virt_addr as u32),
- "cannot force compile near end of page"
- );
- jit_increase_hotness_and_maybe_compile(
- virt_addr,
- cpu::translate_address_read(virt_addr).unwrap(),
- cpu::get_seg_cs() as u32,
- cpu::get_state_flags(),
- JIT_THRESHOLD,
- );
- dbg_assert!(get_jit_state().compiling.is_some());
- }
- #[inline(never)]
- fn jit_analyze_and_generate(
- ctx: &mut JitState,
- virt_entry_point: i32,
- phys_entry_point: u32,
- cs_offset: u32,
- state_flags: CachedStateFlags,
- ) {
- let page = Page::page_of(phys_entry_point);
- dbg_assert!(ctx.compiling.is_none());
- let (_, entry_points) = match ctx.entry_points.get(&page) {
- None => return,
- Some(entry_points) => entry_points,
- };
- let existing_entry_points = match ctx.pages.get(&page) {
- Some(PageInfo { entry_points, .. }) => HashSet::from_iter(entry_points.iter().map(|x| x.0)),
- None => HashSet::new(),
- };
- if entry_points
- .iter()
- .all(|entry_point| existing_entry_points.contains(entry_point))
- {
- profiler::stat_increment(stat::COMPILE_SKIPPED_NO_NEW_ENTRY_POINTS);
- return;
- }
- // XXX: check and remove
- //let old_length = entry_points.len();
- //entry_points.extend(existing_entry_points);
- //dbg_log!(
- // "{} + {} = {}",
- // entry_points.len(),
- // existing_entry_points.len(),
- // entry_points.union(&existing_entry_points).count()
- //);
- //dbg_assert!(entry_points.union(&existing_entry_points).count() == entry_points.len());
- profiler::stat_increment(stat::COMPILE);
- let cpu = CpuContext {
- eip: 0,
- prefixes: 0,
- cs_offset,
- state_flags,
- };
- dbg_assert!(
- cpu::translate_address_read_no_side_effects(virt_entry_point).unwrap() == phys_entry_point
- );
- let virt_page = Page::page_of(virt_entry_point as u32);
- let entry_points: HashSet<i32> = entry_points
- .iter()
- .map(|e| virt_page.to_address() as i32 | *e as i32)
- .collect();
- let basic_blocks = jit_find_basic_blocks(ctx, entry_points, cpu.clone());
- let mut pages = HashSet::new();
- for b in basic_blocks.iter() {
- // Remove this assertion once page-crossing jit is enabled
- dbg_assert!(Page::page_of(b.addr) == Page::page_of(b.end_addr));
- pages.insert(Page::page_of(b.addr));
- }
- let print = false;
- for b in basic_blocks.iter() {
- if !print {
- break;
- }
- let last_instruction_opcode = memory::read32s(b.last_instruction_addr);
- let op = opstats::decode(last_instruction_opcode as u32);
- dbg_log!(
- "BB: 0x{:x} {}{:02x} {} {}",
- b.addr,
- if op.is_0f { "0f" } else { "" },
- op.opcode,
- if b.is_entry_block { "entry" } else { "noentry" },
- match &b.ty {
- BasicBlockType::ConditionalJump {
- next_block_addr: Some(next_block_addr),
- next_block_branch_taken_addr: Some(next_block_branch_taken_addr),
- ..
- } => format!(
- "0x{:x} 0x{:x}",
- next_block_addr, next_block_branch_taken_addr
- ),
- BasicBlockType::ConditionalJump {
- next_block_addr: None,
- next_block_branch_taken_addr: Some(next_block_branch_taken_addr),
- ..
- } => format!("0x{:x}", next_block_branch_taken_addr),
- BasicBlockType::ConditionalJump {
- next_block_addr: Some(next_block_addr),
- next_block_branch_taken_addr: None,
- ..
- } => format!("0x{:x}", next_block_addr),
- BasicBlockType::ConditionalJump {
- next_block_addr: None,
- next_block_branch_taken_addr: None,
- ..
- } => format!(""),
- BasicBlockType::Normal {
- next_block_addr: Some(next_block_addr),
- ..
- } => format!("0x{:x}", next_block_addr),
- BasicBlockType::Normal {
- next_block_addr: None,
- ..
- } => format!(""),
- BasicBlockType::Exit => format!(""),
- BasicBlockType::AbsoluteEip => format!(""),
- }
- );
- }
- let graph = control_flow::make_graph(&basic_blocks);
- let mut structure = control_flow::loopify(&graph);
- if print {
- dbg_log!("before blockify:");
- for group in &structure {
- dbg_log!("=> Group");
- group.print(0);
- }
- }
- control_flow::blockify(&mut structure, &graph);
- if cfg!(debug_assertions) {
- control_flow::assert_invariants(&structure);
- }
- if print {
- dbg_log!("after blockify:");
- for group in &structure {
- dbg_log!("=> Group");
- group.print(0);
- }
- }
- if ctx.wasm_table_index_free_list.is_empty() {
- dbg_log!("wasm_table_index_free_list empty, clearing cache");
- // When no free slots are available, delete all cached modules. We could increase the
- // size of the table, but this way the initial size acts as an upper bound for the
- // number of wasm modules that we generate, which we want anyway to avoid getting our
- // tab killed by browsers due to memory constraints.
- jit_clear_cache(ctx);
- profiler::stat_increment(stat::INVALIDATE_ALL_MODULES_NO_FREE_WASM_INDICES);
- dbg_log!(
- "after jit_clear_cache: {} free",
- ctx.wasm_table_index_free_list.len(),
- );
- // This assertion can fail if all entries are pending (not possible unless
- // WASM_TABLE_SIZE is set very low)
- dbg_assert!(!ctx.wasm_table_index_free_list.is_empty());
- }
- // allocate an index in the wasm table
- let wasm_table_index = ctx
- .wasm_table_index_free_list
- .pop()
- .expect("allocate wasm table index");
- dbg_assert!(wasm_table_index != WasmTableIndex(0));
- dbg_assert!(!pages.is_empty());
- dbg_assert!(pages.len() <= MAX_PAGES);
- let basic_block_by_addr: HashMap<u32, BasicBlock> =
- basic_blocks.into_iter().map(|b| (b.addr, b)).collect();
- let entries = jit_generate_module(
- structure,
- &basic_block_by_addr,
- cpu,
- &mut ctx.wasm_builder,
- wasm_table_index,
- state_flags,
- );
- dbg_assert!(!entries.is_empty());
- let mut page_info = HashMap::new();
- for &(addr, state) in &entries {
- let code = page_info
- .entry(Page::page_of(addr))
- .or_insert_with(|| PageInfo {
- wasm_table_index,
- state_flags,
- entry_points: Vec::new(),
- hidden_wasm_table_indices: Vec::new(),
- });
- code.entry_points.push((addr as u16 & 0xFFF, state));
- }
- profiler::stat_increment_by(
- stat::COMPILE_WASM_TOTAL_BYTES,
- ctx.wasm_builder.get_output_len() as u64,
- );
- profiler::stat_increment_by(stat::COMPILE_PAGE, pages.len() as u64);
- for &p in &pages {
- ctx.entry_points
- .entry(p)
- .or_insert_with(|| (0, HashSet::new()));
- }
- cpu::tlb_set_has_code_multiple(&pages, true);
- dbg_assert!(ctx.compiling.is_none());
- ctx.compiling = Some((
- wasm_table_index,
- CompilingPageState::Compiling { pages: page_info },
- ));
- let phys_addr = page.to_address();
- // will call codegen_finalize_finished asynchronously when finished
- codegen_finalize(
- wasm_table_index,
- phys_addr,
- state_flags,
- ctx.wasm_builder.get_output_ptr() as u32,
- ctx.wasm_builder.get_output_len(),
- );
- check_jit_state_invariants(ctx);
- }
- #[no_mangle]
- pub fn codegen_finalize_finished(
- wasm_table_index: WasmTableIndex,
- phys_addr: u32,
- state_flags: CachedStateFlags,
- ) {
- let ctx = get_jit_state();
- dbg_assert!(wasm_table_index != WasmTableIndex(0));
- dbg_log!(
- "Finished compiling for page at {:x}",
- Page::page_of(phys_addr).to_address()
- );
- let pages = match mem::replace(&mut ctx.compiling, None) {
- None => {
- dbg_assert!(false);
- return;
- },
- Some((in_progress_wasm_table_index, CompilingPageState::CompilingWritten)) => {
- dbg_assert!(wasm_table_index == in_progress_wasm_table_index);
- profiler::stat_increment(stat::INVALIDATE_MODULE_WRITTEN_WHILE_COMPILED);
- free_wasm_table_index(ctx, wasm_table_index);
- check_jit_state_invariants(ctx);
- return;
- },
- Some((in_progress_wasm_table_index, CompilingPageState::Compiling { pages })) => {
- dbg_assert!(wasm_table_index == in_progress_wasm_table_index);
- dbg_assert!(!pages.is_empty());
- pages
- },
- };
- for i in 0..unsafe { cpu::valid_tlb_entries_count } {
- let page = unsafe { cpu::valid_tlb_entries[i as usize] };
- let entry = unsafe { cpu::tlb_data[page as usize] };
- if 0 != entry {
- let tlb_physical_page = Page::of_u32(
- (entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12),
- );
- if let Some(info) = pages.get(&tlb_physical_page) {
- set_tlb_code(
- Page::of_u32(page as u32),
- wasm_table_index,
- &info.entry_points,
- state_flags,
- );
- }
- }
- }
- let mut check_for_unused_wasm_table_index = HashSet::new();
- for (page, mut info) in pages {
- if let Some(old_entry) = ctx.pages.remove(&page) {
- info.hidden_wasm_table_indices
- .extend(old_entry.hidden_wasm_table_indices);
- info.hidden_wasm_table_indices
- .push(old_entry.wasm_table_index);
- check_for_unused_wasm_table_index.insert(old_entry.wasm_table_index);
- }
- ctx.pages.insert(page, info);
- }
- let unused: Vec<&WasmTableIndex> = check_for_unused_wasm_table_index
- .iter()
- .filter(|&&i| ctx.pages.values().all(|page| page.wasm_table_index != i))
- .collect();
- for &index in unused {
- for p in ctx.pages.values_mut() {
- p.hidden_wasm_table_indices.retain(|&w| w != index);
- }
- dbg_log!("unused after overwrite {}", index.to_u16());
- profiler::stat_increment(stat::INVALIDATE_MODULE_UNUSED_AFTER_OVERWRITE);
- free_wasm_table_index(ctx, index);
- }
- check_jit_state_invariants(ctx);
- }
- pub fn update_tlb_code(virt_page: Page, phys_page: Page) {
- let ctx = get_jit_state();
- match ctx.pages.get(&phys_page) {
- Some(PageInfo {
- wasm_table_index,
- entry_points,
- state_flags,
- hidden_wasm_table_indices: _,
- }) => set_tlb_code(virt_page, *wasm_table_index, entry_points, *state_flags),
- None => cpu::clear_tlb_code(phys_page.to_u32() as i32),
- };
- }
- pub fn set_tlb_code(
- virt_page: Page,
- wasm_table_index: WasmTableIndex,
- entries: &Vec<(u16, u16)>,
- state_flags: CachedStateFlags,
- ) {
- let c = match unsafe { cpu::tlb_code[virt_page.to_u32() as usize] } {
- None => {
- let state_table = [u16::MAX; 0x1000];
- unsafe {
- let mut c = NonNull::new_unchecked(Box::into_raw(Box::new(cpu::Code {
- wasm_table_index,
- state_flags,
- state_table,
- })));
- cpu::tlb_code[virt_page.to_u32() as usize] = Some(c);
- c.as_mut()
- }
- },
- Some(mut c) => unsafe {
- let c = c.as_mut();
- c.state_table.fill(u16::MAX);
- c.state_flags = state_flags;
- c.wasm_table_index = wasm_table_index;
- c
- },
- };
- for &(addr, state) in entries {
- dbg_assert!(state != u16::MAX);
- c.state_table[addr as usize] = state;
- }
- }
- fn jit_generate_module(
- structure: Vec<WasmStructure>,
- basic_blocks: &HashMap<u32, BasicBlock>,
- mut cpu: CpuContext,
- builder: &mut WasmBuilder,
- wasm_table_index: WasmTableIndex,
- state_flags: CachedStateFlags,
- ) -> Vec<(u32, u16)> {
- builder.reset();
- let mut register_locals = (0..8)
- .map(|i| {
- builder.load_fixed_i32(global_pointers::get_reg32_offset(i));
- builder.set_new_local()
- })
- .collect();
- builder.const_i32(0);
- let instruction_counter = builder.set_new_local();
- let exit_label = builder.block_void();
- let exit_with_fault_label = builder.block_void();
- let main_loop_label = builder.loop_void();
- if JIT_USE_LOOP_SAFETY {
- builder.get_local(&instruction_counter);
- builder.const_i32(cpu::LOOP_COUNTER);
- builder.geu_i32();
- if cfg!(feature = "profiler") {
- builder.if_void();
- codegen::gen_debug_track_jit_exit(builder, 0);
- builder.br(exit_label);
- builder.block_end();
- }
- else {
- builder.br_if(exit_label);
- }
- }
- let brtable_default = builder.block_void();
- let ctx = &mut JitContext {
- cpu: &mut cpu,
- builder,
- register_locals: &mut register_locals,
- start_of_current_instruction: 0,
- exit_with_fault_label,
- exit_label,
- current_instruction: Instruction::Other,
- previous_instruction: Instruction::Other,
- instruction_counter,
- };
- let entry_blocks = {
- let mut nodes = &structure;
- let result;
- loop {
- match &nodes[0] {
- WasmStructure::Dispatcher(e) => {
- result = e.clone();
- break;
- },
- WasmStructure::Loop { .. } => {
- dbg_assert!(false);
- },
- WasmStructure::BasicBlock(_) => {
- dbg_assert!(false);
- },
- // Note: We could use these blocks as entry points, which will yield
- // more entries for free, but it requires adding those to the dispatcher
- // It's to be investigated if this yields a performance improvement
- // See also the comment at the bottom of this function when creating entry
- // points
- WasmStructure::Block(children) => {
- nodes = children;
- },
- }
- }
- result
- };
- let mut index_for_addr = HashMap::new();
- for (i, &addr) in entry_blocks.iter().enumerate() {
- index_for_addr.insert(addr, i as i32);
- }
- for b in basic_blocks.values() {
- if !index_for_addr.contains_key(&b.addr) {
- let i = index_for_addr.len();
- index_for_addr.insert(b.addr, i as i32);
- }
- }
- let mut label_for_addr: HashMap<u32, (Label, Option<i32>)> = HashMap::new();
- enum Work {
- WasmStructure(WasmStructure),
- BlockEnd {
- label: Label,
- targets: Vec<u32>,
- olds: HashMap<u32, (Label, Option<i32>)>,
- },
- LoopEnd {
- label: Label,
- entries: Vec<u32>,
- olds: HashMap<u32, (Label, Option<i32>)>,
- },
- }
- let mut work: VecDeque<Work> = structure
- .into_iter()
- .map(|x| Work::WasmStructure(x))
- .collect();
- while let Some(block) = work.pop_front() {
- let next_addr: Option<Vec<u32>> = work.iter().find_map(|x| match x {
- Work::WasmStructure(l) => Some(l.head().collect()),
- _ => None,
- });
- let target_block = &ctx.builder.arg_local_initial_state.unsafe_clone();
- match block {
- Work::WasmStructure(WasmStructure::BasicBlock(addr)) => {
- let block = basic_blocks.get(&addr).unwrap();
- jit_generate_basic_block(ctx, block);
- if block.has_sti {
- match block.ty {
- BasicBlockType::ConditionalJump {
- condition,
- jump_offset,
- jump_offset_is_32,
- ..
- } => {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_condition_fn(ctx, condition);
- ctx.builder.if_void();
- if jump_offset_is_32 {
- codegen::gen_relative_jump(ctx.builder, jump_offset);
- }
- else {
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- ctx.builder.block_end();
- },
- BasicBlockType::Normal {
- jump_offset,
- jump_offset_is_32,
- ..
- } => {
- if jump_offset_is_32 {
- codegen::gen_set_eip_low_bits_and_jump_rel32(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- jump_offset,
- );
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- },
- BasicBlockType::Exit => {},
- BasicBlockType::AbsoluteEip => {},
- };
- codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
- codegen::gen_move_registers_from_locals_to_memory(ctx);
- codegen::gen_fn0_const(ctx.builder, "handle_irqs");
- codegen::gen_update_instruction_counter(ctx);
- ctx.builder.return_();
- continue;
- }
- match &block.ty {
- BasicBlockType::Exit => {
- // Exit this function
- codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
- codegen::gen_profiler_stat_increment(ctx.builder, stat::DIRECT_EXIT);
- ctx.builder.br(ctx.exit_label);
- },
- BasicBlockType::AbsoluteEip => {
- // Check if we can stay in this module, if not exit
- codegen::gen_get_eip(ctx.builder);
- ctx.builder.const_i32(wasm_table_index.to_u16() as i32);
- ctx.builder.const_i32(state_flags.to_u32() as i32);
- ctx.builder.call_fn3_ret("jit_find_cache_entry_in_page");
- ctx.builder.tee_local(target_block);
- ctx.builder.const_i32(0);
- ctx.builder.ge_i32();
- // TODO: Could make this unconditional by including exit_label in the main br_table
- ctx.builder.br_if(main_loop_label);
- codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
- ctx.builder.br(ctx.exit_label);
- },
- &BasicBlockType::Normal {
- next_block_addr: None,
- jump_offset,
- jump_offset_is_32,
- } => {
- if jump_offset_is_32 {
- codegen::gen_set_eip_low_bits_and_jump_rel32(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- jump_offset,
- );
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
- codegen::gen_profiler_stat_increment(ctx.builder, stat::DIRECT_EXIT);
- ctx.builder.br(ctx.exit_label);
- },
- &BasicBlockType::Normal {
- next_block_addr: Some(next_block_addr),
- jump_offset,
- jump_offset_is_32,
- } => {
- // Unconditional jump to next basic block
- // - All instructions that don't change eip
- // - Unconditional jumps
- if Page::page_of(next_block_addr) != Page::page_of(block.addr) {
- if jump_offset_is_32 {
- codegen::gen_set_eip_low_bits_and_jump_rel32(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- jump_offset,
- );
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::NORMAL_PAGE_CHANGE,
- );
- codegen::gen_page_switch_check(
- ctx,
- next_block_addr,
- block.last_instruction_addr,
- );
- #[cfg(debug_assertions)]
- codegen::gen_fn2_const(
- ctx.builder,
- "check_page_switch",
- block.addr,
- next_block_addr,
- );
- }
- if next_addr
- .as_ref()
- .map_or(false, |n| n.contains(&next_block_addr))
- {
- // Blocks are consecutive
- if next_addr.unwrap().len() > 1 {
- let target_index = *index_for_addr.get(&next_block_addr).unwrap();
- if cfg!(feature = "profiler") {
- ctx.builder.const_i32(target_index);
- ctx.builder.call_fn1("debug_set_dispatcher_target");
- }
- ctx.builder.const_i32(target_index);
- ctx.builder.set_local(target_block);
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::NORMAL_FALLTHRU_WITH_TARGET_BLOCK,
- );
- }
- else {
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::NORMAL_FALLTHRU,
- );
- }
- }
- else {
- let &(br, target_index) = label_for_addr.get(&next_block_addr).unwrap();
- if let Some(target_index) = target_index {
- if cfg!(feature = "profiler") {
- ctx.builder.const_i32(target_index);
- ctx.builder.call_fn1("debug_set_dispatcher_target");
- }
- ctx.builder.const_i32(target_index);
- ctx.builder.set_local(target_block);
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::NORMAL_BRANCH_WITH_TARGET_BLOCK,
- );
- }
- else {
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::NORMAL_BRANCH,
- );
- }
- ctx.builder.br(br);
- }
- },
- &BasicBlockType::ConditionalJump {
- next_block_addr,
- next_block_branch_taken_addr,
- condition,
- jump_offset,
- jump_offset_is_32,
- } => {
- // Conditional jump to next basic block
- // - jnz, jc, loop, jcxz, etc.
- // Generate:
- // (1) condition()
- // (2) br_if()
- // (3) br()
- // Except:
- // If we need to update eip in case (2), it's replaced by if { update_eip(); br() }
- // If case (3) can fall through to the next basic block, the branch is eliminated
- // Dispatcher target writes can be generated in either case
- // Condition may be inverted if it helps generate a fallthrough instead of the second branch
- codegen::gen_profiler_stat_increment(ctx.builder, stat::CONDITIONAL_JUMP);
- #[derive(PartialEq)]
- enum Case {
- BranchTaken,
- BranchNotTaken,
- }
- let mut handle_case = |case: Case, is_first| {
- // first case generates condition and *has* to branch away,
- // second case branches unconditionally or falls through
- if is_first {
- if case == Case::BranchNotTaken {
- codegen::gen_condition_fn_negated(ctx, condition);
- }
- else {
- codegen::gen_condition_fn(ctx, condition);
- }
- }
- let next_block_addr = if case == Case::BranchTaken {
- next_block_branch_taken_addr
- }
- else {
- next_block_addr
- };
- if let Some(next_block_addr) = next_block_addr {
- if Page::page_of(next_block_addr) != Page::page_of(block.addr) {
- dbg_assert!(case == Case::BranchTaken); // currently not possible in other case
- if is_first {
- ctx.builder.if_i32();
- }
- if jump_offset_is_32 {
- codegen::gen_set_eip_low_bits_and_jump_rel32(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- jump_offset,
- );
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::CONDITIONAL_JUMP_PAGE_CHANGE,
- );
- codegen::gen_page_switch_check(
- ctx,
- next_block_addr,
- block.last_instruction_addr,
- );
- #[cfg(debug_assertions)]
- codegen::gen_fn2_const(
- ctx.builder,
- "check_page_switch",
- block.addr,
- next_block_addr,
- );
- if is_first {
- ctx.builder.const_i32(1);
- ctx.builder.else_();
- ctx.builder.const_i32(0);
- ctx.builder.block_end();
- }
- }
- if next_addr
- .as_ref()
- .map_or(false, |n| n.contains(&next_block_addr))
- {
- // blocks are consecutive
- // fallthrough, has to be second
- dbg_assert!(!is_first);
- if next_addr.as_ref().unwrap().len() > 1 {
- let target_index =
- *index_for_addr.get(&next_block_addr).unwrap();
- if cfg!(feature = "profiler") {
- ctx.builder.const_i32(target_index);
- ctx.builder.call_fn1("debug_set_dispatcher_target");
- }
- ctx.builder.const_i32(target_index);
- ctx.builder.set_local(target_block);
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::CONDITIONAL_JUMP_FALLTHRU_WITH_TARGET_BLOCK,
- );
- }
- else {
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::CONDITIONAL_JUMP_FALLTHRU,
- );
- }
- }
- else {
- let &(br, target_index) =
- label_for_addr.get(&next_block_addr).unwrap();
- if let Some(target_index) = target_index {
- if cfg!(feature = "profiler") {
- // Note: Currently called unconditionally, even if the
- // br_if below doesn't branch
- ctx.builder.const_i32(target_index);
- ctx.builder.call_fn1("debug_set_dispatcher_target");
- }
- ctx.builder.const_i32(target_index);
- ctx.builder.set_local(target_block);
- }
- if is_first {
- if cfg!(feature = "profiler") {
- ctx.builder.if_void();
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- if target_index.is_some() {
- stat::CONDITIONAL_JUMP_BRANCH_WITH_TARGET_BLOCK
- }
- else {
- stat::CONDITIONAL_JUMP_BRANCH
- },
- );
- ctx.builder.br(br);
- ctx.builder.block_end();
- }
- else {
- ctx.builder.br_if(br);
- }
- }
- else {
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- if target_index.is_some() {
- stat::CONDITIONAL_JUMP_BRANCH_WITH_TARGET_BLOCK
- }
- else {
- stat::CONDITIONAL_JUMP_BRANCH
- },
- );
- ctx.builder.br(br);
- }
- }
- }
- else {
- // target is outside of this module, update eip and exit
- if is_first {
- ctx.builder.if_void();
- }
- if case == Case::BranchTaken {
- if jump_offset_is_32 {
- codegen::gen_set_eip_low_bits_and_jump_rel32(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- jump_offset,
- );
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- }
- codegen::gen_debug_track_jit_exit(
- ctx.builder,
- block.last_instruction_addr,
- );
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::CONDITIONAL_JUMP_EXIT,
- );
- ctx.builder.br(ctx.exit_label);
- if is_first {
- ctx.builder.block_end();
- }
- }
- };
- let branch_taken_is_fallthrough = next_block_branch_taken_addr
- .map_or(false, |addr| {
- next_addr.as_ref().map_or(false, |n| n.contains(&addr))
- });
- let branch_not_taken_is_fallthrough = next_block_addr
- .map_or(false, |addr| {
- next_addr.as_ref().map_or(false, |n| n.contains(&addr))
- });
- if branch_not_taken_is_fallthrough && branch_taken_is_fallthrough {
- let next_block_addr = next_block_addr.unwrap();
- let next_block_branch_taken_addr =
- next_block_branch_taken_addr.unwrap();
- dbg_assert!(
- Page::page_of(next_block_addr) == Page::page_of(block.addr)
- ); // currently not possible
- if Page::page_of(next_block_branch_taken_addr)
- != Page::page_of(block.addr)
- {
- if jump_offset_is_32 {
- codegen::gen_set_eip_low_bits_and_jump_rel32(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- jump_offset,
- );
- }
- else {
- codegen::gen_set_eip_low_bits(
- ctx.builder,
- block.end_addr as i32 & 0xFFF,
- );
- codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
- }
- codegen::gen_profiler_stat_increment(
- ctx.builder,
- stat::CONDITIONAL_JUMP_PAGE_CHANGE,
- );
- codegen::gen_page_switch_check(
- ctx,
- next_block_branch_taken_addr,
- block.last_instruction_addr,
- );
- #[cfg(debug_assertions)]
- codegen::gen_fn2_const(
- ctx.builder,
- "check_page_switch",
- block.addr,
- next_block_branch_taken_addr,
- );
- }
- if next_addr.unwrap().len() > 1 {
- let target_index_taken =
- *index_for_addr.get(&next_block_branch_taken_addr).unwrap();
- let target_index_not_taken =
- *index_for_addr.get(&next_block_addr).unwrap();
- codegen::gen_condition_fn(ctx, condition);
- ctx.builder.if_i32();
- ctx.builder.const_i32(target_index_taken);
- ctx.builder.else_();
- ctx.builder.const_i32(target_index_not_taken);
- ctx.builder.block_end();
- ctx.builder.set_local(target_block);
- }
- }
- else if branch_taken_is_fallthrough {
- handle_case(Case::BranchNotTaken, true);
- handle_case(Case::BranchTaken, false);
- }
- else {
- handle_case(Case::BranchTaken, true);
- handle_case(Case::BranchNotTaken, false);
- }
- },
- }
- },
- Work::WasmStructure(WasmStructure::Dispatcher(entries)) => {
- profiler::stat_increment(stat::COMPILE_DISPATCHER);
- if cfg!(feature = "profiler") {
- ctx.builder.get_local(target_block);
- ctx.builder.const_i32(index_for_addr.len() as i32);
- ctx.builder.call_fn2("check_dispatcher_target");
- }
- if entries.len() > BRTABLE_CUTOFF {
- // generate a brtable
- codegen::gen_profiler_stat_increment(ctx.builder, stat::DISPATCHER_LARGE);
- let mut cases = Vec::new();
- for &addr in &entries {
- let &(label, target_index) = label_for_addr.get(&addr).unwrap();
- let &index = index_for_addr.get(&addr).unwrap();
- dbg_assert!(target_index.is_none() || target_index == Some(index));
- while index as usize >= cases.len() {
- cases.push(brtable_default);
- }
- cases[index as usize] = label;
- }
- ctx.builder.get_local(target_block);
- ctx.builder.brtable(brtable_default, &mut cases.iter());
- }
- else {
- // generate a if target == block.addr then br block.label ...
- codegen::gen_profiler_stat_increment(ctx.builder, stat::DISPATCHER_SMALL);
- let nexts: HashSet<u32> = next_addr
- .as_ref()
- .map_or(HashSet::new(), |nexts| nexts.iter().copied().collect());
- for &addr in &entries {
- if nexts.contains(&addr) {
- continue;
- }
- let index = *index_for_addr.get(&addr).unwrap();
- let &(label, _) = label_for_addr.get(&addr).unwrap();
- ctx.builder.get_local(target_block);
- ctx.builder.const_i32(index);
- ctx.builder.eq_i32();
- ctx.builder.br_if(label);
- }
- }
- },
- Work::WasmStructure(WasmStructure::Loop(children)) => {
- profiler::stat_increment(stat::COMPILE_WASM_LOOP);
- let entries: Vec<u32> = children[0].head().collect();
- let label = ctx.builder.loop_void();
- codegen::gen_profiler_stat_increment(ctx.builder, stat::LOOP);
- if entries.len() == 1 {
- let addr = entries[0];
- codegen::gen_set_eip_low_bits(ctx.builder, addr as i32 & 0xFFF);
- profiler::stat_increment(stat::COMPILE_WITH_LOOP_SAFETY);
- codegen::gen_profiler_stat_increment(ctx.builder, stat::LOOP_SAFETY);
- if JIT_USE_LOOP_SAFETY {
- ctx.builder.get_local(&ctx.instruction_counter);
- ctx.builder.const_i32(cpu::LOOP_COUNTER);
- ctx.builder.geu_i32();
- if cfg!(feature = "profiler") {
- ctx.builder.if_void();
- codegen::gen_debug_track_jit_exit(ctx.builder, addr);
- ctx.builder.br(exit_label);
- ctx.builder.block_end();
- }
- else {
- ctx.builder.br_if(exit_label);
- }
- }
- }
- let mut olds = HashMap::new();
- for &target in entries.iter() {
- let index = if entries.len() == 1 {
- None
- }
- else {
- Some(*index_for_addr.get(&target).unwrap())
- };
- let old = label_for_addr.insert(target, (label, index));
- if let Some(old) = old {
- olds.insert(target, old);
- }
- }
- work.push_front(Work::LoopEnd {
- label,
- entries,
- olds,
- });
- for c in children.into_iter().rev() {
- work.push_front(Work::WasmStructure(c));
- }
- },
- Work::LoopEnd {
- label,
- entries,
- olds,
- } => {
- for target in entries {
- let old = label_for_addr.remove(&target);
- dbg_assert!(old.map(|(l, _)| l) == Some(label));
- }
- for (target, old) in olds {
- let old = label_for_addr.insert(target, old);
- dbg_assert!(old.is_none());
- }
- ctx.builder.block_end();
- },
- Work::WasmStructure(WasmStructure::Block(children)) => {
- profiler::stat_increment(stat::COMPILE_WASM_BLOCK);
- let targets = next_addr.clone().unwrap();
- let label = ctx.builder.block_void();
- let mut olds = HashMap::new();
- for &target in targets.iter() {
- let index = if targets.len() == 1 {
- None
- }
- else {
- Some(*index_for_addr.get(&target).unwrap())
- };
- let old = label_for_addr.insert(target, (label, index));
- if let Some(old) = old {
- olds.insert(target, old);
- }
- }
- work.push_front(Work::BlockEnd {
- label,
- targets,
- olds,
- });
- for c in children.into_iter().rev() {
- work.push_front(Work::WasmStructure(c));
- }
- },
- Work::BlockEnd {
- label,
- targets,
- olds,
- } => {
- for target in targets {
- let old = label_for_addr.remove(&target);
- dbg_assert!(old.map(|(l, _)| l) == Some(label));
- }
- for (target, old) in olds {
- let old = label_for_addr.insert(target, old);
- dbg_assert!(old.is_none());
- }
- ctx.builder.block_end();
- },
- }
- }
- dbg_assert!(label_for_addr.is_empty());
- {
- ctx.builder.block_end(); // default case for the brtable
- ctx.builder.unreachable();
- }
- {
- ctx.builder.block_end(); // main loop
- }
- {
- // exit-with-fault case
- ctx.builder.block_end();
- codegen::gen_move_registers_from_locals_to_memory(ctx);
- codegen::gen_fn0_const(ctx.builder, "trigger_fault_end_jit");
- codegen::gen_update_instruction_counter(ctx);
- ctx.builder.return_();
- }
- {
- // exit
- ctx.builder.block_end();
- codegen::gen_move_registers_from_locals_to_memory(ctx);
- codegen::gen_update_instruction_counter(ctx);
- }
- for local in ctx.register_locals.drain(..) {
- ctx.builder.free_local(local);
- }
- ctx.builder
- .free_local(ctx.instruction_counter.unsafe_clone());
- ctx.builder.finish();
- let entries = Vec::from_iter(entry_blocks.iter().map(|addr| {
- let block = basic_blocks.get(&addr).unwrap();
- let index = *index_for_addr.get(&addr).unwrap();
- profiler::stat_increment(stat::COMPILE_ENTRY_POINT);
- dbg_assert!(block.addr < block.end_addr);
- // Note: We also insert blocks that weren't originally marked as entries here
- // This doesn't have any downside, besides making the hash table slightly larger
- let initial_state = index.safe_to_u16();
- (block.addr, initial_state)
- }));
- for b in basic_blocks.values() {
- if b.is_entry_block {
- dbg_assert!(entries.iter().find(|(addr, _)| *addr == b.addr).is_some());
- }
- }
- return entries;
- }
- fn jit_generate_basic_block(ctx: &mut JitContext, block: &BasicBlock) {
- let needs_eip_updated = match block.ty {
- BasicBlockType::Exit => true,
- _ => false,
- };
- profiler::stat_increment(stat::COMPILE_BASIC_BLOCK);
- let start_addr = block.addr;
- let last_instruction_addr = block.last_instruction_addr;
- let stop_addr = block.end_addr;
- // First iteration of do-while assumes the caller confirms this condition
- dbg_assert!(!is_near_end_of_page(start_addr));
- if cfg!(feature = "profiler") {
- ctx.builder.const_i32(start_addr as i32);
- ctx.builder.call_fn1("enter_basic_block");
- }
- ctx.builder.get_local(&ctx.instruction_counter);
- ctx.builder.const_i32(block.number_of_instructions as i32);
- ctx.builder.add_i32();
- ctx.builder.set_local(&ctx.instruction_counter);
- ctx.cpu.eip = start_addr;
- ctx.current_instruction = Instruction::Other;
- ctx.previous_instruction = Instruction::Other;
- loop {
- let mut instruction = 0;
- if cfg!(feature = "profiler") {
- instruction = memory::read32s(ctx.cpu.eip) as u32;
- opstats::gen_opstats(ctx.builder, instruction);
- opstats::record_opstat_compiled(instruction);
- }
- if ctx.cpu.eip == last_instruction_addr {
- // Before the last instruction:
- // - Set eip to *after* the instruction
- // - Set previous_eip to *before* the instruction
- if needs_eip_updated {
- codegen::gen_set_previous_eip_offset_from_eip_with_low_bits(
- ctx.builder,
- last_instruction_addr as i32 & 0xFFF,
- );
- codegen::gen_set_eip_low_bits(ctx.builder, stop_addr as i32 & 0xFFF);
- }
- }
- let wasm_length_before = ctx.builder.instruction_body_length();
- ctx.start_of_current_instruction = ctx.cpu.eip;
- let start_eip = ctx.cpu.eip;
- let mut instruction_flags = 0;
- jit_instructions::jit_instruction(ctx, &mut instruction_flags);
- let end_eip = ctx.cpu.eip;
- let instruction_length = end_eip - start_eip;
- let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;
- let wasm_length = ctx.builder.instruction_body_length() - wasm_length_before;
- opstats::record_opstat_size_wasm(instruction, wasm_length as u64);
- dbg_assert!((end_eip == stop_addr) == (start_eip == last_instruction_addr));
- dbg_assert!(instruction_length < MAX_INSTRUCTION_LENGTH);
- let end_addr = ctx.cpu.eip;
- if end_addr == stop_addr {
- // no page was crossed
- dbg_assert!(Page::page_of(end_addr) == Page::page_of(start_addr));
- break;
- }
- if was_block_boundary || is_near_end_of_page(end_addr) || end_addr > stop_addr {
- dbg_log!(
- "Overlapping basic blocks start={:x} expected_end={:x} end={:x} was_block_boundary={} near_end_of_page={}",
- start_addr,
- stop_addr,
- end_addr,
- was_block_boundary,
- is_near_end_of_page(end_addr)
- );
- dbg_assert!(false);
- break;
- }
- ctx.previous_instruction = mem::replace(&mut ctx.current_instruction, Instruction::Other);
- }
- }
- pub fn jit_increase_hotness_and_maybe_compile(
- virt_address: i32,
- phys_address: u32,
- cs_offset: u32,
- state_flags: CachedStateFlags,
- heat: u32,
- ) {
- let ctx = get_jit_state();
- let page = Page::page_of(phys_address);
- let (hotness, entry_points) = ctx.entry_points.entry(page).or_insert_with(|| {
- cpu::tlb_set_has_code(page, true);
- profiler::stat_increment(stat::RUN_INTERPRETED_NEW_PAGE);
- (0, HashSet::new())
- });
- if !is_near_end_of_page(phys_address) {
- entry_points.insert(phys_address as u16 & 0xFFF);
- }
- *hotness += heat;
- if *hotness >= JIT_THRESHOLD {
- if ctx.compiling.is_some() {
- return;
- }
- // only try generating if we're in the correct address space
- if cpu::translate_address_read_no_side_effects(virt_address) == Ok(phys_address) {
- *hotness = 0;
- jit_analyze_and_generate(ctx, virt_address, phys_address, cs_offset, state_flags)
- }
- else {
- profiler::stat_increment(stat::COMPILE_WRONG_ADDRESS_SPACE);
- }
- };
- }
- fn free_wasm_table_index(ctx: &mut JitState, wasm_table_index: WasmTableIndex) {
- if CHECK_JIT_STATE_INVARIANTS {
- dbg_assert!(!ctx.wasm_table_index_free_list.contains(&wasm_table_index));
- match &ctx.compiling {
- Some((wasm_table_index_compiling, _)) => {
- dbg_assert!(
- *wasm_table_index_compiling != wasm_table_index,
- "Attempt to free wasm table index that is currently being compiled"
- );
- },
- _ => {},
- }
- dbg_assert!(
- !ctx.pages
- .values()
- .any(|info| info.wasm_table_index == wasm_table_index)
- );
- dbg_assert!(
- !ctx.pages
- .values()
- .any(|info| info.hidden_wasm_table_indices.contains(&wasm_table_index))
- );
- for i in 0..unsafe { cpu::valid_tlb_entries_count } {
- let page = unsafe { cpu::valid_tlb_entries[i as usize] };
- unsafe {
- match cpu::tlb_code[page as usize] {
- None => {},
- Some(c) => {
- let c = c.as_ref();
- dbg_assert!(c.wasm_table_index != wasm_table_index);
- },
- }
- }
- }
- }
- ctx.wasm_table_index_free_list.push(wasm_table_index);
- // It is not strictly necessary to clear the function, but it will fail more predictably if we
- // accidentally use the function and may garbage collect unused modules earlier
- jit_clear_func(wasm_table_index);
- }
- /// Register a write in this page: Delete all present code
- pub fn jit_dirty_page(ctx: &mut JitState, page: Page) {
- let mut did_have_code = false;
- if let Some(PageInfo {
- wasm_table_index,
- hidden_wasm_table_indices,
- state_flags: _,
- entry_points: _,
- }) = ctx.pages.remove(&page)
- {
- profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_CODE);
- did_have_code = true;
- free(ctx, wasm_table_index);
- for wasm_table_index in hidden_wasm_table_indices {
- free(ctx, wasm_table_index);
- }
- fn free(ctx: &mut JitState, wasm_table_index: WasmTableIndex) {
- for i in 0..unsafe { cpu::valid_tlb_entries_count } {
- let page = unsafe { cpu::valid_tlb_entries[i as usize] };
- let entry = unsafe { cpu::tlb_data[page as usize] };
- if 0 != entry {
- let tlb_physical_page = Page::of_u32(
- (entry as u32 >> 12 ^ page as u32) - (unsafe { memory::mem8 } as u32 >> 12),
- );
- match unsafe { cpu::tlb_code[page as usize] } {
- None => {},
- Some(c) => unsafe {
- let w = c.as_ref().wasm_table_index;
- if wasm_table_index == w {
- drop(Box::from_raw(c.as_ptr()));
- cpu::tlb_code[page as usize] = None;
- if !ctx.entry_points.contains_key(&tlb_physical_page) {
- cpu::tlb_data[page as usize] &= !cpu::TLB_HAS_CODE; // XXX
- }
- }
- },
- }
- }
- }
- ctx.pages.retain(
- |
- _,
- &mut PageInfo {
- wasm_table_index: w,
- ..
- },
- | w != wasm_table_index,
- );
- for info in ctx.pages.values_mut() {
- info.hidden_wasm_table_indices
- .retain(|&w| w != wasm_table_index)
- }
- free_wasm_table_index(ctx, wasm_table_index);
- }
- }
- match ctx.entry_points.remove(&page) {
- None => {},
- Some(_) => {
- profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_ENTRY_POINTS);
- did_have_code = true;
- match &ctx.compiling {
- Some((index, CompilingPageState::Compiling { pages })) => {
- if pages.contains_key(&page) {
- ctx.compiling = Some((*index, CompilingPageState::CompilingWritten));
- }
- },
- _ => {},
- }
- },
- }
- match &ctx.compiling {
- Some((_, CompilingPageState::Compiling { pages })) => {
- dbg_assert!(!pages.contains_key(&page));
- },
- _ => {},
- }
- check_jit_state_invariants(ctx);
- dbg_assert!(!jit_page_has_code_ctx(ctx, page));
- if did_have_code {
- cpu::tlb_set_has_code(page, false);
- }
- if !did_have_code {
- profiler::stat_increment(stat::DIRTY_PAGE_DID_NOT_HAVE_CODE);
- }
- }
- #[no_mangle]
- pub fn jit_dirty_cache(start_addr: u32, end_addr: u32) {
- dbg_assert!(start_addr < end_addr);
- let start_page = Page::page_of(start_addr);
- let end_page = Page::page_of(end_addr - 1);
- for page in start_page.to_u32()..end_page.to_u32() + 1 {
- jit_dirty_page(get_jit_state(), Page::page_of(page << 12));
- }
- }
- /// dirty pages in the range of start_addr and end_addr, which must span at most two pages
- pub fn jit_dirty_cache_small(start_addr: u32, end_addr: u32) {
- dbg_assert!(start_addr < end_addr);
- let start_page = Page::page_of(start_addr);
- let end_page = Page::page_of(end_addr - 1);
- let ctx = get_jit_state();
- jit_dirty_page(ctx, start_page);
- // Note: This can't happen when paging is enabled, as writes across
- // boundaries are split up on two pages
- if start_page != end_page {
- dbg_assert!(start_page.to_u32() + 1 == end_page.to_u32());
- jit_dirty_page(ctx, end_page);
- }
- }
- #[no_mangle]
- pub fn jit_clear_cache_js() { jit_clear_cache(get_jit_state()) }
- pub fn jit_clear_cache(ctx: &mut JitState) {
- let mut pages_with_code = HashSet::new();
- for &p in ctx.entry_points.keys() {
- pages_with_code.insert(p);
- }
- for &p in ctx.pages.keys() {
- pages_with_code.insert(p);
- }
- for page in pages_with_code {
- jit_dirty_page(ctx, page);
- }
- }
- pub fn jit_page_has_code(page: Page) -> bool { jit_page_has_code_ctx(get_jit_state(), page) }
- pub fn jit_page_has_code_ctx(ctx: &mut JitState, page: Page) -> bool {
- ctx.pages.contains_key(&page) || ctx.entry_points.contains_key(&page)
- }
- #[no_mangle]
- pub fn jit_get_wasm_table_index_free_list_count() -> u32 {
- if cfg!(feature = "profiler") {
- get_jit_state().wasm_table_index_free_list.len() as u32
- }
- else {
- 0
- }
- }
- #[no_mangle]
- pub fn jit_get_cache_size() -> u32 {
- if cfg!(feature = "profiler") {
- get_jit_state()
- .pages
- .values()
- .map(|p| p.entry_points.len() as u32)
- .sum()
- }
- else {
- 0
- }
- }
- #[cfg(feature = "profiler")]
- pub fn check_missed_entry_points(phys_address: u32, state_flags: CachedStateFlags) {
- let ctx = get_jit_state();
- if let Some(infos) = ctx.pages.get(&Page::page_of(phys_address)) {
- if infos.state_flags != state_flags {
- return;
- }
- let last_jump_type = unsafe { cpu::debug_last_jump.name() };
- let last_jump_addr = unsafe { cpu::debug_last_jump.phys_address() }.unwrap_or(0);
- let last_jump_opcode =
- if last_jump_addr != 0 { memory::read32s(last_jump_addr) } else { 0 };
- let opcode = memory::read32s(phys_address);
- dbg_log!(
- "Compiled exists, but no entry point, \
- phys_addr={:x} opcode={:02x} {:02x} {:02x} {:02x}. \
- Last jump at {:x} ({}) opcode={:02x} {:02x} {:02x} {:02x}",
- phys_address,
- opcode & 0xFF,
- opcode >> 8 & 0xFF,
- opcode >> 16 & 0xFF,
- opcode >> 16 & 0xFF,
- last_jump_addr,
- last_jump_type,
- last_jump_opcode & 0xFF,
- last_jump_opcode >> 8 & 0xFF,
- last_jump_opcode >> 16 & 0xFF,
- last_jump_opcode >> 16 & 0xFF,
- );
- }
- }
- #[no_mangle]
- #[cfg(feature = "profiler")]
- pub fn debug_set_dispatcher_target(_target_index: i32) {
- //dbg_log!("About to call dispatcher target_index={}", target_index);
- }
- #[no_mangle]
- #[cfg(feature = "profiler")]
- pub fn check_dispatcher_target(target_index: i32, max: i32) {
- //dbg_log!("Dispatcher called target={}", target_index);
- dbg_assert!(target_index >= 0);
- dbg_assert!(target_index < max);
- }
- #[no_mangle]
- #[cfg(feature = "profiler")]
- pub fn enter_basic_block(phys_eip: u32) {
- let eip =
- unsafe { cpu::translate_address_read(*global_pointers::instruction_pointer).unwrap() };
- if Page::page_of(eip) != Page::page_of(phys_eip) {
- dbg_log!(
- "enter basic block failed block=0x{:x} actual eip=0x{:x}",
- phys_eip,
- eip
- );
- panic!();
- }
- }
- #[no_mangle]
- #[cfg(feature = "profiler")]
- pub fn get_config(index: u32) -> u32 {
- match index {
- 0 => MAX_PAGES as u32,
- 1 => JIT_USE_LOOP_SAFETY as u32,
- 2 => MAX_EXTRA_BASIC_BLOCKS as u32,
- _ => 0,
- }
- }
|