v86/src/rust/cpu/string.rs

// string operations
//
//       cmp  si  di
// movs   0    1   1/w    A4
// cmps   1    1   1/r    A6
// stos   0    0   1/w    AA
// lods   0    1   0      AC
// scas   1    0   1/r    AE
// ins    0    0   1/w
// outs   0    1   0

use cpu::arith::{cmp8, cmp16, cmp32};
use cpu::cpu::{
    get_seg, io_port_read8, io_port_read16, io_port_read32, io_port_write8, io_port_write16,
    io_port_write32, read_reg16, read_reg32, safe_read8, safe_read16, safe_read32s, safe_write8,
    safe_write16, safe_write32, set_reg_asize, test_privileges_for_io, translate_address_read,
    translate_address_write_and_can_skip_dirty, writable_or_pagefault, write_reg8, write_reg16,
    write_reg32, AL, AX, DX, EAX, ECX, EDI, ES, ESI, FLAG_DIRECTION,
};
use cpu::global_pointers::{flags, instruction_pointer, previous_ip};
use cpu::memory::{
    in_mapped_range, in_svga_lfb, memcpy_into_svga_lfb, memcpy_no_mmap_or_dirty_check,
    memset_no_mmap_or_dirty_check, read8_no_mmap_check, read16_no_mmap_check, read32_no_mmap_check,
    write8_no_mmap_or_dirty_check, write16_no_mmap_or_dirty_check, write32_no_mmap_or_dirty_check,
};
use page::Page;

fn count_until_end_of_page(direction: i32, size: i32, addr: u32) -> u32 {
    (if direction == 1 {
        (0x1000 - (addr & 0xFFF)) / size as u32
    }
    else {
        (addr & 0xFFF) / size as u32 + 1
    }) as u32
}

#[derive(Copy, Clone)]
enum Instruction {
    Movs,
    Lods,
    Stos,
    Scas,
    Cmps,
    Ins,
    Outs,
}
#[derive(PartialEq)]
enum Size {
    B,
    W,
    D,
}
#[derive(Copy, Clone)]
enum Rep {
    None,
    Z,
    NZ,
}

// We implement all string instructions here and rely on the inliner on doing its job of optimising
// away anything known at compile time (check with `wasm-dis build/v86.wasm`)
#[inline(always)]
unsafe fn string_instruction(
    is_asize_32: bool,
    ds: i32,
    instruction: Instruction,
    size: Size,
    rep: Rep,
) {
    let asize_mask = if is_asize_32 { -1 } else { 0xFFFF };

    let direction = if 0 != *flags & FLAG_DIRECTION { -1 } else { 1 };

    let es = match instruction {
        Instruction::Movs
        | Instruction::Cmps
        | Instruction::Stos
        | Instruction::Scas
        | Instruction::Ins => return_on_pagefault!(get_seg(ES)),
        _ => 0,
    };

    let size_bytes = match size {
        Size::B => 1,
        Size::W => 2,
        Size::D => 4,
    };
    let size_mask = match size {
        Size::B => 0xFF,
        Size::W => 0xFFFF,
        Size::D => -1,
    };

    let increment = direction * size_bytes;

    let data = match instruction {
        Instruction::Stos | Instruction::Scas => read_reg32(EAX),
        _ => 0,
    };

    let mut src = match instruction {
        Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
            read_reg32(ESI) & asize_mask
        },
        _ => 0,
    };
    let mut dst = match instruction {
        Instruction::Movs
        | Instruction::Cmps
        | Instruction::Stos
        | Instruction::Scas
        | Instruction::Ins => read_reg32(EDI) & asize_mask,
        _ => 0,
    };
    let mut count = match rep {
        Rep::Z | Rep::NZ => {
            let c = (read_reg32(ECX) & asize_mask) as u32;
            if c == 0 {
                return;
            };
            c
        },
        Rep::None => 0,
    };

    let port = match instruction {
        Instruction::Ins | Instruction::Outs => {
            let port = read_reg16(DX);
            if !test_privileges_for_io(port, size_bytes) {
                return;
            }
            port
        },
        _ => 0,
    };

    let is_aligned = (ds + src) & (size_bytes - 1) == 0 && (es + dst) & (size_bytes - 1) == 0;
    let mut rep_fast = is_aligned
        && is_asize_32 // 16-bit address wraparound
        && match rep {
            Rep::NZ | Rep::Z => true,
            Rep::None => false,
        };

    let mut phys_dst = 0;
    let mut phys_src = 0;
    let mut skip_dirty_page = false;
    let mut movs_into_svga_lfb = false;

    if rep_fast {
        match instruction {
            Instruction::Movs => {
                let (addr, skip) =
                    return_on_pagefault!(translate_address_write_and_can_skip_dirty(es + dst));
                movs_into_svga_lfb = in_svga_lfb(addr);
                rep_fast = rep_fast && (!in_mapped_range(addr) || movs_into_svga_lfb);
                phys_dst = addr;
                skip_dirty_page = skip;
            },
            Instruction::Stos | Instruction::Ins => {
                let (addr, skip) =
                    return_on_pagefault!(translate_address_write_and_can_skip_dirty(es + dst));
                rep_fast = rep_fast && !in_mapped_range(addr);
                phys_dst = addr;
                skip_dirty_page = skip;
            },
            Instruction::Cmps | Instruction::Scas => {
                let addr = return_on_pagefault!(translate_address_read(es + dst));
                rep_fast = rep_fast && !in_mapped_range(addr);
                phys_dst = addr;
                skip_dirty_page = true;
            },
            _ => {},
        };

        match instruction {
            Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
                let addr = return_on_pagefault!(translate_address_read(ds + src));
                rep_fast = rep_fast && !in_mapped_range(addr);
                phys_src = addr;
            },
            _ => {},
        };

        match instruction {
            Instruction::Movs => {
                // note: This check is also valid for both direction == 1 and direction == -1
                let overlap = u32::max(phys_src, phys_dst) - u32::min(phys_src, phys_dst)
                    < count * size_bytes as u32;
                rep_fast = rep_fast && !overlap;
            },
            _ => {},
        }
    }

    if rep_fast {
        let count_until_end_of_page = u32::min(
            count,
            match instruction {
                Instruction::Movs | Instruction::Cmps => u32::min(
                    count_until_end_of_page(direction, size_bytes, phys_src),
                    count_until_end_of_page(direction, size_bytes, phys_dst),
                ),
                Instruction::Stos | Instruction::Ins | Instruction::Scas => {
                    count_until_end_of_page(direction, size_bytes, phys_dst)
                },
                Instruction::Lods | Instruction::Outs => {
                    count_until_end_of_page(direction, size_bytes, phys_src)
                },
            },
        );
        dbg_assert!(count_until_end_of_page > 0);

        if !skip_dirty_page {
            ::jit::jit_dirty_page(::jit::get_jit_state(), Page::page_of(phys_dst));
        }

        let mut rep_cmp_finished = false;

        let mut i = 0;
        while i < count_until_end_of_page {
            i += 1;

            let src_val = match instruction {
                Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
                    match size {
                        Size::B => read8_no_mmap_check(phys_src),
                        Size::W => read16_no_mmap_check(phys_src),
                        Size::D => read32_no_mmap_check(phys_src),
                    }
                },
                Instruction::Scas | Instruction::Stos => data & size_mask,
                Instruction::Ins => match size {
                    Size::B => io_port_read8(port),
                    Size::W => io_port_read16(port),
                    Size::D => io_port_read32(port),
                },
            };

            let mut dst_val = 0;

            match instruction {
                Instruction::Cmps | Instruction::Scas => match size {
                    Size::B => dst_val = read8_no_mmap_check(phys_dst),
                    Size::W => dst_val = read16_no_mmap_check(phys_dst),
                    Size::D => dst_val = read32_no_mmap_check(phys_dst),
                },
                Instruction::Outs => match size {
                    Size::B => io_port_write8(port, src_val),
                    Size::W => io_port_write16(port, src_val),
                    Size::D => io_port_write32(port, src_val),
                },
                Instruction::Lods => match size {
                    Size::B => write_reg8(AL, src_val),
                    Size::W => write_reg16(AX, src_val),
                    Size::D => write_reg32(EAX, src_val),
                },
                Instruction::Ins => match size {
                    Size::B => write8_no_mmap_or_dirty_check(phys_dst, src_val),
                    Size::W => write16_no_mmap_or_dirty_check(phys_dst, src_val),
                    Size::D => write32_no_mmap_or_dirty_check(phys_dst, src_val),
                },
                Instruction::Movs => {
                    if direction == -1 {
                        phys_src -= (count_until_end_of_page - 1) * size_bytes as u32;
                        phys_dst -= (count_until_end_of_page - 1) * size_bytes as u32;
                    }
                    if movs_into_svga_lfb {
                        ::cpu::vga::mark_dirty(phys_dst);
                        memcpy_into_svga_lfb(
                            phys_src,
                            phys_dst,
                            count_until_end_of_page * size_bytes as u32,
                        );
                    }
                    else {
                        memcpy_no_mmap_or_dirty_check(
                            phys_src,
                            phys_dst,
                            count_until_end_of_page * size_bytes as u32,
                        );
                    }
                    i = count_until_end_of_page;
                    break;
                },
                Instruction::Stos => match size {
                    Size::B => {
                        if direction == -1 {
                            phys_dst -= count_until_end_of_page - 1
                        }
                        memset_no_mmap_or_dirty_check(
                            phys_dst,
                            src_val as u8,
                            count_until_end_of_page,
                        );
                        i = count_until_end_of_page;
                        break;
                    },
                    Size::W => write16_no_mmap_or_dirty_check(phys_dst, src_val),
                    Size::D => write32_no_mmap_or_dirty_check(phys_dst, src_val),
                },
            };

            match instruction {
                Instruction::Movs
                | Instruction::Cmps
                | Instruction::Stos
                | Instruction::Scas
                | Instruction::Ins => {
                    phys_dst += increment as u32;
                },
                _ => {},
            }
            match instruction {
                Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
                    phys_src += increment as u32;
                },
                _ => {},
            };

            match instruction {
                Instruction::Scas | Instruction::Cmps => {
                    let rep_cmp = match rep {
                        Rep::Z => src_val == dst_val,
                        Rep::NZ => src_val != dst_val,
                        Rep::None => {
                            dbg_assert!(false);
                            true
                        },
                    };
                    if !rep_cmp || count == i {
                        match size {
                            Size::B => cmp8(src_val, dst_val),
                            Size::W => cmp16(src_val, dst_val),
                            Size::D => cmp32(src_val, dst_val),
                        };
                        rep_cmp_finished = true;
                        break;
                    }
                },
                _ => {},
            }
        }

        dbg_assert!(i <= count);
        count -= i;

        if !rep_cmp_finished && count != 0 {
            // go back to the current instruction, since this loop just handles a single page
            *instruction_pointer = *previous_ip;
        }

        src += i as i32 * increment;
        dst += i as i32 * increment;
    }
    else {
        loop {
            match instruction {
                Instruction::Ins => {
                    // check fault *before* reading from port
                    // (technically not necessary according to Intel manuals)
                    break_on_pagefault!(writable_or_pagefault(es + dst, size_bytes));
                },
                _ => {},
            };
            let src_val = match instruction {
                Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
                    break_on_pagefault!(match size {
                        Size::B => safe_read8(ds + src),
                        Size::W => safe_read16(ds + src),
                        Size::D => safe_read32s(ds + src),
                    })
                },
                Instruction::Scas | Instruction::Stos => data & size_mask,
                Instruction::Ins => match size {
                    Size::B => io_port_read8(port),
                    Size::W => io_port_read16(port),
                    Size::D => io_port_read32(port),
                },
            };

            let mut dst_val = 0;

            match instruction {
                Instruction::Cmps | Instruction::Scas => match size {
                    Size::B => dst_val = break_on_pagefault!(safe_read8(es + dst)),
                    Size::W => dst_val = break_on_pagefault!(safe_read16(es + dst)),
                    Size::D => dst_val = break_on_pagefault!(safe_read32s(es + dst)),
                },
                Instruction::Outs => match size {
                    Size::B => io_port_write8(port, src_val),
                    Size::W => io_port_write16(port, src_val),
                    Size::D => io_port_write32(port, src_val),
                },
                Instruction::Lods => match size {
                    Size::B => write_reg8(AL, src_val),
                    Size::W => write_reg16(AX, src_val),
                    Size::D => write_reg32(EAX, src_val),
                },
                Instruction::Movs | Instruction::Stos | Instruction::Ins => match size {
                    Size::B => break_on_pagefault!(safe_write8(es + dst, src_val)),
                    Size::W => break_on_pagefault!(safe_write16(es + dst, src_val)),
                    Size::D => break_on_pagefault!(safe_write32(es + dst, src_val)),
                },
            };

            match instruction {
                Instruction::Movs
                | Instruction::Cmps
                | Instruction::Stos
                | Instruction::Scas
                | Instruction::Ins => dst = dst + increment & asize_mask,
                _ => {},
            }
            match instruction {
                Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
                    src = src + increment & asize_mask
                },
                _ => {},
            };

            let finished = match rep {
                Rep::Z | Rep::NZ => {
                    let rep_cmp = match (rep, instruction) {
                        (Rep::Z, Instruction::Cmps) => src_val == dst_val,
                        (Rep::Z, Instruction::Scas) => src_val == dst_val,
                        (Rep::NZ, Instruction::Cmps) => src_val != dst_val,
                        (Rep::NZ, Instruction::Scas) => src_val != dst_val,
                        _ => true,
                    };
                    count -= 1;
                    if count != 0 && rep_cmp {
                        //*instruction_pointer = *previous_ip
                        false
                    }
                    else {
                        true
                    }
                },
                Rep::None => true,
            };

            if finished {
                match instruction {
                    Instruction::Scas | Instruction::Cmps => match size {
                        Size::B => cmp8(src_val, dst_val),
                        Size::W => cmp16(src_val, dst_val),
                        Size::D => cmp32(src_val, dst_val),
                    },
                    _ => {},
                }
                break;
            }
        }
    }

    match instruction {
        Instruction::Movs
        | Instruction::Cmps
        | Instruction::Stos
        | Instruction::Scas
        | Instruction::Ins => set_reg_asize(is_asize_32, EDI, dst),
        _ => {},
    }
    match instruction {
        Instruction::Movs | Instruction::Cmps | Instruction::Lods | Instruction::Outs => {
            set_reg_asize(is_asize_32, ESI, src)
        },
        _ => {},
    };

    match rep {
        Rep::Z | Rep::NZ => {
            set_reg_asize(is_asize_32, ECX, count as i32);
        },
        Rep::None => {},
    }
}

#[no_mangle]
pub unsafe fn movsb_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Movs, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn movsw_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Movs, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn movsd_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Movs, Size::D, Rep::Z)
}
pub unsafe fn movsb_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Movs, Size::B, Rep::None)
}
pub unsafe fn movsw_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Movs, Size::W, Rep::None)
}
pub unsafe fn movsd_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Movs, Size::D, Rep::None)
}

#[no_mangle]
pub unsafe fn lodsb_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Lods, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn lodsw_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Lods, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn lodsd_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Lods, Size::D, Rep::Z)
}
pub unsafe fn lodsb_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Lods, Size::B, Rep::None)
}
pub unsafe fn lodsw_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Lods, Size::W, Rep::None)
}
pub unsafe fn lodsd_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Lods, Size::D, Rep::None)
}

#[no_mangle]
pub unsafe fn stosb_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Stos, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn stosw_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Stos, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn stosd_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Stos, Size::D, Rep::Z)
}
pub unsafe fn stosb_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Stos, Size::B, Rep::None)
}
pub unsafe fn stosw_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Stos, Size::W, Rep::None)
}
pub unsafe fn stosd_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Stos, Size::D, Rep::None)
}

#[no_mangle]
pub unsafe fn cmpsb_repz(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn cmpsw_repz(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn cmpsd_repz(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::D, Rep::Z)
}
#[no_mangle]
pub unsafe fn cmpsb_repnz(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::B, Rep::NZ)
}
#[no_mangle]
pub unsafe fn cmpsw_repnz(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::W, Rep::NZ)
}
#[no_mangle]
pub unsafe fn cmpsd_repnz(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::D, Rep::NZ)
}
#[no_mangle]
pub unsafe fn cmpsb_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::B, Rep::None)
}
#[no_mangle]
pub unsafe fn cmpsw_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::W, Rep::None)
}
#[no_mangle]
pub unsafe fn cmpsd_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Cmps, Size::D, Rep::None)
}

#[no_mangle]
pub unsafe fn scasb_repz(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn scasw_repz(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn scasd_repz(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::D, Rep::Z)
}
#[no_mangle]
pub unsafe fn scasb_repnz(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::B, Rep::NZ)
}
#[no_mangle]
pub unsafe fn scasw_repnz(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::W, Rep::NZ)
}
#[no_mangle]
pub unsafe fn scasd_repnz(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::D, Rep::NZ)
}
pub unsafe fn scasb_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::B, Rep::None)
}
pub unsafe fn scasw_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::W, Rep::None)
}
pub unsafe fn scasd_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Scas, Size::D, Rep::None)
}

#[no_mangle]
pub unsafe fn outsb_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Outs, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn outsw_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Outs, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn outsd_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Outs, Size::D, Rep::Z)
}
#[no_mangle]
pub unsafe fn outsb_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Outs, Size::B, Rep::None)
}
#[no_mangle]
pub unsafe fn outsw_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Outs, Size::W, Rep::None)
}
#[no_mangle]
pub unsafe fn outsd_no_rep(is_asize_32: bool, ds: i32) {
    string_instruction(is_asize_32, ds, Instruction::Outs, Size::D, Rep::None)
}

#[no_mangle]
pub unsafe fn insb_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Ins, Size::B, Rep::Z)
}
#[no_mangle]
pub unsafe fn insw_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Ins, Size::W, Rep::Z)
}
#[no_mangle]
pub unsafe fn insd_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Ins, Size::D, Rep::Z)
}
#[no_mangle]
pub unsafe fn insb_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Ins, Size::B, Rep::None)
}
#[no_mangle]
pub unsafe fn insw_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Ins, Size::W, Rep::None)
}
#[no_mangle]
pub unsafe fn insd_no_rep(is_asize_32: bool) {
    string_instruction(is_asize_32, 0, Instruction::Ins, Size::D, Rep::None)
}