ソースを参照

OS loader paging functions for x64.

This change adds paging functions to get a little further in the OS
loader. Right now the kernel doesn't link, so the OS loader dies shortly
before trying to load the kernel. The page tables created by this mapping
code haven't been tested yet, so there may be some kinks to work out.

I changed the way the page tables are set up by the boot manager. Instead
of asking the OS loader to manage firmware mappings for every page it
uses, I'm simply identity mapping the first 8GB. Really I should only need
to identity map the first 4GB, since otherwise that particular machine
would be incapable of booting 32-bit OSes. But 8GB really ought to do it.

With 1GB pages, it's trivial. Some new machines don't support 1GB pages,
so it takes 4096 entries at 2MB pages to map those 8GB. Still pretty easy.
This should also allow us to keep using the firmware tables in UEFI, since
there won't be weird self-map regions to contend with.

Next up, I'll need to get the kernel linking, which involves at least
stubs for nearly everything.
Evan Green 7 年 前
コミット
e57c6cee21

+ 80 - 307
boot/bootman/pcat/paging.c

@@ -47,50 +47,10 @@ Environment:
 // ------------------------------------------------------ Data Type Definitions
 //
 
-/*++
-
-Structure Description:
-
-    This structure defines the iteration context for mapping all the boot
-    manager allocations.
-
-Members:
-
-    PagesMapped - Stores the number of pages that were successfully mapped.
-
-    Status - Stores the overall status code.
-
---*/
-
-typedef struct _BOOTMAN_MAPPING_CONTEXT {
-    UINTN PagesMapped;
-    KSTATUS Status;
-} BOOTMAN_MAPPING_CONTEXT, *PBOOTMAN_MAPPING_CONTEXT;
-
 //
 // ----------------------------------------------- Internal Function Prototypes
 //
 
-VOID
-BmpFwBootMappingIterationRoutine (
-    PMEMORY_DESCRIPTOR_LIST DescriptorList,
-    PMEMORY_DESCRIPTOR Descriptor,
-    PVOID Context
-    );
-
-KSTATUS
-BmpFwIdentityMapPages (
-    ULONGLONG Address,
-    UINTN Size,
-    PUINTN PagesMapped
-    );
-
-KSTATUS
-BmpFwIdentityMapPage (
-    ULONGLONG Address,
-    PUINTN PagesMapped
-    );
-
 //
 // -------------------------------------------------------------------- Globals
 //
@@ -101,12 +61,6 @@ BmpFwIdentityMapPage (
 
 PPTE FwPml4Table;
 
-//
-// Define the PML4 self map index.
-//
-
-ULONG FwSelfMapIndex;
-
 //
 // ------------------------------------------------------------------ Functions
 //
@@ -134,313 +88,132 @@ Return Value:
 
 {
 
-    BOOTMAN_MAPPING_CONTEXT Context;
+    ULONG Count;
+    ULONG Eax;
+    ULONG Ebx;
+    ULONG Ecx;
+    ULONG Edx;
+    BOOL HugePages;
+    ULONG Index;
     ULONGLONG Page;
+    ULONG PageCount;
     KSTATUS Status;
     PPTE Table;
 
-    Context.PagesMapped = 0;
-    Context.Status = STATUS_SUCCESS;
-
     //
-    // Allocate and initialize a PML4, plus a PDPT and PDT for the first 2MB.
+    // Allocate and initialize a PML4, then like lazy slobs just identity
+    // map the first 8GB and call it a day. The PCAT memory allocation has a
+    // governer in there as well to avoid accidentally allocating pages greater
+    // than that.
     //
 
     if (FwPml4Table == INVALID_PHYSICAL_ADDRESS) {
-        Status = FwAllocatePages(&Page,
-                                 PAGE_SIZE * 3,
-                                 PAGE_SIZE,
-                                 MemoryTypeLoaderTemporary);
-
-        if (!KSUCCESS(Status)) {
-            goto FwCreatePageTablesEnd;
-        }
-
-        ASSERT(Page == (UINTN)Page);
-
-        FwPml4Table = (PVOID)(UINTN)Page;
-        RtlZeroMemory(FwPml4Table, PAGE_SIZE * 3);
 
         //
-        // Just use the highest value as the self map index. This conveniently
-        // keeps it out of a range where the self map might collide with real
-        // physical pages.
+        // First find out if the processor supports 1GB pages. This leaf must
+        // be supported because it's the one that determined the machine was
+        // long-mode capable.
         //
 
-        FwSelfMapIndex = X64_PTE_COUNT;
-        FwPml4Table[FwSelfMapIndex] =
-                              X86_ENTRY_PTE((UINTN)FwPml4Table >> PAGE_SHIFT) |
-                              X86_PTE_PRESENT |
-                              X86_PTE_WRITABLE;
+        Eax = X86_CPUID_EXTENDED_INFORMATION;
+        ArCpuid(&Eax, &Ebx, &Ecx, &Edx);
 
         //
-        // Identity map the first 2MB with a large page, it's just easier that
-        // way.
+        // With 1GB pages, just a PML4T and PDPT are needed (4 entries).
         //
 
-        Page += PAGE_SIZE;
-        FwPml4Table[0] = X86_ENTRY_PTE(Page >> PAGE_SHIFT) |
-                         X86_PTE_PRESENT |
-                         X86_PTE_WRITABLE;
+        if ((Edx & X86_CPUID_EXTENDED_INFORMATION_EDX_1GB_PAGES) != 0) {
+            HugePages = TRUE;
+            PageCount = 2;
 
         //
-        // Set the PDPT.
-        //
-
-        Table = (PPTE)(UINTN)Page;
-        Page += PAGE_SIZE;
-        Table[0] = X86_ENTRY_PTE(Page >> PAGE_SHIFT) |
-                   X86_PTE_PRESENT |
-                   X86_PTE_WRITABLE;
-
+        // With only 2MB pages, 4 PDTs are needed too.
         //
-        // Set the PD entry as a large 2MB page mapping the first 2MB.
-        //
-
-        Table = (PPTE)(UINTN)Page;
-        Table[0] = X86_ENTRY_PTE(0) |
-                   X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_LARGE;
-    }
-
-    MmMdIterate(&BoMemoryMap,
-                BmpFwBootMappingIterationRoutine,
-                &Context);
 
-    if (!KSUCCESS(Context.Status)) {
-        goto FwCreatePageTablesEnd;
-    }
-
-    Parameters->PageDirectory = (UINTN)FwPml4Table;
-    Parameters->PageTables = ((ULONGLONG)FwSelfMapIndex << X64_PML4E_SHIFT) |
-                             X64_CANONICAL_HIGH;
-
-FwCreatePageTablesEnd:
-    return Status;
-}
-
-//
-// --------------------------------------------------------- Internal Functions
-//
-
-VOID
-BmpFwBootMappingIterationRoutine (
-    PMEMORY_DESCRIPTOR_LIST DescriptorList,
-    PMEMORY_DESCRIPTOR Descriptor,
-    PVOID Context
-    )
-
-/*++
-
-Routine Description:
-
-    This routine is called once for each descriptor in the memory descriptor
-    list.
-
-Arguments:
-
-    DescriptorList - Supplies a pointer to the descriptor list being iterated
-        over.
-
-    Descriptor - Supplies a pointer to the current descriptor.
-
-    Context - Supplies an optional opaque pointer of context that was provided
-        when the iteration was requested.
-
-Return Value:
-
-    None.
-
---*/
-
-{
-
-    PBOOTMAN_MAPPING_CONTEXT IterationContext;
-    UINTN PagesMapped;
-    KSTATUS Status;
-
-    IterationContext = Context;
-    if (!KSUCCESS(IterationContext->Status)) {
-        return;
-    }
-
-    //
-    // Skip all except interesting descriptors.
-    //
-
-    if ((Descriptor->Type == MemoryTypeFirmwareTemporary) ||
-        (Descriptor->Type == MemoryTypeLoaderTemporary) ||
-        (Descriptor->Type == MemoryTypeLoaderPermanent)) {
-
-        PagesMapped = 0;
-        Status = BmpFwIdentityMapPages(Descriptor->BaseAddress,
-                                       Descriptor->Size,
-                                       &PagesMapped);
-
-        IterationContext->PagesMapped += PagesMapped;
-        if (!KSUCCESS(Status)) {
-            IterationContext->Status = Status;
+        } else {
+            HugePages = FALSE;
+            PageCount = 2 + 4;
         }
-    }
 
-    return;
-}
-
-KSTATUS
-BmpFwIdentityMapPages (
-    ULONGLONG Address,
-    UINTN Size,
-    PUINTN PagesMapped
-    )
-
-/*++
-
-Routine Description:
-
-    This routine identity maps a region of memory in preparation for switching
-    64-bit paging on.
-
-Arguments:
-
-    Address - Supplies the address to identity map.
-
-    Size - Supplies the size to map.
-
-    PagesMapped - Supplies a pointer where the number of pages successfully
-        mapped will be incremented. Pages already mapped do not count.
-
-Return Value:
-
-    Status code.
-
---*/
-
-{
-
-    ULONGLONG Current;
-    UINTN Index;
-    UINTN PageCount;
-    KSTATUS Status;
-
-    Current = ALIGN_RANGE_DOWN(Address, PAGE_SIZE);
-    PageCount = (ALIGN_RANGE_UP(Address + Size, PAGE_SIZE) - Current) >>
-                PAGE_SHIFT;
+        Status = FwAllocatePages(&Page,
+                                 PageCount * PAGE_SIZE,
+                                 PAGE_SIZE,
+                                 MemoryTypeLoaderTemporary);
 
-    for (Index = 0; Index < PageCount; Index += 1) {
-        Status = BmpFwIdentityMapPage(Current, PagesMapped);
         if (!KSUCCESS(Status)) {
-            return Status;
+            goto FwCreatePageTablesEnd;
         }
 
-        Current += PAGE_SIZE;
-    }
-
-    return STATUS_SUCCESS;
-}
-
-KSTATUS
-BmpFwIdentityMapPage (
-    ULONGLONG Address,
-    PUINTN PagesMapped
-    )
-
-/*++
-
-Routine Description:
-
-    This routine identity maps a page of memory in preparation for switching
-    64-bit paging on.
-
-Arguments:
+        ASSERT(Page == (UINTN)Page);
 
-    Address - Supplies the address of the page to identity map.
+        FwPml4Table = (PVOID)(UINTN)Page;
+        Page += PAGE_SIZE;
 
-    PagesMapped - Supplies a pointer whose value will be incremented if a new
-        page was mapped.
+        //
+        // Zero out the PML4T and the PDPT. Point the PML4T at the PDPT.
+        //
 
-Return Value:
+        RtlZeroMemory(FwPml4Table, PAGE_SIZE * 2);
+        FwPml4Table[0] = Page | X86_PTE_PRESENT | X86_PTE_WRITABLE;
 
-    Status code.
+        //
+        // If 1GB pages are supported, just fill in the four entries for the
+        // PDPT.
+        //
 
---*/
+        Table = (PPTE)(UINTN)Page;
+        Page += PAGE_SIZE;
+        if (HugePages != FALSE) {
+            Table[0] = X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_LARGE;
+            Table[1] = (2ULL * _1GB) |
+                       X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_LARGE;
 
-{
+            Table[2] = (4ULL * _1GB) |
+                       X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_LARGE;
 
-    ULONG EntryIndex;
-    ULONG Index;
-    ULONGLONG NewPage;
-    ULONG Shift;
-    KSTATUS Status;
-    PPTE Table;
+            Table[3] = (6ULL * _1GB) |
+                       X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_LARGE;
 
-    //
-    // If it's in the first 2MB, don't worry about it, those are mapped with a
-    // large page.
-    //
+        //
+        // 1GB pages are not supported, so map 2MB pages.
+        //
 
-    if (Address < (2 * _1MB)) {
-        return STATUS_SUCCESS;
-    }
+        } else {
 
-    //
-    // Walk the page tables, creating any needed pages along the way.
-    //
+            //
+            // Fill in the PDPT.
+            //
 
-    Table = FwPml4Table;
-    Shift = X64_PML4E_SHIFT;
-    for (Index = 0; Index < X64_PAGE_LEVEL - 1; Index += 1) {
-        EntryIndex = (Address >> Shift) & X64_PT_MASK;
-        if (X86_PTE_ENTRY(Table[EntryIndex]) == 0) {
-            Status = FwAllocatePages(&NewPage,
-                                     PAGE_SIZE,
-                                     PAGE_SIZE,
-                                     MemoryTypeLoaderTemporary);
-
-            if (!KSUCCESS(Status)) {
-                return Status;
+            for (Index = 0; Index < 4; Index += 1) {
+                Table[Index] = Page | X86_PTE_PRESENT | X86_PTE_WRITABLE;
+                Page += PAGE_SIZE;
             }
 
-            ASSERT(NewPage == (UINTN)NewPage);
-
-            RtlZeroMemory((PVOID)(UINTN)NewPage, PAGE_SIZE);
-            Table[EntryIndex] = X86_ENTRY_PTE(NewPage >> PAGE_SHIFT) |
-                                X86_PTE_PRESENT |
-                                X86_PTE_WRITABLE;
-        }
-
-        ASSERT(X86_PTE_ENTRY(Table[EntryIndex]) ==
-               (UINTN)(X86_PTE_ENTRY(Table[EntryIndex])));
-
-        Table = (PPTE)(UINTN)(X86_PTE_ENTRY(Table[EntryIndex]) << PAGE_SHIFT);
-        Shift -= X64_PTE_BITS;
-    }
-
-    ASSERT(Shift == PAGE_SHIFT);
-
-    EntryIndex = (Address >> PAGE_SHIFT) & X64_PT_MASK;
-    if ((Table[EntryIndex] & X86_PTE_PRESENT) != 0) {
-        if (X86_PTE_ENTRY(Table[EntryIndex]) != (Address >> PAGE_SHIFT)) {
-
             //
-            // Some page is already mapped here, and it's not the right one.
+            // Now fill in all the little 2MB pages up to 8GB. 4096 entries in
+            // all. The PDTs are contiguous so this loop spans all 4 of them.
             //
 
-            ASSERT(FALSE);
+            Table += X64_PTE_COUNT;
+            Count = (8LL * _1GB) / (2LL * _1MB);
+            Page = 0;
+            for (Index = 0; Index < Count; Index += 1) {
+                Table[Index] =
+                     Page | X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_LARGE;
 
-            return STATUS_MEMORY_CONFLICT;
+                Page += 2 * _1MB;
+            }
         }
-
-        return STATUS_SUCCESS;
     }
 
-    //
-    // Set the PTE to point at the page itself.
-    //
-
-    Table[EntryIndex] = X86_ENTRY_PTE(Address >> PAGE_SHIFT) |
-                        X86_PTE_PRESENT |
-                        X86_PTE_WRITABLE;
+    Parameters->PageDirectory = (UINTN)FwPml4Table;
+    Status = STATUS_SUCCESS;
 
-    *PagesMapped += 1;
-    return STATUS_SUCCESS;
+FwCreatePageTablesEnd:
+    return Status;
 }
 
+//
+// --------------------------------------------------------- Internal Functions
+//
+

+ 15 - 1
boot/lib/pcat/memory.c

@@ -44,6 +44,20 @@ Environment:
 #define E820_MAGIC 0x534D4150    // 'SMAP'
 #define MAX_E820_DESCRIPTORS 100
 
+#if __SIZEOF_LONG__ == 8
+
+//
+// Keep things to the first 8GB since that's what's mapped by default on x64.
+//
+
+#define PCAT_MAX_ALLOCATION_ADDRESS ((8ULL * _1GB) - 1)
+
+#else
+
+#define PCAT_MAX_ALLOCATION_ADDRESS MAX_UINTN
+
+#endif
+
 //
 // ----------------------------------------------- Internal Function Prototypes
 //
@@ -346,7 +360,7 @@ Return Value:
                                  Size,
                                  Alignment,
                                  0,
-                                 MAX_UINTN,
+                                 PCAT_MAX_ALLOCATION_ADDRESS,
                                  MemoryType,
                                  AllocationStrategyLowestAddress);
 

+ 479 - 9
boot/loader/x64/paging.c

@@ -129,6 +129,25 @@ Return Value:
     *PageDirectory = PhysicalAddress;
     BoPageDirectory = (PVOID)(UINTN)*PageDirectory;
     RtlZeroMemory(BoPageDirectory, PAGE_SIZE);
+
+    //
+    // Set up the self map.
+    //
+
+    BoPageDirectory[X64_SELF_MAP_INDEX] =
+             PhysicalAddress | X86_PTE_PRESENT | X86_PTE_WRITABLE | X86_PTE_NX;
+
+    MmMdInitDescriptor(
+              &KernelSpace,
+              X64_CANONICAL_HIGH | (X64_SELF_MAP_INDEX << X64_PML4E_SHIFT),
+              X64_CANONICAL_HIGH | (X64_SELF_MAP_INDEX + 1) << X64_PML4E_SHIFT,
+              MemoryTypePageTables);
+
+    Status = MmMdAddDescriptorToList(&BoVirtualMap, &KernelSpace);
+    if (!KSUCCESS(Status)) {
+        goto InitializePagingStructuresEnd;
+    }
+
     Status = STATUS_SUCCESS;
 
 InitializePagingStructuresEnd:
@@ -185,9 +204,237 @@ Return Value:
 
 {
 
-    ASSERT(FALSE);
+    UINTN AlignedSize;
+    UINTN CurrentVirtual;
+    ULONG EntryShift;
+    PMEMORY_DESCRIPTOR ExistingDescriptor;
+    BOOL HugePage;
+    ULONG Level;
+    ULONGLONG MappedAddress;
+    UINTN PageCount;
+    ULONG PageOffset;
+    UINTN PagesMapped;
+    PPTE PageTable;
+    ULONG PageTableIndex;
+    MEMORY_TYPE PageTableMemoryType;
+    PHYSICAL_ADDRESS PageTablePhysical;
+    KSTATUS Status;
+    ALLOCATION_STRATEGY Strategy;
+    MEMORY_DESCRIPTOR VirtualSpace;
+    BOOL VirtualSpaceAllocated;
+
+    VirtualSpaceAllocated = FALSE;
+    PageCount = 0;
+    PageOffset = (ULONG)((UINTN)PhysicalAddress & PAGE_MASK);
+    Size += PageOffset;
+    if (BoPageDirectory == NULL) {
+        return STATUS_NOT_INITIALIZED;
+    }
+
+    if ((VirtualAddress != NULL) &&
+        (*VirtualAddress != (PVOID)-1) &&
+        (((UINTN)*VirtualAddress & PAGE_MASK) !=
+         ((UINTN)PhysicalAddress & PAGE_MASK))) {
 
-    return STATUS_NOT_IMPLEMENTED;
+        return STATUS_INVALID_PARAMETER;
+    }
+
+    Strategy = AllocationStrategyAnyAddress;
+    if (MemoryType == MemoryTypeLoaderTemporary) {
+        Strategy = AllocationStrategyHighestAddress;
+    }
+
+    //
+    // Get the requested address, or find a virtual address if one was not
+    // supplied.
+    //
+
+    if ((VirtualAddress == NULL) || (*VirtualAddress == (PVOID)-1)) {
+        AlignedSize = ALIGN_RANGE_UP(Size, PAGE_SIZE);
+        MappedAddress = 0;
+        Status = MmMdAllocateFromMdl(&BoVirtualMap,
+                                     &MappedAddress,
+                                     AlignedSize,
+                                     PAGE_SIZE,
+                                     0,
+                                     MAX_UINTN,
+                                     MemoryType,
+                                     Strategy);
+
+        if (!KSUCCESS(Status)) {
+            Status = STATUS_NO_MEMORY;
+            goto MapPhysicalAddressEnd;
+        }
+
+        if (VirtualAddress != NULL) {
+            *VirtualAddress = (PVOID)MappedAddress;
+        }
+
+    } else {
+        MappedAddress = (UINTN)*VirtualAddress;
+
+        //
+        // Check to see if this region is occupied already, and fail if it is.
+        //
+
+        ExistingDescriptor = MmMdLookupDescriptor(
+                                                &BoVirtualMap,
+                                                (UINTN)*VirtualAddress,
+                                                (UINTN)*VirtualAddress + Size);
+
+        if ((ExistingDescriptor != NULL) &&
+            (ExistingDescriptor->Type != MemoryTypeFree)) {
+
+            Status = STATUS_MEMORY_CONFLICT;
+            goto MapPhysicalAddressEnd;
+        }
+
+        //
+        // Add the descriptor to the virtual memory map to account for its use.
+        //
+
+        MmMdInitDescriptor(&VirtualSpace,
+                           MappedAddress,
+                           MappedAddress + Size,
+                           MemoryType);
+
+        Status = MmMdAddDescriptorToList(&BoVirtualMap, &VirtualSpace);
+        if (!KSUCCESS(Status)) {
+            goto MapPhysicalAddressEnd;
+        }
+    }
+
+    VirtualSpaceAllocated = TRUE;
+    if ((VirtualAddress != NULL) && (*VirtualAddress != NULL)) {
+        *VirtualAddress = (PVOID)((UINTN)*VirtualAddress + PageOffset);
+    }
+
+    //
+    // Ensure the space is big enough.
+    //
+
+    if (MappedAddress + Size < MappedAddress) {
+        Status = STATUS_INVALID_PARAMETER;
+        goto MapPhysicalAddressEnd;
+    }
+
+    CurrentVirtual = (UINTN)MappedAddress;
+    PageCount = ALIGN_RANGE_UP(Size, PAGE_SIZE) / PAGE_SIZE;
+    PagesMapped = 0;
+    HugePage = FALSE;
+    while (PageCount != PagesMapped) {
+
+        //
+        // Get to the lowest level page table, allocating page tables along the
+        // way.
+        //
+
+        PageTable = BoPageDirectory;
+        EntryShift = X64_PML4E_SHIFT;
+        for (Level = 0; Level < X64_PAGE_LEVEL - 1; Level += 1) {
+            PageTableIndex = (CurrentVirtual >> EntryShift) & X64_PT_MASK;
+            PageTable += PageTableIndex;
+            EntryShift -= X64_PTE_BITS;
+            if ((*PageTable & X86_PTE_PRESENT) == 0) {
+
+                //
+                // Check to see if the thing being mapped here could fit in a
+                // 2MB page. If so, stop before the last level page table, and
+                // the code outside the loop will fill in a PDE instead of a
+                // PTE.
+                //
+
+                if ((Level == X64_PAGE_LEVEL - 2) &&
+                    ((PageCount - PagesMapped) >= (_2MB / PAGE_SIZE)) &&
+                    ((CurrentVirtual & (_2MB - 1)) == 0)) {
+
+                    RtlDebugPrint("Using huge page at VA 0x%llx\n",
+                                  CurrentVirtual);
+
+                    HugePage = TRUE;
+                    break;
+                }
+
+                PageTableMemoryType = MemoryTypePageTables;
+                if (CurrentVirtual < (UINTN)KERNEL_VA_START) {
+
+                    ASSERT(MemoryType == MemoryTypeLoaderTemporary);
+
+                    PageTableMemoryType = MemoryTypeBootPageTables;
+                }
+
+                Status = FwAllocatePages(&PageTablePhysical,
+                                         PAGE_SIZE,
+                                         PAGE_SIZE,
+                                         PageTableMemoryType);
+
+                if (!KSUCCESS(Status)) {
+                    goto MapPhysicalAddressEnd;
+                }
+
+                RtlZeroMemory((PVOID)PageTablePhysical, PAGE_SIZE);
+                *PageTable =
+                        PageTablePhysical | X86_PTE_PRESENT | X86_PTE_WRITABLE;
+            }
+
+            PageTable = (PPTE)(X86_PTE_ENTRY(*PageTable));
+        }
+
+        //
+        // Set up the page.
+        //
+
+        PageTableIndex = (CurrentVirtual & X64_PTE_MASK) >> PAGE_SHIFT;
+        PageTable += PageTableIndex;
+        *PageTable = PhysicalAddress | X86_PTE_PRESENT;
+        if ((Attributes & MAP_FLAG_READ_ONLY) == 0) {
+            *PageTable |= X86_PTE_WRITABLE;
+        }
+
+        if ((Attributes & MAP_FLAG_USER_MODE) != 0) {
+            *PageTable |= X86_PTE_USER_MODE;
+        }
+
+        if ((Attributes & MAP_FLAG_WRITE_THROUGH) != 0) {
+            *PageTable |= X86_PTE_WRITE_THROUGH;
+        }
+
+        if ((Attributes & MAP_FLAG_CACHE_DISABLE) != 0) {
+            *PageTable |= X86_PTE_CACHE_DISABLED;
+        }
+
+        if ((Attributes & MAP_FLAG_GLOBAL) != 0) {
+            *PageTable |= X86_PTE_GLOBAL;
+        }
+
+        if ((Attributes & MAP_FLAG_EXECUTE) == 0) {
+            *PageTable |= X86_PTE_NX;
+        }
+
+        if (HugePage != FALSE) {
+            *PageTable |= X86_PTE_LARGE;
+            PagesMapped += _2MB / PAGE_SIZE;
+            PhysicalAddress += _2MB;
+            CurrentVirtual += _2MB;
+            HugePage = FALSE;
+            continue;
+        }
+
+        ASSERT((Attributes & MAP_FLAG_LARGE_PAGE) == 0);
+
+        PagesMapped += 1;
+        PhysicalAddress += PAGE_SIZE;
+        CurrentVirtual += PAGE_SIZE;
+    }
+
+    Status = STATUS_SUCCESS;
+
+MapPhysicalAddressEnd:
+    if ((!KSUCCESS(Status)) && (VirtualSpaceAllocated != FALSE)) {
+        BoUnmapPhysicalAddress((PVOID)MappedAddress, PageCount);
+    }
+
+    return Status;
 }
 
 KSTATUS
@@ -217,9 +464,74 @@ Return Value:
 
 {
 
-    ASSERT(FALSE);
+    UINTN CurrentVirtual;
+    ULONGLONG EndAddress;
+    PPTE PageTable;
+    ULONG PageTableIndex;
+    KSTATUS Status;
+    MEMORY_DESCRIPTOR VirtualSpace;
+
+    if (BoPageDirectory == NULL) {
+        return STATUS_NOT_INITIALIZED;
+    }
+
+    EndAddress = (ULONGLONG)(UINTN)VirtualAddress +
+                 ((ULONGLONG)PageCount << PAGE_SHIFT);
+
+    MmMdInitDescriptor(&VirtualSpace,
+                       (UINTN)VirtualAddress,
+                       EndAddress,
+                       MemoryTypeFree);
 
-    return STATUS_NOT_IMPLEMENTED;
+    Status = MmMdAddDescriptorToList(&BoVirtualMap, &VirtualSpace);
+    CurrentVirtual = (UINTN)VirtualAddress;
+    while (CurrentVirtual < EndAddress) {
+
+        //
+        // Get down to the lowest level page directory.
+        //
+
+        PageTable = BoPageDirectory;
+        PageTableIndex = (CurrentVirtual >> X64_PML4E_SHIFT) & X64_PT_MASK;
+        PageTable += PageTableIndex;
+        if ((*PageTable & X86_PTE_PRESENT) == 0) {
+            CurrentVirtual += PAGE_SIZE;
+            continue;
+        }
+
+        PageTable = (PPTE)(X86_PTE_ENTRY(*PageTable));
+        PageTableIndex = (CurrentVirtual >> X64_PDPE_SHIFT) & X64_PT_MASK;
+        PageTable += PageTableIndex;
+        if ((*PageTable & X86_PTE_PRESENT) == 0) {
+            CurrentVirtual += PAGE_SIZE;
+            continue;
+        }
+
+        PageTable = (PPTE)(X86_PTE_ENTRY(*PageTable));
+        PageTableIndex = (CurrentVirtual >> X64_PDE_SHIFT) & X64_PT_MASK;
+        PageTable += PageTableIndex;
+        if ((*PageTable & X86_PTE_PRESENT) == 0) {
+            CurrentVirtual += PAGE_SIZE;
+            continue;
+        }
+
+        if ((*PageTable & X86_PTE_LARGE) != 0) {
+
+            ASSERT((CurrentVirtual & (_2MB - 1)) == 0);
+            ASSERT((EndAddress - CurrentVirtual) >= _2MB);
+
+            *PageTable = 0;
+            CurrentVirtual += _2MB;
+            continue;
+        }
+
+        PageTable = (PPTE)(X86_PTE_ENTRY(*PageTable));
+        PageTableIndex = (CurrentVirtual >> X64_PTE_SHIFT) & X64_PT_MASK;
+        PageTable[PageTableIndex] = 0;
+        CurrentVirtual += PAGE_SIZE;
+    }
+
+    return Status;
 }
 
 VOID
@@ -253,7 +565,124 @@ Return Value:
 
 {
 
-    ASSERT(FALSE);
+    UINTN CurrentVirtual;
+    ULONGLONG EndAddress;
+    ULONG EntryShift;
+    ULONG Level;
+    ULONG NewAttributesMask;
+    PPTE PageTable;
+    UINTN PageTableIndex;
+
+    NewAttributesMask = (NewAttributes >> MAP_FLAG_PROTECT_SHIFT) &
+                        MAP_FLAG_PROTECT_MASK;
+
+    CurrentVirtual = (UINTN)VirtualAddress;
+    EndAddress = CurrentVirtual + Size;
+    while (CurrentVirtual < EndAddress) {
+
+        //
+        // Get down to the lowest level page table.
+        //
+
+        PageTable = BoPageDirectory;
+        EntryShift = X64_PML4E_SHIFT;
+        for (Level = 0; Level < X64_PAGE_LEVEL - 1; Level += 1) {
+            PageTableIndex = (CurrentVirtual >> EntryShift) & X64_PT_MASK;
+            PageTable += PageTableIndex;
+            EntryShift -= X64_PTE_BITS;
+            if ((*PageTable & X86_PTE_PRESENT) == 0) {
+                PageTable = NULL;
+                break;
+            }
+
+            //
+            // Also stop if a huge page was found. Consider adding some code
+            // to break apart a huge page if only part of it has attributes
+            // being modified.
+            //
+
+            if ((*PageTable & X86_PTE_LARGE) != 0) {
+
+                ASSERT(Level == X64_PAGE_LEVEL - 2);
+
+                if (((EndAddress - CurrentVirtual) < _2MB) ||
+                    ((CurrentVirtual & (_2MB - 1)) != 0)) {
+
+                    RtlDebugPrint("Skipping modification of huge page at "
+                                  "0x%llx because modification is only "
+                                  "0x%llx bytes.\n",
+                                  CurrentVirtual,
+                                  EndAddress - CurrentVirtual);
+
+                    PageTable = NULL;
+                }
+
+                break;
+            }
+
+            PageTable = (PPTE)(X86_PTE_ENTRY(*PageTable));
+        }
+
+        if (PageTable == NULL) {
+            CurrentVirtual += PAGE_SIZE;
+            continue;
+        }
+
+        PageTableIndex = (CurrentVirtual >> X64_PTE_SHIFT) & X64_PT_MASK;
+        PageTable += PageTableIndex;
+
+        //
+        // Look up the entry in the page table.
+        //
+
+        ASSERT((*PageTable & X86_PTE_PRESENT) != 0);
+
+        //
+        // Set the various attributes and set the entry.
+        //
+
+        if ((NewAttributesMask & MAP_FLAG_READ_ONLY) != 0) {
+            *PageTable &= ~X86_PTE_WRITABLE;
+            if ((NewAttributes & MAP_FLAG_READ_ONLY) == 0) {
+                *PageTable |= X86_PTE_WRITABLE;
+            }
+        }
+
+        if ((NewAttributesMask & MAP_FLAG_USER_MODE) != 0) {
+            *PageTable &= ~X86_PTE_USER_MODE;
+            if ((NewAttributes & MAP_FLAG_USER_MODE) != 0) {
+                *PageTable |= X86_PTE_USER_MODE;
+            }
+        }
+
+        if ((NewAttributesMask & MAP_FLAG_WRITE_THROUGH) != 0) {
+            *PageTable &= ~X86_PTE_WRITE_THROUGH;
+            if ((NewAttributes & MAP_FLAG_WRITE_THROUGH) != 0) {
+                *PageTable |= X86_PTE_WRITE_THROUGH;
+            }
+        }
+
+        if ((NewAttributesMask & MAP_FLAG_CACHE_DISABLE) != 0) {
+            *PageTable &= ~X86_PTE_CACHE_DISABLED;
+            if ((NewAttributes & MAP_FLAG_CACHE_DISABLE) != 0) {
+                *PageTable |= X86_PTE_CACHE_DISABLED;
+            }
+        }
+
+        if ((NewAttributesMask & MAP_FLAG_GLOBAL) != 0) {
+            *PageTable &= ~X86_PTE_GLOBAL;
+            if ((NewAttributes & MAP_FLAG_GLOBAL) != 0) {
+                *PageTable |= X86_PTE_GLOBAL;
+            }
+        }
+
+        if ((NewAttributesMask & MAP_FLAG_EXECUTE) != 0) {
+            *PageTable &= ~X86_PTE_NX;
+            if ((NewAttributes & MAP_FLAG_EXECUTE) == 0) {
+                *PageTable |= X86_PTE_NX;
+            }
+        }
+    }
 
     return;
 }
@@ -290,9 +719,14 @@ Return Value:
 
 {
 
-    ASSERT(FALSE);
+    //
+    // The self map location is hardcoded and already set up, so these aren't
+    // needed.
+    //
 
-    return STATUS_NOT_IMPLEMENTED;
+    *PageDirectoryVirtual = NULL;
+    *PageTablesVirtual = NULL;
+    return STATUS_SUCCESS;
 }
 
 KSTATUS
@@ -330,9 +764,45 @@ Return Value:
 
 {
 
-    ASSERT(FALSE);
+    ULONGLONG Address;
+    KSTATUS Status;
+    PPTE Table;
+    ULONG TableIndex;
+
+    *PageTableStage = NULL;
+
+    //
+    // "Map" the page table stage, which is really just done to set up a
+    // page table for it.
+    //
+
+    *PageTableStage = (PVOID)-1;
+    Status = BoMapPhysicalAddress(PageTableStage,
+                                  0,
+                                  PAGE_SIZE,
+                                  MAP_FLAG_READ_ONLY,
+                                  MemoryTypeLoaderPermanent);
+
+    if (!KSUCCESS(Status)) {
+        return Status;
+    }
 
-    return STATUS_NOT_IMPLEMENTED;
+    //
+    // Manually unmap the page. Don't use the unmap routine because that frees
+    // the region in the MDL, which isn't cool.
+    //
+
+    Address = (UINTN)*PageTableStage;
+    Table = BoPageDirectory;
+    TableIndex = (Address >> X64_PML4E_SHIFT) & X64_PT_MASK;
+    Table = (PPTE)(X86_PTE_ENTRY(Table[TableIndex]));
+    TableIndex = (Address >> X64_PDPE_SHIFT) & X64_PT_MASK;
+    Table = (PPTE)(X86_PTE_ENTRY(Table[TableIndex]));
+    TableIndex = (Address >> X64_PDE_SHIFT) & X64_PT_MASK;
+    Table = (PPTE)(X86_PTE_ENTRY(Table[TableIndex]));
+    TableIndex = (Address >> X64_PTE_SHIFT) & X64_PT_MASK;
+    Table[TableIndex] = 0;
+    return STATUS_SUCCESS;
 }
 
 //

+ 0 - 3
include/minoca/kernel/bootload.h

@@ -143,8 +143,6 @@ Members:
 
     PageDirectory - Stores the address of the top level page table in use.
 
-    PageTables - Stores the address of the page table self map.
-
     DriveNumber - Stores the drive number of the OS partition for legacy PC/AT
         systems.
 
@@ -171,7 +169,6 @@ typedef struct _BOOT_INITIALIZATION_BLOCK {
     ULONGLONG ApplicationSize;
     ULONGLONG ApplicationArguments;
     ULONGLONG PageDirectory;
-    ULONGLONG PageTables;
     ULONG DriveNumber;
     ULONG Flags;
 } BOOT_INITIALIZATION_BLOCK, *PBOOT_INITIALIZATION_BLOCK;

+ 24 - 1
include/minoca/kernel/mm.h

@@ -60,9 +60,24 @@ Author:
 #define NON_PAGED_POOL_MAGIC 0x506E6F4E
 #define PAGED_POOL_MAGIC 0x50676150
 
+//
+// Define the kernel address space. For 64-bit mode, leave a page at the end
+// to avoid rollover issues and to keep the space immediately underflowing NULL
+// clear.
+//
+
+#if __SIZEOF_LONG__ == 8
+
+#define KERNEL_VA_START (PVOID)0xFFF8000000000000
+#define KERNEL_VA_END 0xFFFFFFFFFFFFF000
+
+#else
+
 #define KERNEL_VA_START (PVOID)0x80000000
 #define KERNEL_VA_END 0x100000000ULL
 
+#endif
+
 #define INVALID_PHYSICAL_ADDRESS 0
 
 //
@@ -224,12 +239,20 @@ Author:
 
 //
 // Define the native sized user write function.
-// TODO: 64-bit.
 //
 
+#if __SIZEOF_LONG__ == 8
+
+#define MmUserWrite MmUserWrite64
+#define MmUserRead MmUserRead64
+
+#else
+
 #define MmUserWrite MmUserWrite32
 #define MmUserRead MmUserRead32
 
+#endif
+
 //
 // Define the bitmask of flags used to initialize or allocate an I/O buffer.
 //

+ 15 - 1
include/minoca/kernel/x64.h

@@ -47,7 +47,7 @@ Author:
 // and level 4 table.
 //
 
-#define X64_PTE_COUNT 512
+#define X64_PTE_COUNT 512ULL
 
 //
 // Define page address masks.
@@ -64,6 +64,20 @@ Author:
 #define X64_PML4E_SHIFT 39
 #define X64_PML4E_MASK (X64_PT_MASK << X64_PML4E_SHIFT)
 
+//
+// Define the fixed self map address. This is set up by the boot loader and
+// used directly by the kernel. The advantage is it's a compile-time constant
+// to get to page tables. The disadvantage is that VA can't be used by anything
+// else. But given that there's no physical memory up there and there's oodles
+// of VA space, that seems fine.
+//
+// Don't use the last index as that would put the PML4T at 0xFFFFFFFFFFFFF000.
+// Any kernel underflows from null would hit that page and be awful to debug.
+// Use the second to last index.
+//
+
+#define X64_SELF_MAP_INDEX (X64_PTE_COUNT - 2)
+
 #define X64_CANONICAL_HIGH 0xFFF8000000000000
 #define X64_CANONICAL_LOW  0x0007FFFFFFFFFFFF
 

+ 3 - 8
include/minoca/kernel/x86defs.h

@@ -251,6 +251,7 @@ Author:
 //
 
 #define X86_CPUID_EXTENDED_INFORMATION_EDX_SYSCALL (1 << 11)
+#define X86_CPUID_EXTENDED_INFORMATION_EDX_1GB_PAGES (1 << 26)
 #define X86_CPUID_EXTENDED_INFORMATION_EDX_LONG_MODE (1 << 29)
 
 //
@@ -341,14 +342,8 @@ Author:
      ((_TrapFrame)->Cs == USER_CS))
 
 //
-// This macro extracts the value out of a PTE (or PDE, etc).
+// This macro extracts the address ut of a PTE (or PDE, etc).
 //
 
-#define X86_PTE_ENTRY(_Pte) ((_Pte) >> X86_PTE_ENTRY_SHIFT)
-
-//
-// This macro creates a PTE (or PDE, etc) from a value.
-//
-
-#define X86_ENTRY_PTE(_Value) ((_Value) << X86_PTE_ENTRY_SHIFT)
+#define X86_PTE_ENTRY(_Pte) ((_Pte) & ~(PAGE_MASK | X86_PTE_NX))
 

+ 1 - 0
include/minoca/lib/types.h

@@ -79,6 +79,7 @@ Author:
 #define _128KB (128 * _1KB)
 #define _512KB (512 * _1KB)
 #define _1MB (1024 * _1KB)
+#define _2MB (2 * _1MB)
 #define _1GB (1024 * _1MB)
 #define _1TB (1024ULL * _1GB)