Attempt to address virtual fragmentation issues.

1. Make heaps and block allocators more resourceful about expansion
failures by making multiple attempts to expand, backing off
logarithmically.
2. Make the MDL free bins contain 2 bits per bucket, for a smaller
structure and a range of 4k-64M rather than 4k-8M.
3. Have two levels of virtual memory warning for the top two MDL buckets.
4. When performing page cache virtual trimming, kill entries that have
backing entries, as those pin their backing entry VA. Also make sure
mapped page cache entries and backed page cache entries don't end up on
the clean unmapped list where they will never be unmapped by trim.

apps/osbase/heap.c

@@ -38,8 +38,7 @@ Environment:
 // Define the minimum heap expansion size.
 //
 
-#define SYSTEM_HEAP_MINIMUM_EXPANSION_SIZE (SYSTEM_HEAP_GRANULARITY * 10)
-#define SYSTEM_HEAP_GRANULARITY 4096
+#define SYSTEM_HEAP_MINIMUM_EXPANSION_PAGES 0x10
 #define SYSTEM_HEAP_MAGIC 0x6C6F6F50 // 'looP'
 
 //
@@ -82,6 +81,13 @@ OspHeapCorruption (
 MEMORY_HEAP OsHeap;
 OS_LOCK OsHeapLock;
 
+//
+// Store the native page shift and mask.
+//
+
+UINTN OsPageShift;
+UINTN OsPageSize;
+
 //
 // ------------------------------------------------------------------ Functions
 //
@@ -255,13 +261,15 @@ Return Value:
     ULONG Flags;
 
     OsInitializeLockDefault(&OsHeapLock);
+    OsPageSize = OsEnvironment->StartData->PageSize;
+    OsPageShift = RtlCountTrailingZeros(OsPageSize);
     Flags = MEMORY_HEAP_FLAG_NO_PARTIAL_FREES;
     RtlHeapInitialize(&OsHeap,
                       OspHeapExpand,
                       OspHeapContract,
                       OspHeapCorruption,
-                      SYSTEM_HEAP_MINIMUM_EXPANSION_SIZE,
-                      SYSTEM_HEAP_GRANULARITY,
+                      SYSTEM_HEAP_MINIMUM_EXPANSION_PAGES << OsPageShift,
+                      OsPageSize,
                       SYSTEM_HEAP_MAGIC,
                       Flags);
 

apps/osbase/osimag.c

@@ -220,13 +220,6 @@ OspImReleaseModuleNumber (
 
 OS_API PIM_GET_ENVIRONMENT_VARIABLE OsImGetEnvironmentVariable;
 
-//
-// Store the page shift and mask for easy use during image section mappings.
-//
-
-UINTN OsPageShift;
-UINTN OsPageSize;
-
 //
 // Store a pointer to the list head of all loaded images.
 //
@@ -782,8 +775,6 @@ Return Value:
 {
 
     OsInitializeLockDefault(&OsLoadedImagesLock);
-    OsPageSize = OsEnvironment->StartData->PageSize;
-    OsPageShift = RtlCountTrailingZeros(OsPageSize);
     ImInitialize(&OsImageFunctionTable);
     return;
 }

include/minoca/kernel/mm.h

@@ -78,7 +78,7 @@ Author:
 // Define the number of bins MDLs keep for free descriptors.
 //
 
-#define MDL_BIN_COUNT 12
+#define MDL_BIN_COUNT 8
 
 //
 // Define the minimum amount of free system descriptors that need to be

kernel/io/irp.c

@@ -1423,11 +1423,14 @@ Return Value:
 
     IoIrp->MinorCode = MinorCodeNumber;
     RtlCopyMemory(&(IoIrp->U.ReadWrite), Request, sizeof(IRP_READ_WRITE));
+    IoIrp->U.ReadWrite.IoBufferState.IoBuffer = NULL;
     Status = IoSendSynchronousIrp(IoIrp);
     if (!KSUCCESS(Status)) {
         goto SendIoIrpEnd;
     }
 
+    ASSERT(IoIrp->U.ReadWrite.IoBufferState.IoBuffer == NULL);
+
     RtlCopyMemory(Request, &(IoIrp->U.ReadWrite), sizeof(IRP_READ_WRITE));
     if (Device->Header.Type == ObjectDevice) {
         if (MinorCodeNumber == IrpMinorIoWrite) {

kernel/io/pagecach.c

@@ -821,6 +821,15 @@ Return Value:
         RtlAtomicAdd(&IoPageCacheMappedPageCount, 1);
         if ((OldFlags & PAGE_CACHE_ENTRY_FLAG_DIRTY) != 0) {
             RtlAtomicAdd(&IoPageCacheMappedDirtyPageCount, 1);
+
+        } else {
+
+            //
+            // If it wasn't dirty, it may need to be moved from the
+            // clean-unmapped list to the clean list.
+            //
+
+            IopUpdatePageCacheEntryList(UnmappedEntry, FALSE);
         }
     }
 
@@ -1994,15 +2003,6 @@ Return Value:
                 Status = STATUS_TRY_AGAIN;
                 goto FlushPageCacheEntriesEnd;
             }
-
-            if ((IopIsPageCacheTooMapped(NULL) != FALSE) &&
-                ((IoPageCacheMappedPageCount -
-                  IoPageCacheMappedDirtyPageCount) >
-                 IoPageCacheLowMemoryCleanPageMinimum)) {
-
-                Status = STATUS_TRY_AGAIN;
-                goto FlushPageCacheEntriesEnd;
-            }
         }
     }
 
@@ -2933,10 +2933,19 @@ Return Value:
                 if ((OldFlags & PAGE_CACHE_ENTRY_FLAG_DIRTY) != 0) {
                     RtlAtomicAdd(&IoPageCacheMappedDirtyPageCount, 1);
                 }
+
+                //
+                // The entry was just used, and may need to come off the clean
+                // unmapped list.
+                //
+
+                IopUpdatePageCacheEntryList(LowerEntry, FALSE);
             }
         }
     }
 
+    IopUpdatePageCacheEntryList(UpperEntry, FALSE);
+
     //
     // Now link the two entries based on their types. Note that nothing should
     // have been able to sneak in and link them since the caller has a
@@ -3393,6 +3402,8 @@ Return Value:
     ASSERT((PageCacheEntry->ReferenceCount == 0) ||
            (PageCacheEntry->Node.Parent == NULL));
 
+    ASSERT(PageCacheEntry->ListEntry.Next == NULL);
+
     //
     // If this is the page owner, then free the physical page.
     //
@@ -4298,14 +4309,20 @@ Return Value:
 
 {
 
+    PPAGE_CACHE_ENTRY BackingEntry;
     PLIST_ENTRY CurrentEntry;
+    UINTN DestroyCount;
+    BOOL DestroyEntry;
+    LIST_ENTRY DestroyList;
     PFILE_OBJECT FileObject;
     UINTN FreeVirtualPages;
     PSHARED_EXCLUSIVE_LOCK Lock;
     UINTN MappedCleanPageCount;
     PPAGE_CACHE_ENTRY PageCacheEntry;
     ULONG PageSize;
+    BOOL PageWasDirty;
     LIST_ENTRY ReturnList;
+    KSTATUS Status;
     UINTN TargetUnmapCount;
     UINTN UnmapCount;
     UINTN UnmapSize;
@@ -4322,6 +4339,7 @@ Return Value:
 
     ASSERT(FreeVirtualPages != -1);
 
+    INITIALIZE_LIST_HEAD(&DestroyList);
     INITIALIZE_LIST_HEAD(&ReturnList);
 
     //
@@ -4379,6 +4397,7 @@ Return Value:
     UnmapStart = NULL;
     UnmapSize = 0;
     UnmapCount = 0;
+    DestroyCount = 0;
     PageSize = MmPageSize();
     KeAcquireQueuedLock(IoPageCacheListLock);
     while ((!LIST_EMPTY(&IoPageCacheCleanList)) &&
@@ -4425,12 +4444,16 @@ Return Value:
         }
 
         //
-        // If the page was not mapped, move it over to the clean unmapped list
-        // to prevent iterating over it again during subsequent invocations of
-        // this function.
+        // If the page was not mapped, and is the page owner, move it over to
+        // the clean unmapped list to prevent iterating over it again during
+        // subsequent invocations of this function.
         //
 
-        if (PageCacheEntry->VirtualAddress == NULL) {
+        if ((PageCacheEntry->Flags &
+             (PAGE_CACHE_ENTRY_FLAG_MAPPED |
+              PAGE_CACHE_ENTRY_FLAG_PAGE_OWNER)) ==
+            PAGE_CACHE_ENTRY_FLAG_PAGE_OWNER) {
+
             LIST_REMOVE(&(PageCacheEntry->ListEntry));
             INSERT_BEFORE(&(PageCacheEntry->ListEntry),
                           &IoPageCacheCleanUnmappedList);
@@ -4496,22 +4519,80 @@ Return Value:
             UnmapSize += PageSize;
         }
 
+        //
+        // If there's a backing entry and no references besides this one, try
+        // to destroy this entry, as it pins the backing entry VA. Note that
+        // new references can still come in via the list. It can't be removed
+        // from the list because new references could come in from the tree and
+        // put it back on the list.
+        //
+
+        DestroyEntry = FALSE;
+        BackingEntry = PageCacheEntry->BackingEntry;
+        if ((BackingEntry != NULL) && (PageCacheEntry->ReferenceCount == 1)) {
+            Status = IopUnmapPageCacheEntrySections(PageCacheEntry,
+                                                    &PageWasDirty);
+
+            if (KSUCCESS(Status)) {
+                if (PageWasDirty != FALSE) {
+                    IopMarkPageCacheEntryDirty(PageCacheEntry);
+                }
+
+                if (PageCacheEntry->Node.Parent != NULL) {
+                    IopRemovePageCacheEntryFromTree(PageCacheEntry);
+                }
+
+                DestroyEntry = TRUE;
+            }
+        }
+
         //
         // Drop the file object lock and reacquire the list lock.
         //
 
         KeReleaseSharedExclusiveLockExclusive(FileObject->Lock);
         KeAcquireQueuedLock(IoPageCacheListLock);
-        if ((PageCacheEntry->Flags & PAGE_CACHE_ENTRY_FLAG_DIRTY) == 0) {
+        if (DestroyEntry != FALSE) {
             if (PageCacheEntry->ListEntry.Next != NULL) {
                 LIST_REMOVE(&(PageCacheEntry->ListEntry));
+                PageCacheEntry->ListEntry.Next = NULL;
             }
 
-            INSERT_BEFORE(&(PageCacheEntry->ListEntry),
-                          &IoPageCacheCleanUnmappedList);
-        }
+            //
+            // If another list cruiser got to it first, stick it on the removal
+            // list (since it's no longer in the tree), and don't destroy it.
+            //
 
-        IoPageCacheEntryReleaseReference(PageCacheEntry);
+            if (PageCacheEntry->ReferenceCount != 1) {
+                INSERT_BEFORE(&(PageCacheEntry->ListEntry),
+                              &IoPageCacheRemovalList);
+
+            } else {
+                PageCacheEntry->ReferenceCount = 0;
+                INSERT_BEFORE(&(PageCacheEntry->ListEntry), &DestroyList);
+                DestroyCount += 1;
+            }
+
+        } else {
+            if ((PageCacheEntry->Flags & PAGE_CACHE_ENTRY_FLAG_DIRTY) == 0) {
+                if (PageCacheEntry->ListEntry.Next != NULL) {
+                    LIST_REMOVE(&(PageCacheEntry->ListEntry));
+                }
+
+                if (((PageCacheEntry->Flags &
+                      PAGE_CACHE_ENTRY_FLAG_MAPPED) == 0) &&
+                    (PageCacheEntry->BackingEntry == NULL)) {
+
+                    INSERT_BEFORE(&(PageCacheEntry->ListEntry),
+                                  &IoPageCacheCleanUnmappedList);
+
+                } else {
+                    INSERT_BEFORE(&(PageCacheEntry->ListEntry), &ReturnList);
+                }
+            }
+
+            IoPageCacheEntryReleaseReference(PageCacheEntry);
+        }
     }
 
     //
@@ -4539,8 +4620,14 @@ Return Value:
         RtlAtomicAdd(&IoPageCacheMappedPageCount, -UnmapCount);
     }
 
+    if (!LIST_EMPTY(&DestroyList)) {
+        IopDestroyPageCacheEntries(&DestroyList);
+    }
+
     if ((IoPageCacheDebugFlags & PAGE_CACHE_DEBUG_MAPPED_MANAGEMENT) != 0) {
-        RtlDebugPrint("PAGE CACHE: Unmapped %lu entries.\n", UnmapCount);
+        RtlDebugPrint("PAGE CACHE: Unmapped %lu entries, destroyed %lu.\n",
+                      UnmapCount,
+                      DestroyCount);
     }
 
     return;

kernel/mm/block.c

@@ -869,6 +869,8 @@ Return Value:
 
     //
     // Try doubling the previous expansion. Don't go below the minimum size.
+    // Keep dividing by two until either the something is found or the minimum
+    // is hit.
     //
 
     ExpansionSize = Allocator->PreviousExpansionBlockCount << 1;
@@ -876,26 +878,17 @@ Return Value:
         ExpansionSize = Allocator->ExpansionBlockCount;
     }
 
-    Status = MmpExpandBlockAllocatorBySize(Allocator,
-                                           AllocatorLockHeld,
-                                           ExpansionSize);
+    Status = STATUS_INVALID_PARAMETER;
+    while (ExpansionSize >= Allocator->ExpansionBlockCount) {
+        Status = MmpExpandBlockAllocatorBySize(Allocator,
+                                               AllocatorLockHeld,
+                                               ExpansionSize);
 
-    if (!KSUCCESS(Status)) {
-
-        //
-        // If that didn't work, try allocating the minimum.
-        //
-
-        if (ExpansionSize > Allocator->ExpansionBlockCount) {
-            ExpansionSize = Allocator->ExpansionBlockCount;
-            Status = MmpExpandBlockAllocatorBySize(Allocator,
-                                                   AllocatorLockHeld,
-                                                   ExpansionSize);
-
-            if (!KSUCCESS(Status)) {
-                ExpansionSize = 0;
-            }
+        if (KSUCCESS(Status)) {
+            break;
         }
+
+        ExpansionSize >>= 1;
     }
 
     Allocator->PreviousExpansionBlockCount = ExpansionSize;

kernel/mm/iobuf.c

@@ -364,6 +364,7 @@ Return Value:
 
 AllocateIoBufferEnd:
     if (!KSUCCESS(Status)) {
+        RtlDebugPrint("MmAllocateNonPagedIoBuffer(%x): %x\n", Size, Status);
         if (VirtualAddress != NULL) {
             UnmapFlags = UNMAP_FLAG_FREE_PHYSICAL_PAGES |
                          UNMAP_FLAG_SEND_INVALIDATE_IPI;
@@ -1223,10 +1224,6 @@ Return Value:
                                              MapFragmentStart,
                                              FragmentCount,
                                              MapFlags);
-
-            if (!KSUCCESS(Status)) {
-                goto MapIoBufferEnd;
-            }
         }
 
         //

kernel/mm/kpools.c

@@ -31,7 +31,7 @@ Environment:
 // ---------------------------------------------------------------- Definitions
 //
 
-#define MINIMUM_POOL_EXPANSION_COUNT 0x80
+#define MINIMUM_POOL_EXPANSION_PAGES 0x10
 
 //
 // Define the initial non-paged pool size needed to successfully bootstrap the
@@ -820,7 +820,7 @@ Return Value:
     //
 
     PageSize = MmPageSize();
-    MinimumExpansionSize = MINIMUM_POOL_EXPANSION_COUNT * PageSize;
+    MinimumExpansionSize = MINIMUM_POOL_EXPANSION_PAGES * PageSize;
     Flags = DEFAULT_NON_PAGED_POOL_MEMORY_HEAP_FLAGS;
     RtlHeapInitialize(&MmNonPagedPool,
                       MmpExpandNonPagedPool,
@@ -880,7 +880,7 @@ Return Value:
     ULONG PageSize;
 
     PageSize = MmPageSize();
-    MinimumExpansionSize = MINIMUM_POOL_EXPANSION_COUNT * PageSize;
+    MinimumExpansionSize = MINIMUM_POOL_EXPANSION_PAGES * PageSize;
 
     //
     // The paged pool does not support partial frees because image sections

kernel/mm/mdl.c

@@ -45,6 +45,12 @@ Environment:
 
 #define MDL_BIN_SHIFT 12
 
+//
+// Define the number of bits per bin.
+//
+
+#define MDL_BITS_PER_BIN 2
+
 //
 // ----------------------------------------------- Internal Function Prototypes
 //
@@ -1979,6 +1985,7 @@ Return Value:
     BinIndex = ((UINTN)sizeof(ULONGLONG) * BITS_PER_BYTE) - 1 -
                RtlCountLeadingZeros64(BinSize);
 
+    BinIndex >>= MDL_BITS_PER_BIN - 1;
     if (BinIndex >= MDL_BIN_COUNT) {
         BinIndex = MDL_BIN_COUNT - 1;
     }

kernel/mm/virtual.c

@@ -1747,8 +1747,10 @@ Return Value:
 
 {
 
+    PLIST_ENTRY BinList;
     BOOL LockAcquired;
     MEMORY_DESCRIPTOR NewDescriptor;
+    MEMORY_WARNING_LEVEL NewWarning;
     BOOL RangeFree;
     BOOL SignalEvent;
     KSTATUS Status;
@@ -1852,13 +1854,24 @@ AllocateAddressRangeEnd:
         // (indicating serious fragmentation), then throw a warning up.
         //
 
+        BinList = &(Accountant->Mdl.FreeLists[MDL_BIN_COUNT - 1]);
         if ((Accountant->Mdl.FreeSpace <
              MmVirtualMemoryWarningLevel1Trigger) ||
-            (LIST_EMPTY(&(Accountant->Mdl.FreeLists[MDL_BIN_COUNT - 1])))) {
+            (LIST_EMPTY(BinList))) {
 
-            if (MmVirtualMemoryWarningLevel == MemoryWarningLevelNone) {
-                MmVirtualMemoryWarningLevel = MemoryWarningLevel1;
-                SignalEvent = TRUE;
+            if (MmVirtualMemoryWarningLevel < MemoryWarningLevel2) {
+                BinList = &(Accountant->Mdl.FreeLists[MDL_BIN_COUNT - 2]);
+                if (LIST_EMPTY(BinList)) {
+                    NewWarning = MemoryWarningLevel2;
+
+                } else {
+                    NewWarning = MemoryWarningLevel1;
+                }
+
+                if (MmVirtualMemoryWarningLevel != NewWarning) {
+                    MmVirtualMemoryWarningLevel = NewWarning;
+                    SignalEvent = TRUE;
+                }
             }
         }
     }

kernel/sp/profiler.c

@@ -258,7 +258,7 @@ SppCollectThreadStatistic (
 
 //
 // Stores a value indicating whether or not profiling is enabled for system
-// initialization. Can be set with PROFILER_TYPE_FLAGS_* values.
+// initialization. Can be set with PROFILER_TYPE_FLAG_* values.
 //
 
 ULONG SpEarlyEnabledFlags = 0x0;

lib/fatlib/fat.c

@@ -2717,6 +2717,9 @@ Return Value:
     }
 
     if (DestinationOffset > MAX_ULONG) {
+
+        ASSERT(FALSE);
+
         Status = STATUS_OUT_OF_BOUNDS;
         goto FatFileSeekEnd;
     }

lib/rtl/base/heap.c

@@ -1811,15 +1811,16 @@ Return Value:
     UINTN ReplacementSize;
     PHEAP_SEGMENT Segment;
 
+    if (Heap->AllocateFunction == NULL) {
+        return NULL;
+    }
+
     //
     // Directly allocate large chunks, but only if the heap is already
     // initialized.
     //
 
-    if ((Heap->AllocateFunction != NULL) &&
-        (Size >= Heap->DirectAllocationThreshold) &&
-        (Heap->TopSize != 0)) {
-
+    if ((Size >= Heap->DirectAllocationThreshold) && (Heap->TopSize != 0)) {
         Memory = RtlpHeapAllocateDirect(Heap, Size, Tag);
         if (Memory != NULL) {
             return Memory;
@@ -1869,19 +1870,18 @@ Return Value:
     }
 
     //
-    // Ask the system for more memory. If the doubling effort failed, fall
-    // back to the original aligned size.
+    // Ask the system for more memory. If the doubling effort failed, divide by
+    // 2 until something is found or the minimum fails as well.
     //
 
     Memory = NULL;
-    if (Heap->AllocateFunction != NULL) {
+    while (AlignedSize >= ConservativeAlignedSize) {
         Memory = Heap->AllocateFunction(Heap, AlignedSize, Heap->AllocationTag);
-        if (Memory == NULL) {
-            AlignedSize = ConservativeAlignedSize;
-            Memory = Heap->AllocateFunction(Heap,
-                                            AlignedSize,
-                                            Heap->AllocationTag);
+        if (Memory != NULL) {
+            break;
         }
+
+        AlignedSize >>= 1;
     }
 
     if (Memory != NULL) {