x86_64/MMU: Map all physical memory into the higher half instead of using recursive mapping

This still has a few problems. Notably, we are not using huge pages for this mapping, which would be a lot more efficient.
Right now, used memory is 8.1 MiB at boot for a 256MiB system.
But after improving it, this system will be much better than recursive mapping.
fork() will be MUCH easier to implement, for example.

kernel/src/arch/x86_64/MMU.cpp

@@ -1,6 +1,8 @@
 #include "arch/MMU.h"
 #include "Log.h"
 #include "memory/MemoryManager.h"
+#include "memory/MemoryMap.h"
+#include <luna/Alignment.h>
 #include <luna/CString.h>
 #include <luna/Result.h>
 #include <luna/ScopeGuard.h>
@@ -12,6 +14,10 @@
 PageDirectory* g_kernel_directory;
 u64 g_kernel_directory_virt;
 
+// The bootloader maps up to 16GiB of physical memory for us at address 0. Using this bootstrap mapping, we'll map (all)
+// physical memory at 0xFFFF800000000000.
+u64 g_physical_mapping_base = 0;
+
 void PageTableEntry::set_address(u64 addr)
 {
     this->address = (addr >> 12);
@@ -36,90 +42,9 @@ static bool has_flag(int flags, MMU::Flags flag)
 
 namespace MMU
 {
-
+    template <typename T> T translate_physical(T phys)
-    constexpr PageDirectory* l4_table()
     {
-        constexpr u64 l4 = sign | (rindex << 39) | (rindex << 30) | (rindex << 21) | (rindex << 12);
+        return (T)(g_physical_mapping_base + (u64)phys);
-        return (PageDirectory*)l4;
-    }
-
-    constexpr u64 l4_index(u64 addr)
-    {
-        return (addr >> 39) & 0777;
-    }
-
-    PageTableEntry& l4_entry(u64 addr)
-    {
-        return l4_table()->entries[l4_index(addr)];
-    }
-
-    constexpr PageDirectory* raw_l3_table(u64 l4)
-    {
-        const u64 l3 = sign | (rindex << 39) | (rindex << 30) | (rindex << 21) | (l4 << 12);
-        return (PageDirectory*)l3;
-    }
-
-    constexpr PageDirectory* l3_table(u64 addr)
-    {
-        const u64 l4 = l4_index(addr);
-        return raw_l3_table(l4);
-    }
-
-    constexpr u64 l3_index(u64 addr)
-    {
-        return (addr >> 30) & 0777;
-    }
-
-    PageTableEntry& l3_entry(u64 addr)
-    {
-        return l3_table(addr)->entries[l3_index(addr)];
-    }
-
-    constexpr PageDirectory* raw_l2_table(u64 l4, u64 l3)
-    {
-        const u64 l2 = sign | (rindex << 39) | (rindex << 30) | (l4 << 21) | (l3 << 12);
-        return (PageDirectory*)l2;
-    }
-
-    constexpr PageDirectory* l2_table(u64 addr)
-    {
-        const u64 l4 = l4_index(addr);
-        const u64 l3 = l3_index(addr);
-        return raw_l2_table(l4, l3);
-    }
-
-    constexpr u64 l2_index(u64 addr)
-    {
-        return (addr >> 21) & 0777;
-    }
-
-    PageTableEntry& l2_entry(u64 addr)
-    {
-        return l2_table(addr)->entries[l2_index(addr)];
-    }
-
-    constexpr PageDirectory* raw_l1_table(u64 l4, u64 l3, u64 l2)
-    {
-        const u64 l1 = sign | (rindex << 39) | (l4 << 30) | (l3 << 21) | (l2 << 12);
-        return (PageDirectory*)l1;
-    }
-
-    constexpr PageDirectory* l1_table(u64 addr)
-    {
-        const u64 l4 = l4_index(addr);
-        const u64 l3 = l3_index(addr);
-        const u64 l2 = l2_index(addr);
-        return raw_l1_table(l4, l3, l2);
-    }
-
-    constexpr u64 l1_index(u64 addr)
-    {
-        return (addr >> 12) & 0777;
-    }
-
-    PageTableEntry& l1_entry(u64 addr)
-    {
-        return l1_table(addr)->entries[l1_index(addr)];
     }
 
     void switch_page_directory(PageDirectory* dir)
@@ -134,6 +59,11 @@ namespace MMU
         return value;
     }
 
+    PageDirectory* get_virtual_page_directory()
+    {
+        return translate_physical(get_page_directory());
+    }
+
     void flush_all()
     {
         switch_page_directory(get_page_directory());
@@ -144,6 +74,26 @@ namespace MMU
         asm volatile("invlpg (%0)" : : "r"(page) : "memory");
     }
 
+    constexpr u64 l4_index(u64 addr)
+    {
+        return (addr >> 39) & 0777;
+    }
+
+    constexpr u64 l3_index(u64 addr)
+    {
+        return (addr >> 30) & 0777;
+    }
+
+    constexpr u64 l2_index(u64 addr)
+    {
+        return (addr >> 21) & 0777;
+    }
+
+    constexpr u64 l1_index(u64 addr)
+    {
+        return (addr >> 12) & 0777;
+    }
+
     int arch_flags_to_mmu(const PageTableEntry& entry)
     {
         int result = Flags::None;
@@ -155,17 +105,46 @@ namespace MMU
         return result;
     }
 
+    PageTableEntry& l4_entry(u64 virt)
+    {
+        auto index = l4_index(virt);
+        return get_virtual_page_directory()->entries[index];
+    }
+
+    PageDirectory& page_table(const PageTableEntry& entry)
+    {
+        return *translate_physical((PageDirectory*)entry.get_address());
+    }
+
+    PageTableEntry& l3_entry(const PageTableEntry& entry, u64 virt)
+    {
+        auto index = l3_index(virt);
+        return page_table(entry).entries[index];
+    }
+
+    PageTableEntry& l2_entry(const PageTableEntry& entry, u64 virt)
+    {
+        auto index = l2_index(virt);
+        return page_table(entry).entries[index];
+    }
+
+    PageTableEntry& l1_entry(const PageTableEntry& entry, u64 virt)
+    {
+        auto index = l1_index(virt);
+        return page_table(entry).entries[index];
+    }
+
     Result<PageTableEntry*> find_entry(u64 virt)
     {
         const auto& l4 = l4_entry(virt);
         if (!l4.present) return err(EFAULT);
-        auto& l3 = l3_entry(virt);
+        auto& l3 = l3_entry(l4, virt);
         if (!l3.present) return err(EFAULT);
         if (l3.larger_pages) return &l3;
-        auto& l2 = l2_entry(virt);
+        auto& l2 = l2_entry(l3, virt);
         if (!l2.present) return err(EFAULT);
         if (l2.larger_pages) return &l2;
-        return &l1_entry(virt);
+        return &l1_entry(l2, virt);
     }
 
     Result<PageTableEntry*> apply_cascading_flags(u64 virt, int flags)
@@ -174,17 +153,17 @@ namespace MMU
         if (!l4.present) return err(EFAULT);
         if (flags & Flags::ReadWrite) l4.read_write = true;
         if (flags & Flags::User) l4.user = true;
-        auto& l3 = l3_entry(virt);
+        auto& l3 = l3_entry(l4, virt);
         if (!l3.present) return err(EFAULT);
         if (l3.larger_pages) return &l3;
         if (flags & Flags::ReadWrite) l3.read_write = true;
         if (flags & Flags::User) l3.user = true;
-        auto& l2 = l2_entry(virt);
+        auto& l2 = l2_entry(l3, virt);
         if (!l2.present) return err(EFAULT);
         if (l2.larger_pages) return &l2;
         if (flags & Flags::ReadWrite) l2.read_write = true;
         if (flags & Flags::User) l2.user = true;
-        auto& l1 = l1_entry(virt);
+        auto& l1 = l1_entry(l2, virt);
         return &l1;
     }
 
@@ -196,38 +175,38 @@ namespace MMU
             const u64 addr = TRY(MemoryManager::alloc_frame());
             l4.present = true;
             l4.set_address(addr);
-            memset(l3_table(virt), 0, ARCH_PAGE_SIZE);
+            memset(&page_table(l4), 0, ARCH_PAGE_SIZE);
         }
         if (flags & Flags::ReadWrite) l4.read_write = true;
         if (flags & Flags::User) l4.user = true;
 
-        auto& l3 = l3_entry(virt);
+        auto& l3 = l3_entry(l4, virt);
         if (!l3.present)
         {
             const u64 addr = TRY(MemoryManager::alloc_frame());
             l3.present = true;
             l3.set_address(addr);
-            memset(l2_table(virt), 0, ARCH_PAGE_SIZE);
+            memset(&page_table(l3), 0, ARCH_PAGE_SIZE);
         }
         if (flags & Flags::ReadWrite) l3.read_write = true;
         if (flags & Flags::User) l3.user = true;
 
         if (l3.larger_pages) return err(EFIXME); // FIXME: Replacing larger pages is not supported ATM
 
-        auto& l2 = l2_entry(virt);
+        auto& l2 = l2_entry(l3, virt);
         if (!l2.present)
         {
             const u64 addr = TRY(MemoryManager::alloc_frame());
             l2.present = true;
             l2.set_address(addr);
-            memset(l1_table(virt), 0, ARCH_PAGE_SIZE);
+            memset(&page_table(l2), 0, ARCH_PAGE_SIZE);
         }
         if (flags & Flags::ReadWrite) l2.read_write = true;
         if (flags & Flags::User) l2.user = true;
 
         if (l2.larger_pages) return err(EFIXME); // FIXME: Replacing larger pages is not supported ATM
 
-        auto& l1 = l1_entry(virt);
+        auto& l1 = l1_entry(l2, virt);
         if (l1.present) return err(EEXIST); // Please explicitly unmap the page before mapping it again.
         l1.present = true;
         l1.read_write = has_flag(flags, Flags::ReadWrite);
@@ -281,39 +260,38 @@ namespace MMU
         PageDirectory* const dir = get_page_directory();
         g_kernel_directory = dir;
 
-        const u64 paddr = (u64)dir;
+        const u64 physical_memory_base = 0xFFFF800000000000;
-        PageTableEntry& recursive_entry = dir->entries[rindex];
-        recursive_entry.read_write = true;
-        recursive_entry.present = true;
-        recursive_entry.set_address(paddr);
-        flush_all();
 
-        g_kernel_directory_virt =
+        MemoryMapIterator iter;
-            MemoryManager::get_kernel_mapping_for_frames((u64)dir, 1, MMU::ReadWrite | MMU::NoExecute).value();
+        const MemoryMapEntry highest_entry = iter.highest();
+
+        const u64 physical_memory_size = highest_entry.address() + highest_entry.size();
+
+        // FIXME: Do this using 2MiB huge pages.
+        MemoryManager::map_frames_at(physical_memory_base, 0, physical_memory_size / ARCH_PAGE_SIZE,
+                                     MMU::ReadWrite | MMU::NoExecute);
+
+        g_physical_mapping_base = physical_memory_base;
+
+        g_kernel_directory_virt = translate_physical((u64)g_kernel_directory);
 
         kdbgln("MMU init page directory (ring0): virt %#.16lx, phys %p", g_kernel_directory_virt, g_kernel_directory);
     }
 
     Result<PageDirectory*> create_page_directory_for_userspace()
     {
-        const u64 directory_virt = TRY(MemoryManager::alloc_for_kernel(1, MMU::ReadWrite | MMU::NoExecute));
+        const u64 directory_phys = TRY(MemoryManager::alloc_frame());
-        const u64 directory_phys = MMU::get_physical(directory_virt).value();
+        const u64 directory_virt = translate_physical(directory_phys);
 
         PageDirectory* const directory = (PageDirectory*)directory_virt;
         memset(directory, 0, ARCH_PAGE_SIZE);
-        PageTableEntry& recursive_entry = directory->entries[rindex];
+
-        recursive_entry.read_write = true;
+        constexpr auto HALF_PAGE = ARCH_PAGE_SIZE / 2;
-        recursive_entry.present = true;
+        // Copy the upper part of the page directory (higher half)
-        recursive_entry.set_address(directory_phys);
+        memcpy(offset_ptr(directory, HALF_PAGE), offset_ptr(g_kernel_directory, HALF_PAGE), HALF_PAGE);
 
         kdbgln("MMU init page directory (ring3): virt %p, phys %#.16lx", directory, directory_phys);
 
-        directory->entries[511] = ((PageDirectory*)g_kernel_directory_virt)->entries[511];
-
-        // From now on, we're only going to use the physical address, since accessing the PageDirectory will be dealt
-        // with using recursive mapping. So let's make sure we don't leak any VM.
-        MemoryManager::unmap_weak_and_free_vm(directory_virt, 1);
-
         return (PageDirectory*)directory_phys;
     }
 
@@ -321,16 +299,11 @@ namespace MMU
     {
         check(directory);
 
-        // Needed in order to access page tables using the recursive mapping system.
+        switch_page_directory(g_kernel_directory);
-        switch_page_directory(directory);
 
-        auto guard = make_scope_guard([&] {
+        auto guard = make_scope_guard([directory] { MemoryManager::free_frame((u64)directory); });
-            check(g_kernel_directory);
-            switch_page_directory(g_kernel_directory);
-            MemoryManager::free_frame((u64)directory);
-        });
 
-        PageDirectory* const table = l4_table();
+        PageDirectory* const table = translate_physical(directory);
 
         // Let's iterate over every top-level entry, skipping the last two entries (recursive mapping and kernel pages)
         for (u64 i = 0; i < 510; i++)
@@ -338,7 +311,7 @@ namespace MMU
             PageTableEntry& l4 = table->entries[i];
             if (!l4.present) continue;
 
-            PageDirectory* const pdp = raw_l3_table(i);
+            PageDirectory* const pdp = &page_table(l4);
 
             for (u64 j = 0; j < 512; j++)
             {
@@ -350,7 +323,7 @@ namespace MMU
                     TRY(MemoryManager::free_frame(l3.get_address()));
                 }
 
-                PageDirectory* const pd = raw_l2_table(i, j);
+                PageDirectory* const pd = &page_table(l3);
 
                 for (u64 k = 0; k < 512; k++)
                 {
@@ -362,7 +335,7 @@ namespace MMU
                         TRY(MemoryManager::free_frame(l2.get_address()));
                     }
 
-                    PageDirectory* const pt = raw_l1_table(i, j, k);
+                    PageDirectory* const pt = &page_table(l2);
 
                     for (u64 l = 0; l < 512; l++)
                     {