diff --git a/kernel/src/memory/UserVM.cpp b/kernel/src/memory/UserVM.cpp index a28c0772..0a5aa440 100644 --- a/kernel/src/memory/UserVM.cpp +++ b/kernel/src/memory/UserVM.cpp @@ -5,118 +5,197 @@ #include #include -static constexpr u64 VM_BASE = 0x10000000; - -static constexpr usize INITIAL_VM_SIZE = 80; -static constexpr usize MAX_VM_SIZE = 1024 * 1024 * 16; +static constexpr u64 VM_START = ARCH_PAGE_SIZE; +static constexpr u64 VM_END = 0x0000800000000000; Result> UserVM::try_create() { - void* const base = TRY(kmalloc(INITIAL_VM_SIZE)); + OwnedPtr ptr = TRY(make_owned()); - auto guard = make_scope_guard([&] { kfree(base); }); - - OwnedPtr ptr = TRY(make_owned(base, INITIAL_VM_SIZE)); - - guard.deactivate(); + TRY(ptr->create_null_region()); + TRY(ptr->create_default_region()); return move(ptr); } +Result UserVM::create_null_region() +{ + // Create a small region at the start of the address space to prevent anyone from mapping page 0. + auto* region = TRY(make()); + region->start = 0; + region->end = VM_START; + region->count = 1; + region->used = true; + region->persistent = true; + m_regions.append(region); + return {}; +} + +Result UserVM::create_default_region() +{ + // Create a free region covering the rest of the address space. + auto* region = TRY(make()); + region->start = VM_START; + region->end = VM_END; + region->count = (VM_END / ARCH_PAGE_SIZE) - 1; + region->used = false; + m_regions.append(region); + return {}; +} + Result> UserVM::clone() { - void* const base = TRY(kmalloc(m_bitmap.size_in_bytes())); + OwnedPtr ptr = TRY(make_owned()); - auto guard = make_scope_guard([&] { kfree(base); }); - - OwnedPtr ptr = TRY(make_owned(base, m_bitmap.size_in_bytes())); - - memcpy(ptr->m_bitmap.location(), m_bitmap.location(), m_bitmap.size_in_bytes()); - - guard.deactivate(); + for (const auto* region : m_regions) + { + auto* copied_region = TRY(make()); + copied_region->start = region->start; + copied_region->end = region->end; + copied_region->count = region->count; + copied_region->used = region->used; + copied_region->persistent = region->persistent; + ptr->m_regions.append(copied_region); + } return move(ptr); } -UserVM::UserVM(void* base, usize size) +UserVM::UserVM() { - m_bitmap.initialize(base, size); - m_bitmap.clear(false); } -Result UserVM::try_expand(usize size) +Result UserVM::alloc_region(usize count, bool persistent) { - if (m_bitmap.size_in_bytes() == MAX_VM_SIZE) { return false; } - - const usize old_size = m_bitmap.size_in_bytes(); - usize new_size = old_size + size; - - if (new_size > MAX_VM_SIZE) new_size = MAX_VM_SIZE; - - m_bitmap.resize(new_size); - m_bitmap.clear_region(old_size * 8, (new_size - old_size) * 8, false); - - return true; -} - -Result UserVM::alloc_one_page() -{ - u64 index; - bool ok = m_bitmap.find_and_toggle(false).try_set_value(index); - if (!ok) + for (auto* region = m_regions.expect_last(); region; region = m_regions.previous(region).value_or(nullptr)) { - bool success = TRY(try_expand()); - if (!success) return err(ENOMEM); - index = TRY(Result::from_option(m_bitmap.find_and_toggle(false), ENOMEM)); + if (!region->used) + { + if (region->count < count) continue; + if (region->count == count) + { + region->used = true; + region->persistent = persistent; + u64 address = region->start; + try_merge_region_with_neighbors(region); + return address; + } + + u64 boundary = region->end - (count * ARCH_PAGE_SIZE); + + auto* new_region = TRY(split_region(region, boundary)); + new_region->used = true; + new_region->persistent = persistent; + try_merge_region_with_neighbors(new_region); + + return boundary; + } } - return VM_BASE + index * ARCH_PAGE_SIZE; + return err(ENOMEM); } -Result UserVM::alloc_several_pages(usize count) +Result UserVM::set_region(u64 address, usize count, bool used) { - u64 index; - bool ok = m_bitmap.find_and_toggle_region(false, count).try_set_value(index); - if (!ok) + if (address >= VM_END) return err(EINVAL); + + u64 end = address + (count * ARCH_PAGE_SIZE); + + for (auto* region : m_regions) { - bool success = TRY(try_expand((count / 8) + INITIAL_VM_SIZE)); - if (!success) return err(ENOMEM); - index = TRY(Result::from_option(m_bitmap.find_and_toggle_region(false, count), ENOMEM)); + if (region->end < address) continue; + if (region->start > end) return false; + + if (region->persistent) return false; + if (region->used == used) + { + if (used) return false; + continue; + } + + if (region->start >= address && region->end <= end) + { + region->used = used; + if (region->start == address && region->end == end) + { + try_merge_region_with_neighbors(region); + return true; + } + continue; + } + + if (region->end > end && region->start < address) + { + auto* middle_region = TRY(split_region(region, address)); + TRY(split_region(middle_region, end)); + middle_region->used = used; + return true; + } + + if (region->start < address) + { + bool finished = region->end == end; + auto* split = TRY(split_region(region, address)); + split->used = used; + try_merge_region_with_neighbors(split); + if (!finished) continue; + return true; + } + + if (region->end > end) + { + TRY(split_region(region, end)); + region->used = used; + try_merge_region_with_neighbors(region); + return true; + } } - return VM_BASE + index * ARCH_PAGE_SIZE; -} - -Result UserVM::free_one_page(u64 address) -{ - if (address < VM_BASE) return err(EINVAL); - const u64 index = (address - VM_BASE) / ARCH_PAGE_SIZE; - if (index > (MAX_VM_SIZE * 8)) return err(EINVAL); - - // NOTE: POSIX says munmap() should silently do nothing if the address is not mapped, instead of throwing an error - // like EFAULT. - if (!m_bitmap.get(index)) return false; - - m_bitmap.set(index, false); - return true; } -Result UserVM::free_several_pages(u64 address, usize count) +void UserVM::merge_contiguous_regions(VMRegion* a, VMRegion* b) { - if (address < VM_BASE) return err(EINVAL); - const u64 index = (address - VM_BASE) / ARCH_PAGE_SIZE; - if ((index + count) > (MAX_VM_SIZE * 8)) return err(EINVAL); + a->end = b->end; + a->count += b->count; + m_regions.remove(b); + delete b; +} - // NOTE: Same as above. - if (!TRY(m_bitmap.try_match_region(index, count, true))) return false; +void UserVM::try_merge_region_with_neighbors(VMRegion* region) +{ + auto prev = m_regions.previous(region); + if (prev.has_value() && (*prev)->used == region->used && (*prev)->persistent == region->persistent) + { + merge_contiguous_regions(*prev, region); + region = *prev; + } - m_bitmap.clear_region(index, count, false); + auto next = m_regions.next(region); + if (next.has_value() && (*next)->used == region->used && (*next)->persistent == region->persistent) + { + merge_contiguous_regions(region, *next); + } +} - return true; +Result UserVM::split_region(VMRegion* parent, u64 boundary) +{ + auto* region = TRY(make()); + + region->start = boundary; + region->end = parent->end; + region->count = (region->end - region->start) / ARCH_PAGE_SIZE; + region->used = parent->used; + region->persistent = parent->persistent; + m_regions.add_after(parent, region); + + parent->end = boundary; + parent->count -= region->count; + + return region; } UserVM::~UserVM() { - m_bitmap.deallocate(); + m_regions.consume([](VMRegion* region) { delete region; }); } diff --git a/kernel/src/memory/UserVM.h b/kernel/src/memory/UserVM.h index 3e9342be..5f9baa29 100644 --- a/kernel/src/memory/UserVM.h +++ b/kernel/src/memory/UserVM.h @@ -1,25 +1,46 @@ #pragma once -#include +#include #include #include +class VMRegion : LinkedListNode +{ + public: + u64 start; + u64 end; + usize count; + bool used { true }; + bool persistent { false }; +}; + class UserVM { public: - UserVM(void* base, usize size); + UserVM(); ~UserVM(); - Result alloc_one_page(); - Result alloc_several_pages(usize count); + Result alloc_region(usize count, bool persistent = false); - Result free_one_page(u64 address); - Result free_several_pages(u64 address, usize count); + Result test_and_alloc_region(u64 address, usize count) + { + return set_region(address, count, true); + } + + Result free_region(u64 address, usize count) + { + return set_region(address, count, false); + } static Result> try_create(); Result> clone(); private: - Result try_expand(usize size = 160); - Bitmap m_bitmap; + Result set_region(u64 address, usize count, bool used); + Result create_default_region(); + Result create_null_region(); + void try_merge_region_with_neighbors(VMRegion* region); + void merge_contiguous_regions(VMRegion* a, VMRegion* b); + Result split_region(VMRegion* parent, u64 boundary); + LinkedList m_regions; }; diff --git a/kernel/src/sys/mmap.cpp b/kernel/src/sys/mmap.cpp index 46f8d5f7..66c89fc4 100644 --- a/kernel/src/sys/mmap.cpp +++ b/kernel/src/sys/mmap.cpp @@ -37,11 +37,13 @@ Result sys_mmap(Registers*, SyscallArgs args) Thread* current = Scheduler::current(); u64 address; - if (!addr) address = TRY(current->vm_allocator->alloc_several_pages(get_blocks_from_size(len, ARCH_PAGE_SIZE))); + if (!addr) address = TRY(current->vm_allocator->alloc_region(get_blocks_from_size(len, ARCH_PAGE_SIZE))); else { - kwarnln("mmap: FIXME: tried to mmap at a given address, instead of letting us choose"); - return err(ENOTSUP); + // FIXME: We should be more flexible if MAP_FIXED was not specified. + if (!TRY(current->vm_allocator->test_and_alloc_region((u64)addr, get_blocks_from_size(len, ARCH_PAGE_SIZE)))) + return err(ENOMEM); + address = (u64)addr; } int mmu_flags = MMU::User | MMU::NoExecute; @@ -53,6 +55,7 @@ Result sys_mmap(Registers*, SyscallArgs args) kdbgln("mmap: mapping memory at %#lx, size=%zu", address, len); #endif + // FIXME: This leaks VM if it fails. return MemoryManager::alloc_at_zeroed(address, get_blocks_from_size(len, ARCH_PAGE_SIZE), mmu_flags); } @@ -66,7 +69,7 @@ Result sys_munmap(Registers*, SyscallArgs args) Thread* current = Scheduler::current(); - bool ok = TRY(current->vm_allocator->free_several_pages(address, get_blocks_from_size(size, ARCH_PAGE_SIZE))); + bool ok = TRY(current->vm_allocator->free_region(address, get_blocks_from_size(size, ARCH_PAGE_SIZE))); // POSIX says munmap should silently do nothing if the memory was not already mapped. if (!ok) return 0; diff --git a/kernel/src/thread/ELF.cpp b/kernel/src/thread/ELF.cpp index b600ca98..db0d870f 100644 --- a/kernel/src/thread/ELF.cpp +++ b/kernel/src/thread/ELF.cpp @@ -25,7 +25,7 @@ static bool can_write_segment(u32 flags) namespace ELFLoader { - Result load(SharedPtr inode) + Result load(SharedPtr inode, UserVM* vm) { Elf64_Ehdr elf_header; usize nread = TRY(inode->read((u8*)&elf_header, 0, sizeof elf_header)); @@ -100,6 +100,11 @@ namespace ELFLoader if (can_write_segment(program_header.p_flags)) flags |= MMU::ReadWrite; if (can_execute_segment(program_header.p_flags)) flags &= ~MMU::NoExecute; + // FIXME: Set this memory range to persistent so that munmap() cannot remove it. + if (!TRY(vm->test_and_alloc_region( + base_vaddr, get_blocks_from_size(program_header.p_memsz + vaddr_diff, ARCH_PAGE_SIZE)))) + return err(ENOMEM); + // Allocate physical memory for the segment TRY(MemoryManager::alloc_at( base_vaddr, get_blocks_from_size(program_header.p_memsz + vaddr_diff, ARCH_PAGE_SIZE), flags)); diff --git a/kernel/src/thread/ELF.h b/kernel/src/thread/ELF.h index 87781d39..21103e43 100644 --- a/kernel/src/thread/ELF.h +++ b/kernel/src/thread/ELF.h @@ -1,5 +1,6 @@ #pragma once #include "fs/VFS.h" +#include "memory/UserVM.h" #include #define ELFMAG "\177ELF" @@ -53,5 +54,5 @@ struct ELFData namespace ELFLoader { - Result load(SharedPtr inode); + Result load(SharedPtr inode, UserVM* vm); }; diff --git a/kernel/src/thread/ThreadImage.cpp b/kernel/src/thread/ThreadImage.cpp index 1ff9e0a9..d937223b 100644 --- a/kernel/src/thread/ThreadImage.cpp +++ b/kernel/src/thread/ThreadImage.cpp @@ -6,10 +6,13 @@ static constexpr usize DEFAULT_USER_STACK_PAGES = 6; static constexpr usize DEFAULT_USER_STACK_SIZE = DEFAULT_USER_STACK_PAGES * ARCH_PAGE_SIZE; -static Result create_stacks(Stack& user_stack, Stack& kernel_stack) +static Result create_stacks(Stack& user_stack, Stack& kernel_stack, UserVM* vm) { const u64 THREAD_STACK_BASE = 0x10000; + // FIXME: Set this memory range to persistent so that munmap() cannot remove it. + if (!TRY(vm->test_and_alloc_region(THREAD_STACK_BASE, DEFAULT_USER_STACK_PAGES))) return err(ENOMEM); + TRY(MemoryManager::alloc_at_zeroed(THREAD_STACK_BASE, DEFAULT_USER_STACK_PAGES, MMU::ReadWrite | MMU::NoExecute | MMU::User)); @@ -42,11 +45,11 @@ Result> ThreadImage::try_load_from_elf(SharedPtr