kernel: Make the stack and loaded program code regions persistent

This way, they can't be modified by mmap() or munmap().

.gitignore

@@ -7,3 +7,4 @@ initrd/bin/**
 initrd/tests/**
 base/
 .fakeroot
+kernel/config.cmake

.vscode/settings.json

@@ -13,5 +13,16 @@
     "files.trimFinalNewlines": true,
     "files.insertFinalNewline": true,
     "git.inputValidationLength": 72,
-    "git.inputValidationSubjectLength": 72
+    "git.inputValidationSubjectLength": 72,
+    "doxdocgen.file.fileOrder": [
+        "file",
+        "author",
+        "brief",
+        "empty",
+        "copyright",
+        "empty"
+    ],
+    "doxdocgen.file.copyrightTag": [
+        "@copyright Copyright (c) {year}, the Luna authors."
+    ]
 }

CMakeLists.txt

@@ -5,7 +5,7 @@ set(CMAKE_CXX_COMPILER_WORKS 1)
 
 set(CMAKE_CROSSCOMPILING true)
 
-project(Luna LANGUAGES C CXX ASM ASM_NASM VERSION 0.2.0)
+project(Luna LANGUAGES C CXX ASM ASM_NASM VERSION 0.3.0)
 
 set(LUNA_ROOT ${CMAKE_CURRENT_LIST_DIR})
 set(LUNA_BASE ${CMAKE_CURRENT_LIST_DIR}/base)

apps/CMakeLists.txt

@@ -29,3 +29,6 @@ luna_app(umount.cpp umount)
 luna_app(ps.cpp ps)
 luna_app(time.cpp time)
 luna_app(ln.cpp ln)
+luna_app(mktemp.cpp mktemp)
+luna_app(sysfuzz.cpp sysfuzz)
+luna_app(pivot_root.cpp pivot_root)

apps/date.cpp

@@ -11,8 +11,7 @@ Result<int> luna_main(int argc, char** argv)
     os::ArgumentParser parser;
     parser.add_description("Display the current (or another) date and time."_sv);
     parser.add_system_program_info("date"_sv);
-    parser.add_value_argument(date, 'd', "date"_sv, true,
-                              "the UNIX timestamp to display instead of the current time"_sv);
+    parser.add_value_argument(date, 'd', "date"_sv, "the UNIX timestamp to display instead of the current time"_sv);
     parser.parse(argc, argv);
 
     time_t now;

apps/init.cpp

@@ -10,10 +10,10 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mount.h>
 #include <sys/stat.h>
 #include <sys/syscall.h>
 #include <sys/sysmacros.h>
-#include <sys/wait.h>
 #include <unistd.h>
 
 FILE* g_init_log;
@@ -96,8 +96,7 @@ static Result<void> try_start_service(Service& service)
         fprintf(g_init_log, "[init] waiting for child process %d to finish\n", pid);
 
         int status;
-        pid_t id = waitpid(pid, &status, 0);
-        if (id < 0) { return err(errno); }
+        TRY(os::Process::wait(pid, &status));
 
         fprintf(g_init_log, "[init] child process %d exited with code %d\n", pid, WEXITSTATUS(status));
     }
@@ -148,10 +147,28 @@ static Result<void> load_service(const os::Path& path)
 
         if (parts[0].view() == "Command")
         {
+            if (!service.command.is_empty())
+            {
+                fprintf(g_init_log, "[init] 'Command' cannot be specified after 'Script' has already been set! (%s)\n",
+                        line.chars());
+                return {};
+            }
             service.command = move(parts[1]);
             continue;
         }
 
+        if (parts[0].view() == "Script")
+        {
+            if (!service.command.is_empty())
+            {
+                fprintf(g_init_log, "[init] 'Script' cannot be specified after 'Command' has already been set! (%s)\n",
+                        line.chars());
+                return {};
+            }
+            service.command = TRY(String::format("/bin/sh -- %s"_sv, parts[1].chars()));
+            continue;
+        }
+
         if (parts[0].view() == "Restart")
         {
             if (parts[1].view() == "true" || parts[1].view().to_uint().value_or(0) == 1)
@@ -209,7 +226,7 @@ static Result<void> load_service(const os::Path& path)
 
     if (service.command.is_empty())
     {
-        fprintf(g_init_log, "[init] service file is missing 'Command' entry, aborting!\n");
+        fprintf(g_init_log, "[init] service file is missing 'Command' or 'Script' entry, aborting!\n");
         return {};
     }
 
@@ -224,7 +241,7 @@ static Result<void> load_services()
 {
     auto dir = TRY(os::Directory::open("/etc/init"));
 
-    auto services = TRY(dir->list(os::Directory::Filter::ParentAndBase));
+    auto services = TRY(dir->list_names(os::Directory::Filter::ParentAndBase));
     sort(services.begin(), services.end(), String::compare);
 
     for (const auto& entry : services) TRY(load_service({ dir->fd(), entry.view() }));
@@ -258,6 +275,13 @@ static Result<void> set_hostname()
     return {};
 }
 
+static void mount_devfs()
+{
+    if (mkdir("/dev", 0755) < 0 && errno != EEXIST) exit(255);
+
+    if (mount("/dev", "devfs") < 0) exit(255);
+}
+
 int main()
 {
     if (getpid() != 1)
@@ -266,6 +290,8 @@ int main()
         return 1;
     }
 
+    mount_devfs();
+
     // Before this point, we don't even have an stdin, stdout and stderr. Set it up now so that child processes (and us)
     // can print stuff.
     stdin = fopen("/dev/console", "r");
@@ -284,7 +310,10 @@ int main()
     while (1)
     {
         int status;
-        pid_t child = wait(&status);
+        auto rc = os::Process::wait(os::Process::ANY_CHILD, &status);
+        if (rc.has_error()) continue;
+
+        pid_t child = rc.release_value();
 
         for (auto& service : g_services)
         {

apps/ls.cpp

@@ -1,4 +1,6 @@
 #include <grp.h>
+#include <luna/Sort.h>
+#include <luna/StringBuilder.h>
 #include <luna/Units.h>
 #include <os/ArgumentParser.h>
 #include <os/Directory.h>
@@ -20,6 +22,48 @@ void find_user_and_group(struct stat& st, StringView& owner, StringView& group)
         group = grp->gr_name;
 }
 
+int sort_name(const os::Directory::Entry* a, const os::Directory::Entry* b)
+{
+    return String::compare(&a->name, &b->name);
+}
+
+int sort_size(const os::Directory::Entry* a, const os::Directory::Entry* b)
+{
+    return (a->size < b->size) ? -1 : ((a->size == b->size) ? 0 : 1);
+}
+
+int sort_time(const os::Directory::Entry* a, const os::Directory::Entry* b)
+{
+    return (a->mtime < b->mtime) ? -1 : ((a->mtime == b->mtime) ? 0 : 1);
+}
+
+static int (*sort_function)(const os::Directory::Entry*, const os::Directory::Entry*) = sort_name;
+
+int sort_reverse(const os::Directory::Entry* a, const os::Directory::Entry* b)
+{
+    int rc = sort_function(a, b);
+    if (rc < 0) return 1;
+    if (rc > 0) return -1;
+    return 0;
+}
+
+static Result<String> entry_join(const Vector<os::Directory::Entry>& vec, StringView delim)
+{
+    if (vec.size() == 0) return String {};
+    if (vec.size() == 1) return vec[0].name.clone();
+
+    StringBuilder sb;
+    TRY(sb.add(vec[0].name));
+
+    for (usize i = 1; i < vec.size(); i++)
+    {
+        TRY(sb.add(delim));
+        TRY(sb.add(vec[i].name));
+    }
+
+    return sb.string();
+}
+
 Result<int> luna_main(int argc, char** argv)
 {
     StringView pathname;
@@ -32,6 +76,12 @@ Result<int> luna_main(int argc, char** argv)
     bool one_per_line { false };
     bool list_directories { false };
 
+    StringView sort_type { "name" };
+
+    bool reverse_sort { false };
+
+    bool should_sort { true };
+
     os::ArgumentParser parser;
     parser.add_description("List files contained in a directory (defaults to '.', the current directory)"_sv);
     parser.add_system_program_info("ls"_sv);
@@ -46,14 +96,26 @@ Result<int> luna_main(int argc, char** argv)
                                "follow symbolic links listed as arguments"_sv);
     parser.add_switch_argument(one_per_line, '1', ""_sv, "list one file per line"_sv);
     parser.add_switch_argument(list_directories, 'd', "directory"_sv, "list directories instead of their contents"_sv);
+    parser.add_value_argument(sort_type, ' ', "sort"_sv, "sort by name, size or time"_sv);
+    parser.add_switch_argument(reverse_sort, 'r', "reverse"_sv, "reverse order while sorting"_sv);
     parser.parse(argc, argv);
 
-    Vector<String> files;
+    Vector<os::Directory::Entry> files;
     int dirfd = AT_FDCWD;
     SharedPtr<os::Directory> dir;
 
     if (!long_list) follow_symlink_args = true;
 
+    if (sort_type == "none"_sv) { should_sort = false; }
+    else if (sort_type == "name"_sv) { sort_function = sort_name; }
+    else if (sort_type == "size"_sv) { sort_function = sort_size; }
+    else if (sort_type == "time"_sv) { sort_function = sort_time; }
+    else
+    {
+        os::eprintln("%s: unknown sort type: %s", argv[0], sort_type.chars());
+        parser.short_usage(argv[0]);
+    }
+
     if (os::FileSystem::is_directory(pathname, follow_symlink_args) && !list_directories)
     {
         dir = TRY(os::Directory::open(pathname));
@@ -68,15 +130,31 @@ Result<int> luna_main(int argc, char** argv)
     }
     else if (os::FileSystem::exists(pathname, follow_symlink_args))
     {
-        auto str = TRY(String::from_string_view(pathname));
-        TRY(files.try_append(move(str)));
+        struct stat st;
+        TRY(os::FileSystem::stat(pathname, st, false));
+
+        os::Directory::Entry entry = {
+            .name = TRY(String::from_string_view(pathname)),
+            .mode = st.st_mode,
+            .size = (usize)st.st_size,
+            .mtime = st.st_mtime,
+        };
+
+        TRY(files.try_append(move(entry)));
     }
     else
         return err(ENOENT);
 
+    if (should_sort)
+    {
+        if (reverse_sort) sort(files.begin(), files.end(), sort_reverse);
+        else
+            sort(files.begin(), files.end(), sort_function);
+    }
+
     if (!long_list)
     {
-        auto list = TRY(String::join(files, one_per_line ? "\n"_sv : " "_sv));
+        auto list = TRY(entry_join(files, one_per_line ? "\n"_sv : " "_sv));
         if (!list.is_empty()) os::println("%s", list.chars());
     }
     else
@@ -84,9 +162,9 @@ Result<int> luna_main(int argc, char** argv)
         for (const auto& file : files)
         {
             struct stat st;
-            TRY(os::FileSystem::stat({ dirfd, file.view() }, st, false));
+            TRY(os::FileSystem::stat({ dirfd, file.name.view() }, st, false));
 
-            auto link = TRY(os::FileSystem::readlink({ dirfd, file.view() }));
+            auto link = TRY(os::FileSystem::readlink({ dirfd, file.name.view() }));
 
             StringView owner;
             StringView group;
@@ -99,13 +177,13 @@ Result<int> luna_main(int argc, char** argv)
             if (!human_readable && !si)
             {
                 os::println("%s %u %4s %4s %10lu %s%s%s", formatted_mode, st.st_nlink, owner.chars(), group.chars(),
-                            st.st_size, file.chars(), link.is_empty() ? "" : " -> ", link.chars());
+                            st.st_size, file.name.chars(), link.is_empty() ? "" : " -> ", link.chars());
             }
             else
             {
                 auto size = TRY(to_dynamic_unit(st.st_size, 10, false, si ? Unit::SI : Unit::Binary, false));
                 os::println("%s %u %4s %4s %6s %s%s%s", formatted_mode, st.st_nlink, owner.chars(), group.chars(),
-                            size.chars(), file.chars(), link.is_empty() ? "" : " -> ", link.chars());
+                            size.chars(), file.name.chars(), link.is_empty() ? "" : " -> ", link.chars());
             }
         }
     }

apps/mkdir.cpp

@@ -20,8 +20,7 @@ begin:
     if (rc.error() == EEXIST) return {};
     if (rc.error() == ENOENT)
     {
-        PathParser parser = TRY(PathParser::create(path.chars()));
-        auto parent = TRY(parser.dirname());
+        auto parent = TRY(PathParser::dirname(path));
 
         TRY(mkdir_recursively(parent.view(), (0777 & ~s_umask) | S_IWUSR | S_IXUSR));
 
@@ -41,7 +40,7 @@ Result<int> luna_main(int argc, char** argv)
     parser.add_description("Create directories."_sv);
     parser.add_system_program_info("mkdir"_sv);
     parser.add_positional_argument(path, "path"_sv, true);
-    parser.add_value_argument(mode_string, 'm', "mode"_sv, true, "set the mode for the newly created directory");
+    parser.add_value_argument(mode_string, 'm', "mode"_sv, "set the mode for the newly created directory");
     parser.add_switch_argument(recursive, 'p', "parents"_sv,
                                "if parent directories do not exist, create them as well"_sv);
     parser.parse(argc, argv);

apps/mktemp.cpp

@@ -0,0 +1,40 @@
+#include <errno.h>
+#include <luna/String.h>
+#include <os/ArgumentParser.h>
+#include <os/File.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+
+Result<int> luna_main(int argc, char** argv)
+{
+    srand((unsigned)time(NULL));
+
+    bool make_directory { false };
+    StringView template_sv;
+
+    os::ArgumentParser parser;
+    parser.add_description("Create a temporary file or directory safely and print its name.");
+    parser.add_system_program_info("mktemp"_sv);
+    parser.add_switch_argument(make_directory, 'd', "directory"_sv, "make a directory instead of a file"_sv);
+    parser.add_positional_argument(template_sv, "template"_sv, "/tmp/tmp.XXXXXX"_sv);
+    parser.parse(argc, argv);
+
+    String str = TRY(String::from_string_view(template_sv));
+
+    if (make_directory)
+    {
+        if (mkdtemp(str.mutable_data()) == nullptr) return err(errno);
+    }
+    else
+    {
+        int fd = -1;
+        fd = mkstemp(str.mutable_data());
+        if (fd < 0) return err(errno);
+        close(fd);
+    }
+
+    os::println("%s"_sv, str.chars());
+
+    return 0;
+}

apps/mount.cpp

@@ -5,13 +5,13 @@
 Result<int> luna_main(int argc, char** argv)
 {
     StringView target;
-    StringView fstype;
+    StringView fstype { "auto" };
 
     os::ArgumentParser parser;
     parser.add_description("Mount a file system.");
     parser.add_system_program_info("mount"_sv);
     parser.add_positional_argument(target, "mountpoint"_sv, true);
-    parser.add_value_argument(fstype, 't', "type"_sv, "auto"_sv, "the file system type to use");
+    parser.add_value_argument(fstype, 't', "type"_sv, "the file system type to use");
     parser.parse(argc, argv);
 
     if (mount(target.chars(), fstype.chars()) < 0)

apps/pivot_root.cpp

@@ -0,0 +1,19 @@
+#include <os/ArgumentParser.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+Result<int> luna_main(int argc, char** argv)
+{
+    StringView new_root;
+    StringView put_old;
+
+    os::ArgumentParser parser;
+    parser.add_description("Move the current root directory to another directory and replace it with another mount.");
+    parser.add_system_program_info("pivot_root"_sv);
+    parser.add_positional_argument(new_root, "new_root", true);
+    parser.add_positional_argument(put_old, "put_old", true);
+    parser.parse(argc, argv);
+
+    long rc = syscall(SYS_pivot_root, new_root.chars(), put_old.chars());
+    return Result<int>::from_syscall(rc);
+}

apps/sh.cpp

@@ -11,7 +11,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/utsname.h>
-#include <sys/wait.h>
 #include <unistd.h>
 
 using os::File;
@@ -23,6 +22,8 @@ static Result<Vector<String>> split_command_into_args(StringView cmd)
 
 static Result<void> execute_command(StringView command)
 {
+    if (strcspn(command.chars(), " #") == 0) return {};
+
     auto args = TRY(split_command_into_args(command));
     if (args.size() < 1) exit(0);
 
@@ -47,7 +48,7 @@ Result<int> luna_main(int argc, char** argv)
     parser.add_description("The Luna system's command shell."_sv);
     parser.add_system_program_info("sh"_sv);
     parser.add_positional_argument(path, "path"_sv, "-"_sv);
-    parser.add_value_argument(command, 'c', "command"_sv, true, "execute a single command and then exit"_sv);
+    parser.add_value_argument(command, 'c', "command"_sv, "execute a single command and then exit"_sv);
     parser.parse(argc, argv);
 
     if (!command.is_empty()) TRY(execute_command(command));
@@ -94,6 +95,8 @@ Result<int> luna_main(int argc, char** argv)
 
         if (strspn(cmd.chars(), " \n") == cmd.length()) continue;
 
+        if (strcspn(cmd.chars(), " #") == 0) continue;
+
         if (!strncmp(cmd.chars(), "cd", 2))
         {
             auto args = TRY(split_command_into_args(cmd.view()));
@@ -114,11 +117,7 @@ Result<int> luna_main(int argc, char** argv)
 
         if (child == 0) { TRY(execute_command(cmd.view())); }
 
-        if (waitpid(child, NULL, 0) < 0)
-        {
-            perror("waitpid");
-            return 1;
-        }
+        TRY(os::Process::wait(child, nullptr));
     }
 
     return 0;

apps/sysfuzz.cpp

@@ -0,0 +1,73 @@
+#include <luna/Syscall.h>
+#include <os/ArgumentParser.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+static const char* syscall_list[] = {
+#undef __enumerate
+#define __enumerate(name) #name,
+    enumerate_syscalls(__enumerate)
+#undef __enumerate
+};
+
+int random_syscall()
+{
+    int sys;
+
+    while (true)
+    {
+        sys = rand() % Syscalls::__count;
+        if (sys == SYS_exit || sys == SYS_usleep || sys == SYS_fork) continue;
+        break;
+    }
+
+    return sys;
+}
+
+void random_args(int args[5])
+{
+    for (int i = 0; i < 5; i++) { args[i] = rand(); }
+}
+
+int main(int argc, char** argv)
+{
+    StringView times_sv = "20"_sv;
+    StringView interval_sv = "1000"_sv;
+
+    os::ArgumentParser parser;
+    parser.add_description("System call fuzzer (invokes system calls with random arguments to test system stability)");
+    parser.add_system_program_info("sysfuzz"_sv);
+    parser.add_value_argument(times_sv, 't', "times"_sv, "the number of syscalls to invoke"_sv);
+    parser.add_value_argument(interval_sv, 'i', "interval"_sv,
+                              "the interval between system calls (in milliseconds)"_sv);
+    parser.parse(argc, argv);
+
+    srand((unsigned)time(nullptr));
+
+    int times = atoi(times_sv.chars());
+    int interval = atoi(interval_sv.chars());
+
+    while (times--)
+    {
+        int args[5];
+        int sys = random_syscall();
+        random_args(args);
+
+        printf("%s(%d, %d, %d, %d, %d) -> ", syscall_list[sys], args[0], args[1], args[2], args[3], args[4]);
+
+        long rc = syscall(sys, args[0], args[1], args[2], args[3], args[4]);
+
+        if (rc < 0)
+        {
+            int error = (int)-rc;
+            printf("%ld (%s)\n", rc, strerror(error));
+        }
+        else
+            printf("%ld\n", rc);
+
+        usleep(interval * 1000);
+    }
+}

apps/time.cpp

@@ -3,7 +3,6 @@
 #include <os/Process.h>
 #include <stdio.h>
 #include <sys/resource.h>
-#include <sys/wait.h>
 
 Result<int> luna_main(int argc, char** argv)
 {
@@ -23,11 +22,7 @@ Result<int> luna_main(int argc, char** argv)
         unreachable();
     }
 
-    if (waitpid(pid, nullptr, 0) < 0)
-    {
-        perror("waitpid");
-        return 1;
-    }
+    TRY(os::Process::wait(pid, nullptr));
 
     struct rusage usage;
     if (getrusage(RUSAGE_CHILDREN, &usage) < 0)

initrd/etc/init/00-tmpfs

@@ -0,0 +1,3 @@
+Name=tmpfs
+Script=/sbin/mount-tmpfs
+Wait=true

initrd/sbin/mount-tmpfs

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mkdir -p /tmp
+mount -t tmpfs /tmp
+chmod 1777 /tmp

kernel/CMakeLists.txt

@@ -43,7 +43,10 @@ set(SOURCES
     src/fs/VFS.cpp
     src/fs/Pipe.cpp
     src/fs/Mount.cpp
+    src/fs/MBR.cpp
+    src/fs/GPT.cpp
     src/fs/tmpfs/FileSystem.cpp
+    src/fs/tmpfs/Inode.cpp
     src/fs/devices/DeviceRegistry.cpp
     src/fs/devices/NullDevice.cpp
     src/fs/devices/ZeroDevice.cpp
@@ -65,6 +68,7 @@ if("${LUNA_ARCH}" MATCHES "x86_64")
         src/arch/x86_64/Thread.cpp
         src/arch/x86_64/PCI.cpp
         src/arch/x86_64/Keyboard.cpp
+        src/arch/x86_64/disk/ATA.cpp
         src/arch/x86_64/init/GDT.cpp
         src/arch/x86_64/init/IDT.cpp
         src/arch/x86_64/init/PIC.cpp
@@ -99,6 +103,10 @@ if(MOON_DEBUG)
     include(debug.cmake)
 endif()
 
+if(EXISTS config.cmake)
+    include(config.cmake)
+endif()
+
 target_link_options(moon PRIVATE -lgcc -Wl,--build-id=none -z max-page-size=0x1000 -mcmodel=kernel)
 
 set_target_properties(moon PROPERTIES CXX_STANDARD 20)

kernel/config.cmake.template

@@ -0,0 +1,12 @@
+# Copy this file and rename it to 'config.cmake' before making your own changes.
+
+# config.cmake: Configuration file for the kernel.
+# Edit/add values to customize kernel compilation flags/defines.
+# This file is automatically ignored by git.
+# To use an example configuration line, just remove the hashtag '#' at the beginning.
+
+# Example: Adding a compiler definition. This will define PCI_DEBUG in the kernel source.
+# target_compile_definitions(moon PRIVATE PCI_DEBUG)
+
+# Example: Adding a compiler flag. This will optimize the kernel aggressively (warning: untested, use at your own discretion).
+# target_compile_options(moon PRIVATE -O3)

kernel/debug.cmake

@@ -3,4 +3,9 @@ target_compile_definitions(moon PRIVATE DEBUG_MODE)
 target_compile_definitions(moon PRIVATE ELF_DEBUG)
 target_compile_definitions(moon PRIVATE MMU_DEBUG)
 target_compile_definitions(moon PRIVATE MMAP_DEBUG)
+target_compile_definitions(moon PRIVATE EXEC_DEBUG)
+target_compile_definitions(moon PRIVATE OPEN_DEBUG)
+target_compile_definitions(moon PRIVATE REAP_DEBUG)
+target_compile_definitions(moon PRIVATE PCI_DEBUG)
+target_compile_definitions(moon PRIVATE DEVICE_REGISTRY_DEBUG)
 target_compile_options(moon PRIVATE -fsanitize=undefined)

kernel/src/Log.cpp

@@ -35,7 +35,9 @@ static void log_serial(LogLevel level, const char* format, va_list origin)
                    "\x1b[0m ",
                    ansi_color_codes_per_log_level[(int)level], log_level_letters[(int)level]);
 
-    Serial::printf("%4zu.%.3zu ", Timer::ticks(), Timer::ticks_ms() - (Timer::ticks() * 1000));
+    auto* time = Timer::monotonic_clock();
+
+    Serial::printf("%4zu.%.3zu ", time->tv_sec, time->tv_nsec / 1'000'000);
 
     // NOTE: We do this manually because of a lack of vprintf() in both Serial and TextConsole.
     cstyle_format(
@@ -134,6 +136,11 @@ bool log_text_console_enabled()
     return g_text_console_enabled;
 }
 
+void set_text_console_initialized()
+{
+    g_text_console_initialized = true;
+}
+
 static bool g_check_already_failed = false;
 
 [[noreturn]] bool __check_failed(SourceLocation location, const char* expr)
@@ -143,9 +150,25 @@ static bool g_check_already_failed = false;
     if (!g_check_already_failed)
     { // Avoid endlessly failing when trying to report a failed check.
         g_check_already_failed = true;
-        kerrorln("ERROR: Check failed at %s:%d, in %s: %s", location.file(), location.line(), location.function(),
-                 expr);
+        kerrorln("KERNEL PANIC: Check failed at %s:%d, in %s: %s", location.file(), location.line(),
+                 location.function(), expr);
         CPU::print_stack_trace();
     }
     CPU::efficient_halt();
 }
+
+[[noreturn]] void __critical_error_handler(SourceLocation location, const char* expr, const char* failmsg,
+                                           const char* errmsg)
+{
+    CPU::disable_interrupts();
+    if (g_text_console_initialized) g_text_console_enabled = true;
+    if (!g_check_already_failed)
+    { // Avoid endlessly failing when trying to report a failed check.
+        g_check_already_failed = true;
+        kerrorln("-- KERNEL PANIC: Critical routine \"%s\" failed in %s: %s (%s) --", expr, location.file(), failmsg,
+                 errmsg);
+        CPU::print_stack_trace();
+        kerrorln("-- END KERNEL PANIC --");
+    }
+    CPU::efficient_halt();
+}

kernel/src/Log.h

@@ -19,7 +19,23 @@ bool log_debug_enabled();
 bool log_serial_enabled();
 bool log_text_console_enabled();
 
+void set_text_console_initialized();
+
 #define kdbgln(...) log(LogLevel::Debug, __VA_ARGS__)
 #define kinfoln(...) log(LogLevel::Info, __VA_ARGS__)
 #define kwarnln(...) log(LogLevel::Warn, __VA_ARGS__)
 #define kerrorln(...) log(LogLevel::Error, __VA_ARGS__)
+
+[[noreturn]] extern void __critical_error_handler(SourceLocation location, const char* expr, const char* failmsg,
+                                                  const char* errmsg);
+
+// Mark an expression that returns a Result<T> as critical. If the operation is successful, the Result will be
+// unwrapped. If the operation throws an error, however, the kernel will panic, showing the expression, the file in
+// which mark_critical() was called, failmsg and the error code.
+#define mark_critical(expr, failmsg)                                                                                   \
+    ({                                                                                                                 \
+        auto _expr_rc = (expr);                                                                                        \
+        if (!_expr_rc.has_value())                                                                                     \
+            __critical_error_handler(SourceLocation::current(), #expr, failmsg, _expr_rc.error_string());              \
+        _expr_rc.release_value();                                                                                      \
+    })

kernel/src/arch/CPU.h

@@ -22,6 +22,9 @@ namespace CPU
     void disable_interrupts();
     void wait_for_interrupt();
 
+    bool save_interrupts();
+    void restore_interrupts(bool saved);
+
     void get_stack_trace(void (*callback)(u64, void*), void* arg);
     void print_stack_trace();
     void get_stack_trace_at(Registers* regs, void (*callback)(u64, void*), void* arg);
@@ -29,5 +32,8 @@ namespace CPU
 
     u16 get_processor_id();
 
+    bool register_interrupt(u8 interrupt, void (*handler)(Registers*, void*), void* context);
+    void sync_interrupts();
+
     void pause();
 }

kernel/src/arch/PCI.cpp

@@ -9,6 +9,10 @@ struct ScanInfo
 
 namespace PCI
 {
+    BAR::BAR(u32 raw) : m_raw(raw)
+    {
+    }
+
     Device::ID read_id(const Device::Address& address)
     {
         const u16 vendor = read16(address, Field::VendorID);
@@ -92,12 +96,16 @@ namespace PCI
         // Single-function PCI bus
         if ((header_type & 0x80) == 0)
         {
+#ifdef PCI_DEBUG
             kdbgln("PCI bus is single-function");
+#endif
             scan_bus(0, { callback, match });
         }
         else
         {
+#ifdef PCI_DEBUG
             kdbgln("PCI bus is multiple-function");
+#endif
             for (u32 function = 0; function < 8; function++)
             {
                 if (read16({ 0, 0, function }, Field::VendorID) != PCI::INVALID_ID)
@@ -108,4 +116,13 @@ namespace PCI
             }
         }
     }
+
+    BAR Device::getBAR(u8 index) const
+    {
+        check(index < 6);
+
+        u32 raw = read32(address, 0x10 + (index * 4));
+
+        return { raw };
+    }
 }

kernel/src/arch/PCI.h

@@ -17,7 +17,67 @@ namespace PCI
 
         HeaderType = 0x0e,
 
-        SecondaryBus = 0x19
+        BAR0 = 0x10,
+        BAR1 = 0x14,
+        BAR2 = 0x18,
+        BAR3 = 0x1c,
+
+        SecondaryBus = 0x19,
+
+        InterruptLine = 0x3c,
+    };
+
+    enum CommandField : u16
+    {
+        CMD_IO_SPACE = 1 << 0,
+        CMD_MEMORY_SPACE = 1 << 1,
+        CMD_BUS_MASTER = 1 << 2,
+        CMD_SPECIAL_CYCLES = 1 << 3,
+        CMD_MEMORY_WRITE_AND_INVALIDATE = 1 << 4,
+        CMD_VGA_PALETTE_SNOOP = 1 << 5,
+        CMD_PARITY_ERROR_RESPONSE = 1 << 6,
+        CMD_SERR = 1 << 8,
+        CMD_FAST_BACK_TO_BACK = 1 << 9,
+        CMD_INTERRUPT_DISABLE = 1 << 10,
+    };
+
+    struct BAR
+    {
+      public:
+        BAR(u32 raw);
+
+        bool is_iospace()
+        {
+            return m_raw & 0x01;
+        }
+
+        bool is_memory_space()
+        {
+            return !is_iospace();
+        }
+
+        u16 port()
+        {
+            return (u16)(m_raw & 0xfffffffc);
+        }
+
+        u8 type()
+        {
+            return (m_raw >> 1) & 0x03;
+        }
+
+        bool is_prefetchable()
+        {
+            return m_raw & (1 << 3);
+        }
+
+        u32 address_32bit()
+        {
+            return m_raw & 0xFFFFFFF0;
+        }
+
+      private:
+        u32 m_raw;
     };
 
     struct Device
@@ -42,6 +102,8 @@ namespace PCI
             u32 function;
         };
 
+        BAR getBAR(u8 index) const;
+
         ID id;
         Type type;
         Address address;

kernel/src/arch/Timer.cpp

@@ -4,10 +4,8 @@
 #include "boot/bootboot.h"
 #include <luna/TypeTraits.h>
 
-// NOTE: Storing these values as unsigned integers doesn't allow for pre-epoch times.
-// We are in 2023 anyway, not sure why anybody would want to set their computer's time to 1945.
-static u64 timer_ticks = 0;
-static u64 boot_timestamp;
+static struct timespec s_monotonic_clock = { 0, 0 };
+static struct timespec s_realtime_clock;
 
 static inline constexpr bool isleap(u32 year)
 {
@@ -56,84 +54,38 @@ extern const BOOTBOOT bootboot;
 
 namespace Timer
 {
+    static struct timespec s_interval = { .tv_sec = 0, .tv_nsec = ARCH_TIMER_RESOLUTION * 1000 };
+
     void tick()
     {
-        timer_ticks += ARCH_TIMER_RESOLUTION;
-    }
-
-    usize raw_ticks()
-    {
-        return timer_ticks;
-    }
-
-    usize ticks()
-    {
-        return ticks_us() / US_PER_SECOND;
+        timespecadd(&s_monotonic_clock, &s_interval, &s_monotonic_clock);
+        timespecadd(&s_realtime_clock, &s_interval, &s_realtime_clock);
     }
 
     usize ticks_ms()
     {
-        return timer_ticks / 1000;
+        return (s_monotonic_clock.tv_sec * 1000) + (s_monotonic_clock.tv_nsec / 1'000'000);
     }
 
-    usize ticks_us()
+    struct timespec* monotonic_clock()
     {
-        return timer_ticks;
+        return &s_monotonic_clock;
     }
 
-    usize ticks_ns()
+    struct timespec* realtime_clock()
     {
-        return ticks_us() * 1000;
-    }
-
-    usize boot()
-    {
-        return boot_timestamp / US_PER_SECOND;
-    }
-
-    usize boot_ms()
-    {
-        return boot_timestamp / 1000;
-    }
-
-    usize boot_us()
-    {
-        return boot_timestamp;
-    }
-
-    usize boot_ns()
-    {
-        return boot_timestamp * 1000;
-    }
-
-    usize clock()
-    {
-        return boot() + ticks();
-    }
-
-    usize clock_ms()
-    {
-        return boot_ms() + ticks_ms();
-    }
-
-    usize clock_us()
-    {
-        return boot_us() + ticks_us();
-    }
-
-    usize clock_ns()
-    {
-        return boot_ns() + ticks_ns();
+        return &s_realtime_clock;
     }
 
     void init()
     {
-        boot_timestamp = bootloader_time_to_unix(bootboot.datetime) * US_PER_SECOND;
+        s_realtime_clock.tv_sec = bootloader_time_to_unix(bootboot.datetime);
+        s_realtime_clock.tv_nsec = 0;
         arch_init();
     }
 }
 
 bool should_invoke_scheduler()
 {
-    return (timer_ticks % 1000) == 0;
+    return (s_realtime_clock.tv_nsec % 1'000'000) == 0;
 }

kernel/src/arch/Timer.h

@@ -1,4 +1,5 @@
 #pragma once
+#include <bits/timespec.h>
 #include <luna/Types.h>
 
 #ifdef ARCH_X86_64
@@ -15,22 +16,11 @@ namespace Timer
 {
     void tick();
 
-    usize raw_ticks();
-
-    usize ticks();
     usize ticks_ms();
-    usize ticks_us();
-    usize ticks_ns();
 
-    usize boot();
-    usize boot_ms();
-    usize boot_us();
-    usize boot_ns();
+    struct timespec* monotonic_clock();
 
-    usize clock();
-    usize clock_ms();
-    usize clock_us();
-    usize clock_ns();
+    struct timespec* realtime_clock();
 
     void arch_init();
     void init();

kernel/src/arch/x86_64/CPU.asm

@@ -188,4 +188,18 @@ ISR_ERROR 21 ; control-protection exception (#CP)
 ; ISR 22-31 reserved
 IRQ 32, 0 ; timer interrupt
 IRQ 33, 1 ; keyboard interrupt
+IRQ 34, 2
+IRQ 35, 3
+IRQ 36, 4
+IRQ 37, 5
+IRQ 38, 6
+IRQ 39, 7
+IRQ 40, 8
+IRQ 41, 9
+IRQ 42, 10
+IRQ 43, 11
+IRQ 44, 12
+IRQ 45, 13
+IRQ 46, 14
+IRQ 47, 15
 ISR 66 ; system call

kernel/src/arch/x86_64/CPU.cpp

@@ -22,12 +22,23 @@ extern "C" void enable_nx();
 
 extern void setup_gdt();
 extern void remap_pic();
+extern void change_pic_masks(u8 pic1_mask, u8 pic2_mask);
 extern void pic_eoi(unsigned char irq);
 extern void pic_eoi(Registers* regs);
 extern void setup_idt();
 
 static Thread* g_io_thread;
 
+typedef void (*interrupt_handler_t)(Registers*, void*);
+
+struct InterruptHandler
+{
+    interrupt_handler_t function;
+    void* context;
+};
+
+static InterruptHandler irq_handlers[16];
+
 void FPData::save()
 {
     asm volatile("fxsave (%0)" : : "r"(m_data));
@@ -69,18 +80,15 @@ void decode_page_fault_error_code(u64 code)
 
     decode_page_fault_error_code(regs->error);
 
+    CPU::print_stack_trace_at(regs);
+
     if (!is_in_kernel(regs))
     {
         // FIXME: Kill this process with SIGSEGV once we have signals and all that.
         kerrorln("Current task %zu was terminated because of a page fault", Scheduler::current()->id);
-        Scheduler::current()->state = ThreadState::Exited;
-        Scheduler::current()->status = 127;
-        kernel_yield();
-        unreachable();
+        Scheduler::current()->exit_and_signal_parent(127);
     }
 
-    CPU::print_stack_trace_at(regs);
-
     CPU::efficient_halt();
 }
 
@@ -92,6 +100,13 @@ void decode_page_fault_error_code(u64 code)
 
     CPU::print_stack_trace_at(regs);
 
+    if (!is_in_kernel(regs))
+    {
+        // FIXME: Kill this process with SIGSEGV once we have signals and all that.
+        kerrorln("Current task %zu was terminated because of a general protection fault", Scheduler::current()->id);
+        Scheduler::current()->exit_and_signal_parent(127);
+    }
+
     CPU::efficient_halt();
 }
 
@@ -135,21 +150,34 @@ void io_thread()
     }
 }
 
+static void timer_interrupt(Registers* regs, void*)
+{
+    Timer::tick();
+    if (should_invoke_scheduler()) Scheduler::invoke(regs);
+}
+
+static void keyboard_interrupt(Registers*, void*)
+{
+    u8 scancode = IO::inb(0x60);
+    scancode_queue.try_push(scancode);
+    g_io_thread->wake_up();
+}
+
 // Called from _asm_interrupt_entry
 extern "C" void arch_interrupt_entry(Registers* regs)
 {
     if (regs->isr < 32) handle_x86_exception(regs);
-    else if (regs->isr == 32) // Timer interrupt
+    else if (regs->isr >= 32 && regs->isr < 48) // IRQ from the PIC
     {
-        Timer::tick();
-        if (should_invoke_scheduler()) Scheduler::invoke(regs);
-        pic_eoi(regs);
-    }
-    else if (regs->isr == 33) // Keyboard interrupt
-    {
-        u8 scancode = IO::inb(0x60);
-        scancode_queue.try_push(scancode);
-        g_io_thread->wake_up();
+        u64 irq = regs->irq;
+        auto handler = irq_handlers[irq];
+        if (!handler.function)
+        {
+            kwarnln("Unhandled IRQ catched! Halting.");
+            CPU::efficient_halt();
+        }
+
+        handler.function(regs, handler.context);
         pic_eoi(regs);
     }
     else if (regs->isr == 66) // System call
@@ -159,7 +187,7 @@ extern "C" void arch_interrupt_entry(Registers* regs)
     }
     else
     {
-        kwarnln("IRQ catched! Halting.");
+        kwarnln("Unhandled interrupt catched! Halting.");
         CPU::efficient_halt();
     }
 }
@@ -218,14 +246,20 @@ namespace CPU
             kwarnln("not setting the NX bit as it is unsupported");
         setup_gdt();
         setup_idt();
+
+        memset(irq_handlers, 0, sizeof(irq_handlers));
+        register_interrupt(0, timer_interrupt, nullptr);
+        register_interrupt(1, keyboard_interrupt, nullptr);
     }
 
     void platform_finish_init()
     {
-        g_io_thread = Scheduler::new_kernel_thread(io_thread, "[x86_64-io]")
-                          .expect_value("Could not create the IO background thread!");
+        g_io_thread = mark_critical(Scheduler::new_kernel_thread(io_thread, "[x86_64-io]"),
+                                    "Could not create the IO background thread!");
 
         remap_pic();
+
+        sync_interrupts();
     }
 
     void enable_interrupts()
@@ -238,19 +272,33 @@ namespace CPU
         asm volatile("cli");
     }
 
+    bool save_interrupts()
+    {
+        u64 flags;
+        asm volatile("pushfq; pop %0" : "=r"(flags));
+        return flags & 0x200;
+    }
+
+    void restore_interrupts(bool saved)
+    {
+        if (saved) enable_interrupts();
+        else
+            disable_interrupts();
+    }
+
     void wait_for_interrupt()
     {
         asm volatile("hlt");
     }
 
-    [[noreturn]] void efficient_halt() // Halt the CPU, using the lowest power possible. On x86-64 we do this using the
-                                       // "hlt" instruction, which puts the CPU into a low-power idle state until the
-                                       // next interrupt arrives... and we disable interrupts beforehand.
+    [[noreturn]] void efficient_halt() // Halt the CPU, using the lowest power possible. On x86-64 we do this using
+                                       // the "hlt" instruction, which puts the CPU into a low-power idle state
+                                       // until the next interrupt arrives... and we disable interrupts beforehand.
     {
         asm volatile("cli"); // Disable interrupts
     loop:
-        asm volatile("hlt"); // Let the cpu rest and pause until the next interrupt arrives... which in this case should
-                             // be never (unless an NMI arrives) :)
+        asm volatile("hlt"); // Let the cpu rest and pause until the next interrupt arrives... which in this case
+                             // should be never (unless an NMI arrives) :)
         goto loop;           // Safeguard: if we ever wake up, start our low-power rest again
     }
 
@@ -338,6 +386,35 @@ namespace CPU
         __get_cpuid(1, &unused, &ebx, &unused, &unused);
         return (u16)(ebx >> 24);
     }
+
+    bool register_interrupt(u8 interrupt, interrupt_handler_t handler, void* context)
+    {
+        if (irq_handlers[interrupt].function) return false;
+
+        irq_handlers[interrupt] = { handler, context };
+
+        sync_interrupts();
+
+        return true;
+    }
+
+    void sync_interrupts()
+    {
+        u8 pic1_mask, pic2_mask;
+        pic1_mask = pic2_mask = 0b11111111;
+        for (int i = 0; i < 8; i++)
+        {
+            if (irq_handlers[i].function) pic1_mask &= (u8)(~(1 << i));
+            if (irq_handlers[i + 8].function) pic2_mask &= (u8)(~(1 << i));
+        }
+
+        if (pic2_mask != 0b11111111) pic1_mask &= 0b11111011;
+
+        auto val = CPU::save_interrupts();
+        CPU::disable_interrupts();
+        change_pic_masks(pic1_mask, pic2_mask);
+        CPU::restore_interrupts(val);
+    }
 }
 
 // called by kernel_yield

kernel/src/arch/x86_64/CPU.h

@@ -5,7 +5,11 @@ struct Registers // Saved CPU registers for x86-64
 {
     u64 r15, r14, r13, r12, r11, r10, r9, r8;
     u64 rbp, rdi, rsi, rdx, rcx, rbx, rax;
-    u64 isr, error;
+    u64 isr;
+    union {
+        u64 error;
+        u64 irq;
+    };
     u64 rip, cs, rflags, rsp, ss;
 };
 

kernel/src/arch/x86_64/PCI.cpp

@@ -33,19 +33,32 @@ namespace PCI
 
     void write8(const Device::Address& address, u32 field, u8 value)
     {
-        ignore(address, field, value);
-        todo();
+        u8 offset = (u8)(field & ~0x3);
+        union {
+            u8 split[4];
+            u32 full;
+        };
+        full = read32(address, offset);
+        split[(field & 0x3)] = value;
+        write32(address, offset, full);
     }
 
-    void write16(const Device::Address& address, u32 field, u8 value)
+    void write16(const Device::Address& address, u32 field, u16 value)
     {
-        ignore(address, field, value);
-        todo();
+        u8 offset = (u8)(field & ~0x3);
+        union {
+            u8 split[4];
+            u32 full;
+        };
+        full = read32(address, offset);
+        split[(field & 0x3)] = (u8)(value >> 8);
+        split[(field & 0x3) + 1] = (u8)(value & 0xff);
+        write32(address, offset, full);
     }
 
-    void write32(const Device::Address& address, u32 field, u8 value)
+    void write32(const Device::Address& address, u32 field, u32 value)
     {
-        ignore(address, field, value);
-        todo();
+        IO::outl(PCI_ADDRESS_PORT, make_pci_address(address, field));
+        IO::outl(PCI_VALUE_PORT, value);
     }
 }

kernel/src/arch/x86_64/Timer.h

@@ -2,4 +2,5 @@
 #include <luna/Types.h>
 
 // Every timer tick is equivalent to 250 microseconds.
+// FIXME: Change ARCH_TIMER_RESOLUTION to use nanoseconds.
 const usize ARCH_TIMER_RESOLUTION = 250;

kernel/src/arch/x86_64/disk/ATA.cpp

@@ -0,0 +1,794 @@
+#include "arch/x86_64/disk/ATA.h"
+#include "Log.h"
+#include "arch/Serial.h"
+#include "arch/Timer.h"
+#include "arch/x86_64/IO.h"
+#include "fs/MBR.h"
+#include "memory/MemoryManager.h"
+#include <luna/Alignment.h>
+#include <luna/CType.h>
+#include <luna/SafeArithmetic.h>
+#include <luna/Vector.h>
+
+SharedPtr<ATA::Controller> g_controller;
+
+static void irq_handler(Registers* regs, void* ctx)
+{
+    ((ATA::Controller*)ctx)->irq_handler(regs);
+}
+
+static usize copy_ata_string(char* out, u16* in, usize size)
+{
+    for (usize i = 0; i < size; i += 2)
+    {
+        u16 val = in[i / 2];
+        out[i] = (u8)(val >> 8);
+        out[i + 1] = (u8)(val & 0xff);
+    }
+
+    out[size + 1] = '\0';
+
+    return size;
+}
+
+namespace ATA
+{
+    Result<void> Controller::scan()
+    {
+        // FIXME: Propagate errors.
+        PCI::scan(
+            [](const PCI::Device& device) {
+                if (!g_controller)
+                {
+                    auto controller = adopt_shared_if_nonnull(new (std::nothrow) Controller(device)).release_value();
+                    kinfoln("ata: Found ATA controller on PCI bus (%x:%x:%x)", device.address.bus,
+                            device.address.function, device.address.slot);
+
+                    if (controller->initialize()) g_controller = controller;
+                }
+            },
+            { .klass = 1, .subclass = 1 });
+
+        if (!g_controller) kwarnln("ata: No ATA controller found.");
+
+        return {};
+    }
+
+    bool Controller::initialize()
+    {
+        u16 command_old = PCI::read16(m_device.address, PCI::Command);
+        u16 command_new = command_old;
+
+        command_new &= ~PCI::CMD_INTERRUPT_DISABLE;
+        command_new |= PCI::CMD_IO_SPACE;
+        command_new |= PCI::CMD_BUS_MASTER;
+
+        if (command_new != command_old) PCI::write16(m_device.address, PCI::Command, command_new);
+
+        if (!m_primary_channel.initialize()) return false;
+
+        return m_secondary_channel.initialize();
+    }
+
+    void Controller::irq_handler(Registers* regs)
+    {
+        if (regs->irq == m_primary_channel.irq_line()) m_primary_channel.irq_handler(regs);
+        if (regs->irq == m_secondary_channel.irq_line()) m_secondary_channel.irq_handler(regs);
+    }
+
+    Controller::Controller(const PCI::Device& device)
+        : m_device(device), m_primary_channel(this, 0, {}), m_secondary_channel(this, 1, {})
+    {
+    }
+
+    Channel::Channel(Controller* controller, u8 channel_index, Badge<Controller>)
+        : m_controller(controller), m_channel_index(channel_index)
+    {
+    }
+
+    u8 Channel::read_register(Register reg)
+    {
+        return IO::inb(m_io_base + (u16)reg);
+    }
+
+    u16 Channel::read_data()
+    {
+        return IO::inw(m_io_base + (u16)Register::Data);
+    }
+
+    void Channel::write_data(u16 value)
+    {
+        IO::outw(m_io_base + (u16)Register::Data, value);
+    }
+
+    void Channel::write_register(Register reg, u8 value)
+    {
+        IO::outb(m_io_base + (u16)reg, value);
+    }
+
+    u8 Channel::read_control(ControlRegister reg)
+    {
+        return IO::inb(m_control_base + (u16)reg);
+    }
+
+    void Channel::write_control(ControlRegister reg, u8 value)
+    {
+        IO::outb(m_control_base + (u16)reg, value);
+    }
+
+    u8 Channel::read_bm(BusmasterRegister reg)
+    {
+        return IO::inb(m_busmaster_base + (u16)reg);
+    }
+
+    void Channel::write_bm(BusmasterRegister reg, u8 value)
+    {
+        IO::outb(m_busmaster_base + (u16)reg, value);
+    }
+
+    u32 Channel::read_prdt_address()
+    {
+        return IO::inl(m_busmaster_base + (u16)BusmasterRegister::PRDTAddress);
+    }
+
+    void Channel::write_prdt_address(u32 value)
+    {
+        IO::outl(m_busmaster_base + (u16)BusmasterRegister::PRDTAddress, value);
+    }
+
+    void Channel::delay_400ns()
+    {
+        // FIXME: We should use kernel_sleep(), but it doesn't support nanosecond granularity.
+        for (int i = 0; i < 14; i++) { [[maybe_unused]] volatile u8 val = read_control(ControlRegister::AltStatus); }
+    }
+
+    void Channel::select(u8 drive)
+    {
+        if (drive == m_current_drive) return;
+
+        u8 value = (drive << 4) | 0xa0;
+        write_register(Register::DriveSelect, value);
+
+        delay_400ns();
+
+        m_current_drive = drive;
+    }
+
+    void Channel::irq_handler(Registers*)
+    {
+        if (!(read_bm(BusmasterRegister::Status) & BMS_IRQPending)) return;
+
+        if (m_current_drive < 2 && m_drives[m_current_drive]) m_drives[m_current_drive]->irq_handler();
+
+        m_irq_called = true;
+
+        if (m_thread) m_thread->wake_up();
+    }
+
+    void Channel::prepare_for_irq()
+    {
+        m_thread = Scheduler::current();
+        m_irq_called = false;
+    }
+
+    void Channel::wait_for_irq()
+    {
+        if (!m_irq_called) kernel_wait_for_event();
+
+        m_irq_called = false;
+    }
+
+    bool Channel::wait_for_irq_or_timeout(u64 timeout)
+    {
+        if (!m_irq_called)
+        {
+            kernel_sleep(timeout);
+            m_irq_called = false;
+            return m_thread->sleep_ticks_left;
+        }
+
+        m_irq_called = false;
+        return true;
+    }
+
+    bool Channel::wait_for_reg_set(Register reg, u8 value, u64 timeout)
+    {
+        u64 begin = Timer::ticks_ms();
+        while (true)
+        {
+            u8 reg_value = reg == Register::Status ? read_control(ControlRegister::AltStatus) : read_register(reg);
+            if (reg_value & value) return true;
+            if ((Timer::ticks_ms() - begin) >= timeout) return false;
+            kernel_sleep(1);
+        }
+    }
+
+    bool Channel::wait_for_reg_clear(Register reg, u8 value, u64 timeout)
+    {
+        u64 begin = Timer::ticks_ms();
+        while (true)
+        {
+            u8 reg_value = reg == Register::Status ? read_control(ControlRegister::AltStatus) : read_register(reg);
+            if ((reg_value & value) == 0) return true;
+            if ((Timer::ticks_ms() - begin) >= timeout) return false;
+            kernel_sleep(1);
+        }
+    }
+
+    Result<void> Channel::wait_until_ready()
+    {
+        if (!wait_for_reg_clear(Register::Status, SR_Busy, 1000))
+        {
+            kwarnln("ata: Drive %d:%d timed out (BSY)", m_channel_index, m_current_drive);
+            return err(EIO);
+        }
+
+        if (!wait_for_reg_set(Register::Status, SR_DataRequestReady | SR_Error, 1000))
+        {
+            kwarnln("ata: Drive %d:%d timed out (DRQ)", m_channel_index, m_current_drive);
+            return err(EIO);
+        }
+
+        u8 status = read_control(ControlRegister::AltStatus);
+        if (status & SR_Error)
+        {
+            kwarnln("ata: An error occurred in drive %d:%d while waiting for data to become available", m_channel_index,
+                    m_current_drive);
+            return err(EIO);
+        }
+
+        return {};
+    }
+
+    bool Channel::initialize()
+    {
+        int offset = m_channel_index ? 2 : 0;
+        m_is_pci_native_mode = m_controller->device().type.prog_if & (1 << offset);
+
+        u16 control_port_base_address;
+        u16 io_base_address;
+
+        if (m_is_pci_native_mode)
+        {
+            auto io_base = m_controller->device().getBAR(m_channel_index ? 2 : 0);
+            if (!io_base.is_iospace())
+            {
+                kwarnln("ata: Channel %d's IO base BAR is not in IO space", m_channel_index);
+                return false;
+            }
+            io_base_address = io_base.port();
+
+            auto io_control = m_controller->device().getBAR(m_channel_index ? 3 : 1);
+            if (!io_control.is_iospace())
+            {
+                kwarnln("ata: Channel %d's control base BAR is not in IO space", m_channel_index);
+                return false;
+            }
+
+            control_port_base_address = io_control.port() + 2;
+        }
+        else
+        {
+            io_base_address = m_channel_index ? 0x170 : 0x1f0;
+            control_port_base_address = m_channel_index ? 0x376 : 0x3f6;
+        }
+
+        m_io_base = io_base_address;
+        m_control_base = control_port_base_address;
+
+        auto io_busmaster = m_controller->device().getBAR(4);
+        if (!io_busmaster.is_iospace())
+        {
+            kwarnln("ata: Channel %d's busmaster base BAR is not in IO space", m_channel_index);
+            return false;
+        }
+        m_busmaster_base = io_busmaster.port() + (u16)(m_channel_index * 8u);
+
+        if (m_is_pci_native_mode) m_interrupt_line = PCI::read8(m_controller->device().address, PCI::InterruptLine);
+        else
+            m_interrupt_line = m_channel_index ? 15 : 14;
+
+        write_control(ControlRegister::DeviceControl, 0);
+
+        for (u8 drive = 0; drive < 2; drive++)
+        {
+            ScopedKMutexLock<100> lock(m_lock);
+
+            select(drive);
+
+            if (read_register(Register::Status) == 0)
+            {
+                // No drive on this slot.
+                continue;
+            }
+
+            kinfoln("ata: Channel %d has a drive on slot %d!", m_channel_index, drive);
+
+            auto rc = adopt_shared_if_nonnull(new (std::nothrow) Drive(this, drive, {}));
+            if (rc.has_error())
+            {
+                kinfoln("ata: Failed to create drive object: %s", rc.error_string());
+                return false;
+            }
+
+            m_drives[drive] = rc.release_value();
+
+            if (!m_drives[drive]->initialize())
+            {
+                m_drives[drive] = {};
+                return false;
+            }
+        }
+
+        CPU::register_interrupt(m_interrupt_line, ::irq_handler, m_controller);
+
+        for (u8 drive = 0; drive < 2; drive++)
+        {
+            if (m_drives[drive])
+            {
+                if (!m_drives[drive]->post_initialize())
+                {
+                    m_drives[drive] = {};
+                    return false;
+                }
+
+                auto rc = ATADevice::create(m_drives[drive]);
+
+                if (rc.has_error())
+                {
+                    kwarnln("ata: Failed to register ATA drive %d:%d in DeviceRegistry", m_channel_index, drive);
+                    continue;
+                }
+
+                auto device = rc.release_value();
+                MBR::identify(device);
+            }
+        }
+
+        return true;
+    }
+
+    Drive::Drive(Channel* channel, u8 drive_index, Badge<Channel>) : m_channel(channel), m_drive_index(drive_index)
+    {
+    }
+
+    bool Drive::identify_ata()
+    {
+        m_channel->write_register(Register::Command, m_is_atapi ? CMD_Identify_Packet : CMD_Identify);
+
+        m_channel->delay_400ns();
+
+        if (!m_channel->wait_for_reg_clear(Register::Status, SR_Busy, 1000))
+        {
+            kwarnln("ata: Drive %d timed out clearing SR_Busy (waited for 1000 ms)", m_drive_index);
+            return false;
+        }
+
+        if (m_channel->read_register(Register::Status) & SR_Error)
+        {
+            u8 lbam = m_channel->read_register(Register::LBAMiddle);
+            u8 lbah = m_channel->read_register(Register::LBAHigh);
+
+            if ((lbam == 0x14 && lbah == 0xeb) || (lbam == 0x69 && lbah == 0x96))
+            {
+                if (!m_is_atapi)
+                {
+                    kinfoln("ata: Drive %d is ATAPI, sending IDENTIFY_PACKET command", m_drive_index);
+                    m_is_atapi = true;
+                    return identify_ata();
+                }
+            }
+
+            kwarnln("ata: IDENTIFY command for drive %d returned error", m_drive_index);
+
+            return false;
+        }
+
+        if (!m_channel->wait_for_reg_set(Register::Status, SR_DataRequestReady | SR_Error, 1000))
+        {
+            kwarnln("ata: Drive %d timed out setting SR_DataRequestReady (waited for 1000 ms)", m_drive_index);
+            return false;
+        }
+
+        u8 status = m_channel->read_register(Register::Status);
+        if (status & SR_Error)
+        {
+            kwarnln("ata: IDENTIFY command for drive %d returned error", m_drive_index);
+            return false;
+        }
+
+        for (usize i = 0; i < 256; i++)
+        {
+            u16 data = m_channel->read_data();
+            m_identify_words[i] = data;
+        }
+
+        return true;
+    }
+
+    bool Drive::initialize()
+    {
+        m_channel->select(m_drive_index);
+
+        m_channel->write_register(Register::SectorCount, 0);
+        m_channel->write_register(Register::LBALow, 0);
+        m_channel->write_register(Register::LBAMiddle, 0);
+        m_channel->write_register(Register::LBAHigh, 0);
+
+        if (!identify_ata()) return false;
+
+        m_serial.set_length(copy_ata_string(m_serial.data(), &m_identify_words[10], SERIAL_LEN));
+        m_revision.set_length(copy_ata_string(m_revision.data(), &m_identify_words[23], REVISION_LEN));
+        m_model.set_length(copy_ata_string(m_model.data(), &m_identify_words[27], MODEL_LEN));
+
+        m_serial.trim(" ");
+        m_revision.trim(" ");
+        m_model.trim(" ");
+
+        kinfoln("ata: Drive IDENTIFY returned serial='%s', revision='%s' and model='%s'", m_serial.chars(),
+                m_revision.chars(), m_model.chars());
+
+        auto status = m_channel->read_bm(BusmasterRegister::Status);
+        if (status & BMS_SimplexOnly)
+        {
+            kwarnln("ata: Drive %d will not use DMA because of simplex shenanigans", m_drive_index);
+            m_uses_dma = false;
+        }
+
+        auto frame = MemoryManager::alloc_frame();
+        if (frame.has_error() || frame.value() > 0xffffffff)
+        {
+            kwarnln("ata: Failed to allocate memory below the 32-bit limit for the PRDT");
+            return false;
+        }
+        m_dma_prdt_phys = frame.release_value();
+        m_dma_prdt = (prdt_entry*)MMU::translate_physical_address(m_dma_prdt_phys);
+
+        memset(m_dma_prdt, 0, ARCH_PAGE_SIZE);
+
+        frame = MemoryManager::alloc_frame();
+        if (frame.has_error() || frame.value() > 0xffffffff)
+        {
+            kwarnln("ata: Failed to allocate memory below the 32-bit limit for DMA memory");
+            return false;
+        }
+        m_dma_mem_phys = frame.release_value();
+        m_dma_mem = (void*)MMU::translate_physical_address(m_dma_mem_phys);
+
+        memset(const_cast<void*>(m_dma_mem), 0, ARCH_PAGE_SIZE);
+
+        if (m_uses_dma)
+        {
+            auto cmd = m_channel->read_bm(BusmasterRegister::Command);
+            cmd &= ~BMC_StartStop;
+            m_channel->write_bm(BusmasterRegister::Command, cmd);
+        }
+
+        return true;
+    }
+
+    bool Drive::post_initialize()
+    {
+        if (m_is_atapi)
+        {
+            atapi_packet packet;
+            memset(&packet, 0, sizeof(packet));
+            packet.command_bytes[0] = ATAPI_ReadCapacity;
+
+            atapi_read_capacity_reply reply;
+
+            if (send_packet_atapi_pio(&packet, &reply, sizeof(reply)).has_error())
+            {
+                kwarnln("ata: Failed to send Read Capacity command to ATAPI drive");
+                return false;
+            }
+
+            m_is_lba48 = true;
+
+            // FIXME: This assumes the host machine is little-endian.
+            u32 last_lba = __builtin_bswap32(reply.last_lba);
+            u32 sector_size = __builtin_bswap32(reply.sector_size);
+
+            m_block_count = last_lba + 1;
+            m_block_size = sector_size;
+        }
+        else
+        {
+            if (m_identify_data.big_lba) m_is_lba48 = true;
+
+            if (m_is_lba48) m_block_count = m_identify_data.sectors_48;
+            else
+                m_block_count = m_identify_data.sectors_28;
+
+            // FIXME: Should we check for CHS?
+
+            // FIXME: Maybe a different block size is in use? Detect that.
+            m_block_size = 512;
+        }
+
+        u64 total_capacity;
+        if (!safe_mul(m_block_count, m_block_size).try_set_value(total_capacity))
+        {
+            kwarnln("ata: Drive %d's total capacity is too large", m_drive_index);
+            return false;
+        }
+
+        kinfoln("ata: Drive %d capacity information: Block Count=%lu, Block Size=%lu, Total Capacity=%lu",
+                m_drive_index, m_block_count, m_block_size, total_capacity);
+
+        return true;
+    }
+
+    Result<void> Drive::send_packet_atapi_pio(const atapi_packet* packet, void* out, u16 response_size)
+    {
+        u8* ptr = (u8*)out;
+
+        m_channel->select(m_drive_index);
+
+        // We use PIO here.
+        m_channel->write_register(Register::Features, 0x00);
+
+        m_channel->write_register(Register::LBAMiddle, (u8)(response_size & 0xff));
+        m_channel->write_register(Register::LBAHigh, (u8)(response_size >> 8));
+
+        m_channel->write_register(Register::Command, CMD_Packet);
+
+        m_channel->delay_400ns();
+
+        usize i = 0;
+
+        TRY(m_channel->wait_until_ready());
+
+        for (int j = 0; j < 6; j++) m_channel->write_data(packet->command_words[j]);
+
+        while (i < response_size)
+        {
+            TRY(m_channel->wait_until_ready());
+
+            usize byte_count =
+                m_channel->read_register(Register::LBAHigh) << 8 | m_channel->read_register(Register::LBAMiddle);
+            usize word_count = byte_count / 2;
+
+            while (word_count--)
+            {
+                u16 value = m_channel->read_data();
+                ptr[0] = (u8)(value & 0xff);
+                ptr[1] = (u8)(value >> 8);
+                ptr += 2;
+            }
+
+            i += byte_count;
+        }
+
+        return {};
+    }
+
+#if 0
+
+    Result<void> Drive::send_packet_atapi_dma(const atapi_packet* packet, void* out, u16 response_size)
+    {
+        check(m_uses_dma);
+
+        m_channel->select(m_drive_index);
+
+        kdbgln("have selected");
+
+        // We use DMA here.
+        m_channel->write_register(Register::Features, 0x01);
+
+        m_channel->write_register(Register::LBAMiddle, 0);
+        m_channel->write_register(Register::LBAHigh, 0);
+
+        kdbgln("will do_dma_command");
+
+        TRY(do_dma_command(CMD_Packet, response_size, false));
+
+        TRY(m_channel->wait_until_ready());
+
+        kdbgln("send atapi packet data");
+
+        for (int j = 0; j < 6; j++) m_channel->write_data(packet->command_words[j]);
+
+        kdbgln("do dma transfer");
+
+        TRY(do_dma_transfer());
+
+        memcpy(out, const_cast<void*>(m_dma_mem), response_size);
+
+        return {};
+    }
+
+    Result<void> Drive::do_dma_command(u8 command, u16 count, bool write)
+    {
+        m_dma_prdt->address = (u32)m_dma_mem_phys;
+        m_dma_prdt->count = count;
+        m_dma_prdt->flags = END_OF_PRDT;
+
+        kdbgln("ata: do_dma_command: phys=%x, command=%x, count=%u, write=%d", m_dma_prdt->address, command, count,
+               write);
+
+        m_channel->write_prdt_address((u32)m_dma_prdt_phys);
+
+        auto status = m_channel->read_bm(BusmasterRegister::Status);
+        status &= ~(BMS_DMAFailure | BMS_IRQPending);
+        m_channel->write_bm(BusmasterRegister::Status, status);
+
+        auto cmd = m_channel->read_bm(BusmasterRegister::Command);
+        if (!write) cmd |= BMC_ReadWrite;
+        else
+            cmd &= ~BMC_ReadWrite;
+        m_channel->write_bm(BusmasterRegister::Command, cmd);
+
+        m_channel->prepare_for_irq();
+
+        m_channel->write_register(Register::Command, command);
+
+        cmd = m_channel->read_bm(BusmasterRegister::Command);
+        cmd |= BMC_StartStop;
+        m_channel->write_bm(BusmasterRegister::Command, cmd);
+
+        m_channel->delay_400ns();
+
+        return {};
+    }
+
+    Result<void> Drive::do_dma_transfer()
+    {
+        if (!m_channel->wait_for_irq_or_timeout(2000))
+        {
+            kwarnln("ata: Drive %d timed out (DMA)", m_drive_index);
+            return err(EIO);
+        }
+
+        u8 status = m_channel->read_control(ControlRegister::AltStatus);
+        kdbgln("ata: status after irq: %#x", status);
+
+        m_channel->delay_400ns();
+
+        auto cmd = m_channel->read_bm(BusmasterRegister::Command);
+        cmd &= ~BMC_StartStop;
+        m_channel->write_bm(BusmasterRegister::Command, cmd);
+
+        status = m_channel->read_bm(BusmasterRegister::Status);
+        m_channel->write_bm(BusmasterRegister::Status, status & ~(BMS_DMAFailure | BMS_IRQPending));
+
+        if (status & BMS_DMAFailure)
+        {
+            kwarnln("ata: DMA failure while trying to read drive %d", m_drive_index);
+            return err(EIO);
+        }
+
+        return {};
+    }
+
+#endif
+
+    Result<void> Drive::atapi_read_pio(u64 lba, void* out, usize size)
+    {
+        check(lba < m_block_count);
+        check(size <= ARCH_PAGE_SIZE);
+
+        atapi_packet read_packet;
+        memset(&read_packet, 0, sizeof(read_packet));
+        read_packet.command_bytes[0] = ATAPI_Read;
+        read_packet.command_bytes[2] = (lba >> 0x18) & 0xff;
+        read_packet.command_bytes[3] = (lba >> 0x10) & 0xff;
+        read_packet.command_bytes[4] = (lba >> 0x08) & 0xff;
+        read_packet.command_bytes[5] = (lba >> 0x00) & 0xff;
+        read_packet.command_bytes[9] = (u8)(size / m_block_size);
+
+        return send_packet_atapi_pio(&read_packet, out, (u16)size);
+    }
+
+    Result<void> Drive::read_lba(u64 lba, void* out, usize nblocks)
+    {
+        const usize blocks_per_page = ARCH_PAGE_SIZE / m_block_size;
+        if (m_is_atapi)
+        {
+            while (nblocks > blocks_per_page)
+            {
+                TRY(atapi_read_pio(lba, out, ARCH_PAGE_SIZE));
+                lba += blocks_per_page;
+                nblocks -= blocks_per_page;
+                out = offset_ptr(out, ARCH_PAGE_SIZE);
+            }
+            return atapi_read_pio(lba, out, nblocks * m_block_size);
+        }
+        else
+            todo();
+    }
+
+    void Drive::irq_handler()
+    {
+        // Clear the IRQ flag.
+        u8 status = m_channel->read_register(Register::Status);
+
+        if (status & SR_Error)
+        {
+            u8 error = m_channel->read_register(Register::Error);
+            (void)error;
+        }
+
+        if (m_uses_dma)
+        {
+            status = m_channel->read_bm(BusmasterRegister::Status);
+            if (status & BMS_DMAFailure) { kwarnln("ata: DMA failure in irq"); }
+            m_channel->write_bm(BusmasterRegister::Status, 4);
+        }
+    }
+}
+
+static u32 next_minor = 0;
+
+Result<String> ATA::Drive::create_drive_name(SharedPtr<ATA::Drive> drive)
+{
+    static u32 cd_index = 0;
+    static u32 sd_index = 0;
+
+    return String::format("%s%d"_sv, drive->m_is_atapi ? "cd" : "sd", drive->m_is_atapi ? cd_index++ : sd_index++);
+}
+
+Result<SharedPtr<Device>> ATADevice::create(SharedPtr<ATA::Drive> drive)
+{
+    auto device = TRY(adopt_shared_if_nonnull(new (std::nothrow) ATADevice()));
+    device->m_drive = drive;
+    device->m_device_path = TRY(ATA::Drive::create_drive_name(drive));
+    TRY(DeviceRegistry::register_special_device(DeviceRegistry::Disk, next_minor++, device, 0400));
+    return (SharedPtr<Device>)device;
+}
+
+Result<u64> ATADevice::read(u8* buf, usize offset, usize size) const
+{
+    if (size == 0) return 0;
+
+    if (offset > m_drive->capacity()) return 0;
+    if (offset + size > m_drive->capacity()) size = m_drive->capacity() - offset;
+
+    usize length = size;
+
+    auto block_size = m_drive->block_size();
+
+    ScopedKMutexLock<100> lock(m_drive->channel()->lock());
+
+    // FIXME: Don't always allocate this if we don't need it.
+    auto* temp = TRY(make_array<u8>(block_size));
+    auto guard = make_scope_guard([temp] { delete[] temp; });
+
+    if (offset % block_size)
+    {
+        // The size we need to read to round up to a block.
+        usize extra_size = block_size - (offset % block_size);
+        // Maybe we don't even want enough to get to the next block?
+        if (extra_size > size) extra_size = size;
+
+        TRY(m_drive->read_lba(offset / block_size, temp, 1));
+        memcpy(buf, temp + (offset % block_size), extra_size);
+        offset += extra_size;
+        size -= extra_size;
+        buf += extra_size;
+    }
+
+    while (size >= ARCH_PAGE_SIZE)
+    {
+        TRY(m_drive->read_lba(offset / block_size, buf, ARCH_PAGE_SIZE / block_size));
+        offset += ARCH_PAGE_SIZE;
+        size -= ARCH_PAGE_SIZE;
+        buf += ARCH_PAGE_SIZE;
+    }
+
+    while (size >= block_size)
+    {
+        TRY(m_drive->read_lba(offset / block_size, buf, 1));
+        offset += block_size;
+        size -= block_size;
+        buf += block_size;
+    }
+
+    if (size)
+    {
+        TRY(m_drive->read_lba(offset / block_size, temp, 1));
+        memcpy(buf, temp, size);
+    }
+
+    return length;
+}

kernel/src/arch/x86_64/disk/ATA.h

@@ -0,0 +1,344 @@
+#pragma once
+#include "arch/PCI.h"
+#include "fs/devices/DeviceRegistry.h"
+#include "lib/KMutex.h"
+#include <luna/Atomic.h>
+#include <luna/SharedPtr.h>
+#include <luna/StaticString.h>
+
+namespace ATA
+{
+    enum class Register : u16
+    {
+        Data = 0,
+        Error = 1,
+        Features = 1,
+        SectorCount = 2,
+        SectorNumber = 3,
+        LBALow = 3,
+        CylinderLow = 4,
+        LBAMiddle = 4,
+        CylinderHigh = 5,
+        LBAHigh = 5,
+        DriveSelect = 6,
+        Status = 7,
+        Command = 7,
+    };
+
+    enum class ControlRegister : u16
+    {
+        AltStatus = 0,
+        DeviceControl = 0,
+        DriveAddress = 1,
+    };
+
+    enum class BusmasterRegister : u16
+    {
+        Command = 0,
+        Status = 2,
+        PRDTAddress = 4,
+    };
+
+    enum StatusRegister : u8
+    {
+        SR_Busy = 0x80,
+        SR_DriveReady = 0x40,
+        SR_WriteFault = 0x20,
+        SR_SeekComplete = 0x10,
+        SR_DataRequestReady = 0x08,
+        SR_CorrectedData = 0x04,
+        SR_Index = 0x02,
+        SR_Error = 0x01
+    };
+
+    enum CommandRegister : u8
+    {
+        CMD_Identify = 0xec,
+        CMD_Packet = 0xa0,
+        CMD_Identify_Packet = 0xa1
+    };
+
+    enum BusMasterStatus : u8
+    {
+        BMS_SimplexOnly = 0x80,
+        BMS_SlaveInit = 0x40,
+        BMS_MasterInit = 0x20,
+        BMS_IRQPending = 0x4,
+        BMS_DMAFailure = 0x2,
+        BMS_DMAMode = 0x1
+    };
+
+    enum BusMasterCommand : u8
+    {
+        BMC_ReadWrite = 0x8,
+        BMC_StartStop = 0x1,
+    };
+
+    struct ATAIdentify
+    {
+        u16 flags;
+        u16 unused1[9];
+        char serial[20];
+        u16 unused2[3];
+        char firmware[8];
+        char model[40];
+        u16 sectors_per_int;
+        u16 unused3;
+        u16 capabilities[2];
+        u16 unused4[2];
+        u16 valid_ext_data;
+        u16 unused5[5];
+        u16 size_of_rw_mult;
+        u32 sectors_28;
+        u16 unused6[21];
+        u16 unused7 : 10;
+        u16 big_lba : 1;
+        u16 unused8 : 5;
+        u16 unused9[17];
+        u64 sectors_48;
+        u16 unused10[152];
+    };
+
+    enum ATAPICommand : u8
+    {
+        ATAPI_ReadCapacity = 0x25,
+        ATAPI_Read = 0xa8,
+    };
+
+    class Controller;
+    class Channel;
+
+    struct prdt_entry
+    {
+        u32 address;
+        u16 count;
+        u16 flags;
+    };
+
+    struct atapi_packet
+    {
+        union {
+            u16 command_words[6];
+            u8 command_bytes[12];
+        };
+    };
+
+    struct atapi_read_capacity_reply
+    {
+        u32 last_lba;
+        u32 sector_size;
+    };
+
+    static constexpr u16 END_OF_PRDT = (1 << 15);
+
+    class Drive
+    {
+      public:
+        Drive(Channel* channel, u8 drive_index, Badge<Channel>);
+
+        bool initialize();
+
+        bool post_initialize();
+
+        void irq_handler();
+
+        usize block_size() const
+        {
+            return m_block_size;
+        }
+
+        usize block_count() const
+        {
+            return m_block_count;
+        }
+
+        usize capacity() const
+        {
+            return m_block_count * m_block_size;
+        }
+
+        Channel* channel() const
+        {
+            return m_channel;
+        }
+
+        Result<void> read_lba(u64 lba, void* out, usize nblocks);
+
+        static Result<String> create_drive_name(SharedPtr<ATA::Drive> drive);
+
+      private:
+        bool identify_ata();
+
+        Result<void> send_packet_atapi_pio(const atapi_packet* packet, void* out, u16 response_size);
+#if 0
+        Result<void> send_packet_atapi_dma(const atapi_packet* packet, void* out, u16 response_size);
+
+        Result<void> do_dma_command(u8 command, u16 count, bool write);
+        Result<void> do_dma_transfer();
+#endif
+
+        Result<void> atapi_read_pio(u64 lba, void* out, usize size);
+
+        Channel* m_channel;
+
+        u8 m_drive_index;
+        union {
+            u16 m_identify_words[256];
+            ATAIdentify m_identify_data;
+        };
+
+        bool m_is_atapi { false };
+        bool m_uses_dma { true };
+
+        bool m_is_lba48;
+        u64 m_block_count;
+        u64 m_block_size;
+
+        prdt_entry* m_dma_prdt;
+        u64 m_dma_prdt_phys;
+        volatile void* m_dma_mem;
+        u64 m_dma_mem_phys;
+
+        constexpr static usize SERIAL_LEN = 20;
+        constexpr static usize REVISION_LEN = 8;
+        constexpr static usize MODEL_LEN = 40;
+
+        StaticString<SERIAL_LEN> m_serial;
+        StaticString<REVISION_LEN> m_revision;
+        StaticString<MODEL_LEN> m_model;
+    };
+
+    class Channel
+    {
+      public:
+        Channel(Controller* controller, u8 channel_index, Badge<Controller>);
+
+        u8 read_register(Register reg);
+        u16 read_data();
+        void write_data(u16 value);
+        void write_register(Register reg, u8 value);
+        u8 read_control(ControlRegister reg);
+        void write_control(ControlRegister reg, u8 value);
+
+        u8 read_bm(BusmasterRegister reg);
+        void write_bm(BusmasterRegister reg, u8 value);
+        u32 read_prdt_address();
+        void write_prdt_address(u32 value);
+
+        bool wait_for_reg_set(Register reg, u8 value, u64 timeout);
+        bool wait_for_reg_clear(Register reg, u8 value, u64 timeout);
+
+        Result<void> wait_until_ready();
+
+        void delay_400ns();
+
+        void prepare_for_irq();
+
+        void wait_for_irq();
+        bool wait_for_irq_or_timeout(u64 timeout);
+        void irq_handler(Registers*);
+
+        u8 irq_line() const
+        {
+            return m_interrupt_line;
+        }
+
+        KMutex<100>& lock()
+        {
+            return m_lock;
+        }
+
+        void select(u8 drive);
+
+        bool initialize();
+
+      private:
+        Controller* m_controller;
+        u8 m_channel_index;
+        bool m_is_pci_native_mode;
+
+        u8 m_interrupt_line;
+
+        KMutex<100> m_lock {};
+
+        Thread* m_thread { nullptr };
+
+        u16 m_io_base;
+        u16 m_control_base;
+        u16 m_busmaster_base;
+
+        bool m_irq_called { false };
+
+        u8 m_current_drive = (u8)-1;
+
+        SharedPtr<Drive> m_drives[2];
+    };
+
+    class Controller
+    {
+      public:
+        static Result<void> scan();
+
+        const PCI::Device& device() const
+        {
+            return m_device;
+        }
+
+        bool initialize();
+
+        void irq_handler(Registers*);
+
+      private:
+        Controller(const PCI::Device& device);
+        PCI::Device m_device;
+        Channel m_primary_channel;
+        Channel m_secondary_channel;
+    };
+
+}
+
+class ATADevice : public Device
+{
+  public:
+    // Initializer for DeviceRegistry.
+    static Result<SharedPtr<Device>> create(SharedPtr<ATA::Drive> drive);
+
+    Result<usize> read(u8*, usize, usize) const override;
+
+    Result<usize> write(const u8*, usize, usize) override
+    {
+        return err(ENOTSUP);
+    }
+
+    bool blocking() const override
+    {
+        return false;
+    }
+
+    bool is_block_device() const override
+    {
+        return true;
+    }
+
+    usize size() const override
+    {
+        return m_drive->capacity();
+    }
+
+    Result<usize> block_size() const override
+    {
+        return m_drive->block_size();
+    }
+
+    StringView device_path() const override
+    {
+        return m_device_path.view();
+    }
+
+    virtual ~ATADevice() = default;
+
+  private:
+    ATADevice() = default;
+    SharedPtr<ATA::Drive> m_drive;
+    String m_device_path;
+};

kernel/src/arch/x86_64/init/IDT.cpp

@@ -84,6 +84,20 @@ INT(20);
 INT(21);
 INT(32);
 INT(33);
+INT(34);
+INT(35);
+INT(36);
+INT(37);
+INT(38);
+INT(39);
+INT(40);
+INT(41);
+INT(42);
+INT(43);
+INT(44);
+INT(45);
+INT(46);
+INT(47);
 INT(66);
 
 void setup_idt()
@@ -112,6 +126,20 @@ void setup_idt()
     TRAP(21);
     IRQ(32);
     IRQ(33);
+    IRQ(34);
+    IRQ(35);
+    IRQ(36);
+    IRQ(37);
+    IRQ(38);
+    IRQ(39);
+    IRQ(40);
+    IRQ(41);
+    IRQ(42);
+    IRQ(43);
+    IRQ(44);
+    IRQ(45);
+    IRQ(46);
+    IRQ(47);
     SYS(66);
 
     static IDTR idtr;

kernel/src/arch/x86_64/init/PIC.cpp

@@ -36,9 +36,18 @@ void remap_pic()
     IO::outb(PIC2_DATA, ICW4_8086);
     io_delay();
 
-    IO::outb(PIC1_DATA, 0b11111100);
+    IO::outb(PIC1_DATA, 0b11111111);
     io_delay();
     IO::outb(PIC2_DATA, 0b11111111);
+    io_delay();
+}
+
+void change_pic_masks(u8 pic1_mask, u8 pic2_mask)
+{
+    IO::outb(PIC1_DATA, pic1_mask);
+    io_delay();
+    IO::outb(PIC2_DATA, pic2_mask);
+    io_delay();
 }
 
 void pic_eoi(unsigned char irq)
@@ -49,5 +58,5 @@ void pic_eoi(unsigned char irq)
 
 void pic_eoi(Registers* regs)
 {
-    pic_eoi((unsigned char)(regs->error)); // On IRQs, the error code is the IRQ number
+    pic_eoi((unsigned char)(regs->irq));
 }

kernel/src/boot/Init.cpp

@@ -27,6 +27,7 @@ void Init::early_init()
     CPU::disable_interrupts();
 
     Framebuffer::init();
+    set_text_console_initialized();
 
 #ifdef DEBUG_MODE
     constexpr bool should_log_console = true;
@@ -40,6 +41,6 @@ void Init::early_init()
     MemoryManager::init();
     InitRD::initialize();
 
-    MemoryManager::protect_kernel_sections().expect_release_value(
-        "Failed to remap kernel data sections as non-executable / read-only");
+    mark_critical(MemoryManager::protect_kernel_sections(),
+                  "Could not remap kernel data sections as non-executable / read-only");
 }

kernel/src/fs/GPT.cpp

@@ -0,0 +1,73 @@
+#include "fs/GPT.h"
+#include <luna/CRC32.h>
+
+static const u8 null_guid[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+namespace GPT
+{
+    Result<bool> identify(SharedPtr<Device> device)
+    {
+        if (!device->is_block_device()) return false;
+
+        Header header;
+        // The header is at LBA 1.
+        usize nread = TRY(device->read((u8*)&header, 1 * GPT_SECTOR_SIZE, sizeof(header)));
+        check(nread == sizeof(header));
+
+        if (memcmp(header.signature, GPT_SIGNATURE, GPT_SIGNATURE_LENGTH)) return false;
+
+        kinfoln("gpt: Found GUID partition table on device %s, revision %#.8x, with space for %d partition entries!",
+                device->device_path().chars(), header.revision, header.num_partitions);
+
+        if (header.revision != GPT_REVISION)
+        {
+            kwarnln("gpt: GUID partition table revision not supported!");
+            return false;
+        }
+
+        check(header.hdr_size == 0x5c);
+        check(header.reserved == 0);
+        check(header.this_lba == 1);
+
+#if 0
+        const u32 chksum = checksum_gpt(header);
+
+        if (chksum != header.checksum)
+        {
+            kwarnln("gpt: Header checksum does not match, %#.8x != %#.8x!", chksum, header.checksum);
+            return false;
+        }
+#endif
+
+        u64 partition_table_start = header.partition_table_lba * GPT_SECTOR_SIZE;
+
+        u32 partition_index = 1;
+
+        auto* table = TRY(make_array<PartitionEntry>(header.num_partitions));
+        auto guard = make_scope_guard([table] { delete[] table; });
+
+        nread = TRY(device->read((u8*)table, partition_table_start, sizeof(PartitionEntry) * header.num_partitions));
+        check(nread == sizeof(PartitionEntry) * header.num_partitions);
+
+        for (u32 i = 0; i < header.num_partitions; i++)
+        {
+            PartitionEntry& entry = table[i];
+
+            if (!memcmp(entry.type_guid, null_guid, 16)) continue;
+
+            kinfoln("gpt: Partition entry #%u is active: start=%lu, end=%lu", i, entry.start_lba, entry.end_lba);
+
+            TRY(MBR::PartitionDevice::create(device, entry.start_lba, entry.end_lba - entry.start_lba,
+                                             partition_index++));
+        }
+
+        return true;
+    }
+
+    u32 checksum_gpt(Header header)
+    {
+        header.checksum = 0;
+
+        return CRC32::checksum((u8*)&header, 0x5c);
+    }
+}

kernel/src/fs/GPT.h

@@ -0,0 +1,43 @@
+#pragma once
+#include "fs/MBR.h"
+
+#define GPT_SIGNATURE "EFI PART"
+#define GPT_SIGNATURE_LENGTH 8
+#define GPT_SECTOR_SIZE 512ul
+#define GPT_REVISION 0x00010000
+
+namespace GPT
+{
+    struct [[gnu::packed]] Header
+    {
+        u8 signature[GPT_SIGNATURE_LENGTH];
+        u32 revision;
+        u32 hdr_size;
+        u32 checksum;
+        u32 reserved;
+        u64 this_lba;
+        u64 alternate_lba;
+        u64 first_usable;
+        u64 last_usable;
+        u8 guid[16];
+        u64 partition_table_lba;
+        u32 num_partitions;
+        u32 partition_entry_size;
+        u32 partition_table_checksum;
+        char blank[512 - 0x5c];
+    };
+
+    struct [[gnu::packed]] PartitionEntry
+    {
+        u8 type_guid[16];
+        u8 unique_guid[16];
+        u64 start_lba;
+        u64 end_lba;
+        u64 attributes;
+        char partition_name[72]; // Could be more or less, see Header::partition_entry_size - 0x38 for the actual size.
+    };
+
+    Result<bool> identify(SharedPtr<Device> device);
+
+    u32 checksum_gpt(Header header);
+}

kernel/src/fs/MBR.cpp

@@ -0,0 +1,84 @@
+#include "fs/MBR.h"
+#include "Log.h"
+#include "fs/GPT.h"
+#include <luna/CType.h>
+
+static Result<String> create_partition_name(SharedPtr<Device> host_device, u32 partition_index)
+{
+    auto host_path = host_device->device_path();
+
+    char last = host_path[host_path.length() - 1];
+
+    if (_isdigit(last)) return String::format("%sp%d"_sv, host_path.chars(), partition_index);
+
+    return String::format("%s%d"_sv, host_path.chars(), partition_index);
+}
+
+namespace MBR
+{
+    Result<void> PartitionDevice::create(SharedPtr<Device> host_device, usize start_block, usize num_blocks,
+                                         u32 partition_index)
+    {
+        static u32 next_minor = 0;
+        auto device = TRY(adopt_shared_if_nonnull(new (std::nothrow) PartitionDevice()));
+        device->m_host_device = host_device;
+        device->m_start_offset = start_block * device->m_block_size;
+        device->m_num_blocks = num_blocks;
+        device->m_device_path = TRY(create_partition_name(host_device, partition_index));
+        return DeviceRegistry::register_special_device(DeviceRegistry::DiskPartition, next_minor++, device, 0400);
+    }
+
+    Result<usize> PartitionDevice::read(u8* buf, usize offset, usize length) const
+    {
+        if (length == 0) return 0;
+
+        if (offset > size()) return 0;
+        if (offset + length > size()) length = size() - offset;
+
+        return m_host_device->read(buf, m_start_offset + offset, length);
+    }
+
+    Result<usize> PartitionDevice::write(const u8* buf, usize offset, usize length)
+    {
+        if (length == 0) return 0;
+
+        if (offset > size()) return 0;
+        if (offset + length > size()) length = size() - offset;
+
+        return m_host_device->write(buf, m_start_offset + offset, length);
+    }
+
+    Result<bool> identify(SharedPtr<Device> device)
+    {
+        // Cannot read a partition table from a character device! Who is even coming up with this silliness?
+        if (!device->is_block_device()) return false;
+
+        DiskHeader hdr;
+        const usize nread = TRY(device->read((u8*)&hdr, 0, sizeof(hdr)));
+        check(nread == 512);
+
+        if (hdr.signature[0] != MBR_SIGNATURE_1 || hdr.signature[1] != MBR_SIGNATURE_2) return false;
+
+        u32 partition_index = 1;
+
+        for (int i = 0; i < 4; i++)
+        {
+            const auto& part = hdr.partitions[i];
+            if (part.partition_type == 0xee) return GPT::identify(device);
+        }
+
+        for (int i = 0; i < 4; i++)
+        {
+            const auto& part = hdr.partitions[i];
+            if (part.partition_type == 0) continue; // Not active.
+
+            bool bootable = part.attributes & MBR_BOOTABLE;
+            kinfoln("mbr: Partition #%d is active: bootable=%d, type=%x, start=%d, sectors=%d", i, bootable,
+                    part.partition_type, part.start_lba, part.num_sectors);
+
+            TRY(PartitionDevice::create(device, part.start_lba, part.num_sectors, partition_index++));
+        }
+
+        return true;
+    }
+}

kernel/src/fs/MBR.h

@@ -0,0 +1,83 @@
+#pragma once
+
+#include "fs/devices/DeviceRegistry.h"
+#include <luna/String.h>
+#include <luna/Types.h>
+
+#define MBR_BOOTABLE 0x80
+
+#define MBR_SIGNATURE_1 0x55
+#define MBR_SIGNATURE_2 0xAA
+
+namespace MBR
+{
+    struct [[gnu::packed]] PartitionHeader
+    {
+        u8 attributes;
+        u8 chs_start[3];
+        u8 partition_type;
+        u8 chs_end[3];
+        u32 start_lba;
+        u32 num_sectors;
+    };
+
+    struct [[gnu::packed]] DiskHeader
+    {
+        u8 mbr_code[440];
+        u8 disk_id[4];
+        u8 reserved[2];
+        PartitionHeader partitions[4];
+        u8 signature[2];
+    };
+
+    class PartitionDevice : public Device
+    {
+      public:
+        // Initializer for DeviceRegistry.
+        static Result<void> create(SharedPtr<Device> host_device, usize start_block, usize num_blocks,
+                                   u32 partition_index);
+
+        Result<usize> read(u8*, usize, usize) const override;
+
+        Result<usize> write(const u8* buf, usize offset, usize length) override;
+
+        bool blocking() const override
+        {
+            return false;
+        }
+
+        bool is_block_device() const override
+        {
+            return true;
+        }
+
+        usize size() const override
+        {
+            return m_num_blocks * m_block_size;
+        }
+
+        Result<usize> block_size() const override
+        {
+            return m_block_size;
+        }
+
+        StringView device_path() const override
+        {
+            return m_device_path.view();
+        }
+
+        virtual ~PartitionDevice() = default;
+
+      private:
+        PartitionDevice() = default;
+        SharedPtr<Device> m_host_device;
+        usize m_block_size { 512ul };
+        usize m_start_offset;
+        usize m_num_blocks;
+        String m_device_path;
+    };
+
+    static_assert(sizeof(DiskHeader) == 512ul);
+
+    Result<bool> identify(SharedPtr<Device> device);
+};

kernel/src/fs/Mount.cpp

@@ -6,14 +6,18 @@ Result<SharedPtr<VFS::Inode>> MountInode::create(SharedPtr<VFS::Inode> source, S
 {
     auto inode = TRY(adopt_shared_if_nonnull(new (std::nothrow) MountInode()));
 
-    inode->m_source = source;
     inode->m_mountee = fs;
     inode->m_mount_root_inode = fs->root_inode();
 
-    auto parent = TRY(source->find(".."));
-    TRY(fs->set_mount_dir(parent));
+    if (source)
+    {
+        inode->m_source = source;
 
-    source->add_handle();
+        auto parent = TRY(source->find(".."));
+        TRY(fs->set_mount_dir(parent));
+
+        source->add_handle();
+    }
 
     g_mounts.append(inode.ptr());
 
@@ -22,5 +26,5 @@ Result<SharedPtr<VFS::Inode>> MountInode::create(SharedPtr<VFS::Inode> source, S
 
 MountInode::~MountInode()
 {
-    m_source->remove_handle();
+    if (m_source) m_source->remove_handle();
 }

kernel/src/fs/Mount.h

@@ -142,10 +142,27 @@ class MountInode : public VFS::Inode, public LinkedListNode<MountInode>
         return m_mount_root_inode->replace_entry(inode, name);
     }
 
+    Result<void> set_source(SharedPtr<VFS::Inode> source)
+    {
+        if (m_source) m_source->remove_handle();
+
+        m_source = source;
+
+        if (source)
+        {
+            auto parent = TRY(source->find(".."));
+            TRY(m_mountee->set_mount_dir(parent));
+
+            source->add_handle();
+        }
+
+        return {};
+    }
+
     virtual ~MountInode();
 
   private:
-    SharedPtr<VFS::Inode> m_source;
+    SharedPtr<VFS::Inode> m_source {};
     SharedPtr<VFS::FileSystem> m_mountee;
     SharedPtr<VFS::Inode> m_mount_root_inode;
 

kernel/src/fs/VFS.cpp

@@ -8,11 +8,11 @@
 
 namespace VFS
 {
-    SharedPtr<FileSystem> root_fs;
+    SharedPtr<VFS::Inode> g_root_inode = {};
 
     Inode& root_inode()
     {
-        return *root_fs->root_inode();
+        return *g_root_inode;
     }
 
     static constexpr int MAX_SYMLINKS = 8;
@@ -40,7 +40,7 @@ namespace VFS
 
                 SharedPtr<VFS::Inode> symlink_root;
 
-                if (PathParser::is_absolute(link.chars())) symlink_root = root_fs->root_inode();
+                if (PathParser::is_absolute(link.chars())) symlink_root = g_root_inode;
                 else
                     symlink_root = parent_inode;
 
@@ -60,7 +60,7 @@ namespace VFS
     {
         SharedPtr<Inode> current_inode;
 
-        if (PathParser::is_absolute(path) || !working_directory) current_inode = root_fs->root_inode();
+        if (PathParser::is_absolute(path) || !working_directory) current_inode = g_root_inode;
         else
             current_inode = working_directory;
 
@@ -71,14 +71,13 @@ namespace VFS
 
     Result<SharedPtr<Inode>> create_directory(const char* path, Credentials auth, SharedPtr<Inode> working_directory)
     {
-        auto parser = TRY(PathParser::create(path));
-        auto parent_path = TRY(parser.dirname());
+        auto parent_path = TRY(PathParser::dirname(path));
 
         auto parent_inode = TRY(resolve_path(parent_path.chars(), auth, working_directory));
 
         if (!can_write(parent_inode, auth)) return err(EACCES);
 
-        auto child_name = TRY(parser.basename());
+        auto child_name = TRY(PathParser::basename(path));
 
         TRY(validate_filename(child_name.view()));
 
@@ -87,14 +86,13 @@ namespace VFS
 
     Result<SharedPtr<Inode>> create_file(const char* path, Credentials auth, SharedPtr<Inode> working_directory)
     {
-        auto parser = TRY(PathParser::create(path));
-        auto parent_path = TRY(parser.dirname());
+        auto parent_path = TRY(PathParser::dirname(path));
 
         auto parent_inode = TRY(resolve_path(parent_path.chars(), auth, working_directory));
 
         if (!can_write(parent_inode, auth)) return err(EACCES);
 
-        auto child_name = TRY(parser.basename());
+        auto child_name = TRY(PathParser::basename(path));
 
         TRY(validate_filename(child_name.view()));
 
@@ -168,6 +166,11 @@ namespace VFS
         return inode->mode() & S_ISGID;
     }
 
+    bool is_sticky(SharedPtr<Inode> inode)
+    {
+        return inode->mode() & S_ISVTX;
+    }
+
     bool is_seekable(SharedPtr<Inode> inode)
     {
         return inode->type() != InodeType::FIFO && inode->type() != InodeType::CharacterDevice;
@@ -175,7 +178,45 @@ namespace VFS
 
     Result<void> mount_root(SharedPtr<VFS::FileSystem> fs)
     {
-        root_fs = fs;
+        check(!g_root_inode);
+
+        g_root_inode = TRY(MountInode::create({}, fs));
+
+        return {};
+    }
+
+    Result<void> pivot_root(const char* new_root, const char* put_old, SharedPtr<VFS::Inode> working_directory)
+    {
+        auto new_root_parent = TRY(PathParser::dirname(new_root));
+        auto new_root_path = TRY(PathParser::basename(new_root));
+
+        auto new_root_parent_inode = TRY(VFS::resolve_path(new_root_parent.chars(), Credentials {}, working_directory));
+        auto new_root_inode = TRY(new_root_parent_inode->find(new_root_path.chars()));
+
+        if (new_root_inode->type() != VFS::InodeType::Directory) return err(ENOTDIR);
+        if (!new_root_inode->is_mountpoint()) return err(EINVAL);
+        if (new_root_inode->fs() == g_root_inode->fs()) return err(EBUSY);
+
+        auto parent_path = TRY(PathParser::dirname(put_old));
+        auto child = TRY(PathParser::basename(put_old));
+
+        kdbgln("vfs: Pivoting root from / to %s, using %s as new root", put_old, new_root);
+
+        auto parent_inode = TRY(resolve_path(parent_path.chars(), Credentials {}, working_directory));
+
+        auto inode = TRY(parent_inode->find(child.chars()));
+        if (inode->type() != VFS::InodeType::Directory) return err(ENOTDIR);
+        if (inode->is_mountpoint()) return err(EBUSY);
+        if (inode->fs() != new_root_inode->fs()) return err(EINVAL);
+
+        auto mount = g_root_inode;
+
+        TRY(parent_inode->replace_entry(mount, child.chars()));
+        ((MountInode*)mount.ptr())->set_source(inode);
+
+        g_root_inode = new_root_inode;
+        TRY(new_root_parent_inode->replace_entry(((MountInode*)g_root_inode.ptr())->source(), new_root_path.chars()));
+        ((MountInode*)g_root_inode.ptr())->set_source({});
 
         return {};
     }
@@ -183,11 +224,10 @@ namespace VFS
     Result<void> mount(const char* path, SharedPtr<VFS::FileSystem> fs, Credentials auth,
                        SharedPtr<VFS::Inode> working_directory)
     {
-        auto parser = TRY(PathParser::create(path));
-        auto parent_path = TRY(parser.dirname());
-        auto child = TRY(parser.basename());
+        auto parent_path = TRY(PathParser::dirname(path));
+        auto child = TRY(PathParser::basename(path));
 
-        kdbgln("vfs: Mounting filesystem on target %s", path);
+        kinfoln("vfs: Mounting filesystem on target %s", path);
 
         auto parent_inode = TRY(resolve_path(parent_path.chars(), auth, working_directory));
 
@@ -204,11 +244,12 @@ namespace VFS
 
     Result<void> umount(const char* path, Credentials auth, SharedPtr<VFS::Inode> working_directory)
     {
-        auto parser = TRY(PathParser::create(path));
-        auto parent_path = TRY(parser.dirname());
-        auto child = TRY(parser.basename());
+        auto parent_path = TRY(PathParser::dirname(path));
+        auto child = TRY(PathParser::basename(path));
 
-        kdbgln("vfs: Unmounting filesystem on target %s", path);
+        if (child.view() == "/") return err(EBUSY);
+
+        kinfoln("vfs: Unmounting filesystem on target %s", path);
 
         auto parent_inode = TRY(resolve_path(parent_path.chars(), auth, working_directory));
 

kernel/src/fs/VFS.h

@@ -290,12 +290,14 @@ namespace VFS
     bool can_write(SharedPtr<Inode> inode, Credentials auth);
     bool is_setuid(SharedPtr<Inode> inode);
     bool is_setgid(SharedPtr<Inode> inode);
+    bool is_sticky(SharedPtr<Inode> inode);
 
     bool is_seekable(SharedPtr<Inode> inode);
 
     Inode& root_inode();
 
     Result<void> mount_root(SharedPtr<VFS::FileSystem> fs);
+    Result<void> pivot_root(const char* new_root, const char* put_old, SharedPtr<VFS::Inode> working_directory);
     Result<void> mount(const char* path, SharedPtr<VFS::FileSystem> fs, Credentials auth,
                        SharedPtr<Inode> working_directory = {});
 

kernel/src/fs/devices/ConsoleDevice.cpp

@@ -18,7 +18,7 @@ static bool g_echo { true };
 Result<void> ConsoleDevice::create()
 {
     auto device = (SharedPtr<Device>)TRY(make_shared<ConsoleDevice>());
-    return DeviceRegistry::register_special_device(DeviceRegistry::Console, 0, device, "console");
+    return DeviceRegistry::register_special_device(DeviceRegistry::Console, 0, device);
 }
 
 Result<usize> ConsoleDevice::read(u8* buf, usize, usize length) const
@@ -89,11 +89,11 @@ void ConsoleDevice::did_press_key(char key)
         return;
     }
 
-    g_temp_input.try_append((u8)key).value();
+    g_temp_input.try_append((u8)key);
 
     if (key == '\n')
     {
-        g_console_input.append_data(g_temp_input.data(), g_temp_input.size()).value();
+        g_console_input.append_data(g_temp_input.data(), g_temp_input.size());
         g_temp_input.clear();
     }
 

kernel/src/fs/devices/ConsoleDevice.h

@@ -17,5 +17,10 @@ class ConsoleDevice : public Device
 
     Result<u64> ioctl(int request, void* arg) override;
 
+    StringView device_path() const override
+    {
+        return "console";
+    }
+
     virtual ~ConsoleDevice() = default;
 };

kernel/src/fs/devices/Device.h

@@ -1,4 +1,5 @@
 #pragma once
+#include "Log.h"
 #include <luna/Result.h>
 
 class Device
@@ -23,6 +24,17 @@ class Device
         return false;
     }
 
+    virtual Result<usize> block_size() const
+    {
+        // Block devices should override this function.
+        kwarnln("Device::block_size() was called on a character device or block device without block size");
+
+        return err(ENOTSUP);
+    }
+
+    // Path in devfs.
+    virtual StringView device_path() const = 0;
+
     virtual bool blocking() const = 0;
 
     virtual ~Device() = default;

kernel/src/fs/devices/DeviceRegistry.cpp

@@ -21,8 +21,6 @@ struct DeviceDescriptor
 
 Vector<DeviceDescriptor> g_available_devices = {};
 
-SharedPtr<VFS::FileSystem> g_device_fs;
-
 namespace DeviceRegistry
 {
     Result<SharedPtr<Device>> fetch_special_device(u32 major, u32 minor)
@@ -35,39 +33,37 @@ namespace DeviceRegistry
         return err(ENODEV);
     }
 
-    Result<void> create_special_device_inode(DeviceDescriptor& descriptor)
+    Result<void> create_special_device_inode(SharedPtr<VFS::FileSystem> fs, const DeviceDescriptor& descriptor)
     {
-        auto inode = TRY(g_device_fs->create_device_inode(descriptor.major, descriptor.minor));
+        auto inode = TRY(fs->create_device_inode(descriptor.major, descriptor.minor));
         inode->chmod(descriptor.mode);
-        TRY(g_device_fs->root_inode()->add_entry(inode, descriptor.name));
+        TRY(fs->root_inode()->add_entry(inode, descriptor.name));
 
         return {};
     }
 
-    Result<void> register_special_device(u32 major, u32 minor, SharedPtr<Device> device, const char* name, mode_t mode)
+    Result<void> register_special_device(u32 major, u32 minor, SharedPtr<Device> device, mode_t mode)
     {
         for (const auto& descriptor : g_available_devices)
         {
             if (descriptor.major == major && descriptor.minor == minor) return err(EEXIST);
         }
 
-        kdbgln("DeviceRegistry: registered new device type %u:%u at path /dev/%s", major, minor, name);
+        const char* name = device->device_path().chars();
+
+#ifdef DEVICE_REGISTRY_DEBUG
+        kdbgln("DeviceRegistry: registered new device type %u:%u at path /%s in devfs", major, minor, name);
+#endif
 
         auto desc = DeviceDescriptor { .device = device, .major = major, .minor = minor, .name = name, .mode = mode };
 
         TRY(g_available_devices.try_append(desc));
 
-        TRY(create_special_device_inode(desc));
-
         return {};
     }
 
     Result<void> init()
     {
-        auto device_fs = TRY(TmpFS::FileSystem::create());
-        TRY(VFS::mount("/dev", device_fs, Credentials {}));
-        g_device_fs = device_fs;
-
         NullDevice::create();
         ZeroDevice::create();
         FullDevice::create();
@@ -82,4 +78,15 @@ namespace DeviceRegistry
         static u32 next_minor = 0;
         return luna_dev_makedev(0, next_minor++);
     }
+
+    // FIXME: This devfs will only contain the devices registered before its creation. Otherwise, created devfs
+    // instances would have to be stored using WeakPtr or something (which doesn't exist atm)...
+    Result<SharedPtr<VFS::FileSystem>> create_devfs_instance()
+    {
+        auto fs = TRY(TmpFS::FileSystem::create());
+
+        for (const auto& descriptor : g_available_devices) TRY(create_special_device_inode(fs, descriptor));
+
+        return fs;
+    }
 }

kernel/src/fs/devices/DeviceRegistry.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "fs/VFS.h"
 #include "fs/devices/Device.h"
 #include <luna/SharedPtr.h>
 #include <sys/types.h>
@@ -12,15 +13,18 @@ namespace DeviceRegistry
         Console = 1,
         Memory = 2,
         Framebuffer = 3,
+        Disk = 4,
+        DiskPartition = 5,
     };
 
     Result<SharedPtr<Device>> fetch_special_device(u32 major, u32 minor);
 
-    Result<void> register_special_device(u32 major, u32 minor, SharedPtr<Device> device, const char* name,
-                                         mode_t mode = 0666);
+    Result<void> register_special_device(u32 major, u32 minor, SharedPtr<Device> device, mode_t mode = 0666);
 
     Result<void> init();
 
     // Used for file systems (like tmpfs) that do not have a host device.
     dev_t next_null_device_id();
+
+    Result<SharedPtr<VFS::FileSystem>> create_devfs_instance();
 }

kernel/src/fs/devices/FramebufferDevice.cpp

@@ -6,7 +6,7 @@
 Result<void> FramebufferDevice::create()
 {
     auto device = (SharedPtr<Device>)TRY(make_shared<FramebufferDevice>());
-    return DeviceRegistry::register_special_device(DeviceRegistry::Framebuffer, 0, device, "fb0", 0600);
+    return DeviceRegistry::register_special_device(DeviceRegistry::Framebuffer, 0, device, 0600);
 }
 
 Result<usize> FramebufferDevice::read(u8*, usize, usize) const

kernel/src/fs/devices/FramebufferDevice.h

@@ -22,5 +22,10 @@ class FramebufferDevice : public Device
 
     usize size() const override;
 
+    StringView device_path() const override
+    {
+        return "fb0";
+    }
+
     virtual ~FramebufferDevice() = default;
 };

kernel/src/fs/devices/FullDevice.cpp

@@ -3,5 +3,5 @@
 Result<void> FullDevice::create()
 {
     auto device = (SharedPtr<Device>)TRY(make_shared<FullDevice>());
-    return DeviceRegistry::register_special_device(DeviceRegistry::Memory, 2, device, "full");
+    return DeviceRegistry::register_special_device(DeviceRegistry::Memory, 2, device);
 }

kernel/src/fs/devices/FullDevice.h

@@ -24,5 +24,10 @@ class FullDevice : public Device
         return false;
     }
 
+    StringView device_path() const override
+    {
+        return "full";
+    }
+
     virtual ~FullDevice() = default;
 };

kernel/src/fs/devices/NullDevice.cpp

@@ -3,5 +3,5 @@
 Result<void> NullDevice::create()
 {
     auto device = (SharedPtr<Device>)TRY(make_shared<NullDevice>());
-    return DeviceRegistry::register_special_device(DeviceRegistry::Memory, 0, device, "null");
+    return DeviceRegistry::register_special_device(DeviceRegistry::Memory, 0, device);
 }

kernel/src/fs/devices/NullDevice.h

@@ -22,5 +22,10 @@ class NullDevice : public Device
         return false;
     }
 
+    StringView device_path() const override
+    {
+        return "null";
+    }
+
     virtual ~NullDevice() = default;
 };

kernel/src/fs/devices/ZeroDevice.cpp

@@ -3,5 +3,5 @@
 Result<void> ZeroDevice::create()
 {
     auto device = (SharedPtr<Device>)TRY(make_shared<ZeroDevice>());
-    return DeviceRegistry::register_special_device(DeviceRegistry::Memory, 1, device, "zero");
+    return DeviceRegistry::register_special_device(DeviceRegistry::Memory, 1, device);
 }

kernel/src/fs/devices/ZeroDevice.h

@@ -24,5 +24,10 @@ class ZeroDevice : public Device
         return false;
     }
 
+    StringView device_path() const override
+    {
+        return "zero";
+    }
+
     virtual ~ZeroDevice() = default;
 };

kernel/src/fs/tmpfs/FileSystem.cpp

@@ -1,6 +1,6 @@
 #include "fs/tmpfs/FileSystem.h"
-#include "fs/Mount.h"
 #include "fs/devices/DeviceRegistry.h"
+#include "fs/tmpfs/Inode.h"
 #include <luna/Alloc.h>
 #include <luna/CString.h>
 #include <luna/Ignore.h>
@@ -76,131 +76,4 @@ namespace TmpFS
     {
         m_root_inode = root;
     }
-
-    Result<SharedPtr<VFS::Inode>> DirInode::find(const char* name) const
-    {
-        for (const auto& entry : m_entries)
-        {
-            if (!strcmp(name, entry.name.chars())) return entry.inode;
-        }
-
-        return err(ENOENT);
-    }
-
-    Result<void> DirInode::replace_entry(SharedPtr<VFS::Inode> inode, const char* name)
-    {
-        for (auto& entry : m_entries)
-        {
-            if (!strcmp(name, entry.name.chars()))
-            {
-                entry.inode = inode;
-                return {};
-            }
-        }
-
-        return err(ENOENT);
-    }
-
-    Option<VFS::DirectoryEntry> DirInode::get(usize index) const
-    {
-        if (index >= m_entries.size()) return {};
-
-        return m_entries[index];
-    }
-
-    Result<void> DirInode::add_entry(SharedPtr<VFS::Inode> inode, const char* name)
-    {
-        if (find(name).has_value()) return err(EEXIST);
-
-        VFS::DirectoryEntry entry { inode, name };
-
-        TRY(m_entries.try_append(move(entry)));
-
-        inode->did_link();
-
-        return {};
-    }
-
-    Result<void> DirInode::remove_entry(const char* name)
-    {
-        SharedPtr<VFS::Inode> inode = TRY(find(name));
-
-        if (inode->type() == VFS::InodeType::Directory && inode->entries() != 2) return err(ENOTEMPTY);
-
-        if (inode->is_mountpoint()) return err(EBUSY);
-
-        m_entries.remove_first_matching(
-            [&](const VFS::DirectoryEntry& entry) { return !strcmp(entry.name.chars(), name); });
-
-        inode->did_unlink();
-
-        return {};
-    }
-
-    Result<SharedPtr<VFS::Inode>> DirInode::create_file(const char* name)
-    {
-        auto inode = TRY(m_fs->create_file_inode());
-
-        TRY(add_entry(inode, name));
-
-        return inode;
-    }
-
-    Result<SharedPtr<VFS::Inode>> DirInode::create_subdirectory(const char* name)
-    {
-        auto inode = TRY(m_fs->create_dir_inode(m_self));
-
-        TRY(add_entry(inode, name));
-
-        return inode;
-    }
-
-    Result<usize> FileInode::read(u8* buf, usize offset, usize length) const
-    {
-        if (length == 0) return 0;
-
-        if (offset > m_data_buffer.size()) return 0;
-        if (offset + length > m_data_buffer.size()) length = m_data_buffer.size() - offset;
-
-        memcpy(buf, m_data_buffer.data() + offset, length);
-
-        return length;
-    }
-
-    Result<usize> FileInode::write(const u8* buf, usize offset, usize length)
-    {
-        if (length == 0) return 0;
-
-        if (offset > m_data_buffer.size())
-        {
-            // Fill the in-between space with zeroes.
-            usize old_size = m_data_buffer.size();
-            usize zeroes = offset - old_size;
-
-            TRY(m_data_buffer.try_resize(offset));
-
-            memset(m_data_buffer.data() + old_size, 0, zeroes);
-        }
-
-        u8* slice = TRY(m_data_buffer.slice(offset, length));
-        memcpy(slice, buf, length);
-
-        return length;
-    }
-
-    Result<void> FileInode::truncate(usize size)
-    {
-        usize old_size = m_data_buffer.size();
-
-        TRY(m_data_buffer.try_resize(size));
-
-        if (size > old_size) memset(m_data_buffer.data() + old_size, 0, size - old_size);
-
-        return {};
-    }
-
-    usize FileInode::size() const
-    {
-        return m_data_buffer.size();
-    }
 }

kernel/src/fs/tmpfs/FileSystem.h

@@ -2,11 +2,6 @@
 #include "fs/VFS.h"
 #include "fs/devices/DeviceRegistry.h"
 #include <luna/Atomic.h>
-#include <luna/Badge.h>
-#include <luna/Buffer.h>
-#include <luna/StaticString.h>
-#include <luna/String.h>
-#include <luna/Vector.h>
 
 namespace TmpFS
 {
@@ -45,466 +40,4 @@ namespace TmpFS
 
         dev_t m_host_device_id;
     };
-
-    class FileInode : public VFS::FileInode
-    {
-      public:
-        FileInode() = default;
-
-        void set_fs(FileSystem& fs, Badge<FileSystem>)
-        {
-            m_fs = &fs;
-        }
-
-        void set_inode_number(usize inum, Badge<FileSystem>)
-        {
-            m_inode_number = inum;
-        }
-
-        VFS::FileSystem* fs() const override
-        {
-            return m_fs;
-        }
-
-        usize inode_number() const override
-        {
-            return m_inode_number;
-        }
-
-        Result<usize> read(u8*, usize, usize) const override;
-
-        Result<usize> write(const u8*, usize, usize) override;
-
-        Result<void> truncate(usize size) override;
-
-        usize size() const override;
-
-        mode_t mode() const override
-        {
-            return m_mode;
-        }
-
-        u32 uid() const override
-        {
-            return m_uid;
-        }
-
-        u32 gid() const override
-        {
-            return m_gid;
-        }
-
-        nlink_t nlinks() const override
-        {
-            return (nlink_t)m_nlinks;
-        }
-
-        Result<void> chmod(mode_t mode) override
-        {
-            m_mode = mode;
-            return {};
-        }
-
-        Result<void> chown(u32 uid, u32 gid) override
-        {
-            m_uid = uid;
-            m_gid = gid;
-            return {};
-        }
-
-        void did_link() override
-        {
-            m_nlinks++;
-        }
-
-        void did_unlink() override
-        {
-            m_nlinks--;
-        }
-
-        virtual ~FileInode() = default;
-
-      private:
-        VFS::FileSystem* m_fs;
-        Buffer m_data_buffer;
-        usize m_inode_number;
-        mode_t m_mode;
-        u32 m_uid { 0 };
-        u32 m_gid { 0 };
-        u32 m_nlinks { 0 };
-    };
-
-    class SymlinkInode : public VFS::FileInode
-    {
-      public:
-        SymlinkInode() = default;
-
-        VFS::InodeType type() const override
-        {
-            return VFS::InodeType::Symlink;
-        }
-
-        void set_fs(FileSystem& fs, Badge<FileSystem>)
-        {
-            m_fs = &fs;
-        }
-
-        void set_inode_number(usize inum, Badge<FileSystem>)
-        {
-            m_inode_number = inum;
-        }
-
-        Result<void> set_link(StringView link, Badge<FileSystem>)
-        {
-            m_link = TRY(String::from_string_view(link));
-            return {};
-        }
-
-        VFS::FileSystem* fs() const override
-        {
-            return m_fs;
-        }
-
-        usize inode_number() const override
-        {
-            return m_inode_number;
-        }
-
-        Result<usize> read(u8*, usize, usize) const override
-        {
-            return err(ENOTSUP);
-        }
-
-        Result<usize> write(const u8*, usize, usize) override
-        {
-            return err(ENOTSUP);
-        }
-
-        Result<void> truncate(usize) override
-        {
-            return err(ENOTSUP);
-        }
-
-        usize size() const override
-        {
-            return m_link.length();
-        }
-
-        mode_t mode() const override
-        {
-            return 0777;
-        }
-
-        u32 uid() const override
-        {
-            return m_uid;
-        }
-
-        u32 gid() const override
-        {
-            return m_gid;
-        }
-
-        nlink_t nlinks() const override
-        {
-            return (nlink_t)m_nlinks;
-        }
-
-        Result<void> chmod(mode_t) override
-        {
-            return {};
-        }
-
-        Result<void> chown(u32 uid, u32 gid) override
-        {
-            m_uid = uid;
-            m_gid = gid;
-            return {};
-        }
-
-        void did_link() override
-        {
-            m_nlinks++;
-        }
-
-        void did_unlink() override
-        {
-            m_nlinks--;
-        }
-
-        Result<StringView> readlink() override
-        {
-            return m_link.view();
-        }
-
-        virtual ~SymlinkInode() = default;
-
-      private:
-        VFS::FileSystem* m_fs;
-        String m_link;
-        usize m_inode_number;
-        u32 m_uid { 0 };
-        u32 m_gid { 0 };
-        u32 m_nlinks { 0 };
-    };
-
-    class DeviceInode : public VFS::DeviceInode
-    {
-      public:
-        DeviceInode() = default;
-
-        VFS::InodeType type() const override
-        {
-            return m_device->is_block_device() ? VFS::InodeType::BlockDevice : VFS::InodeType::CharacterDevice;
-        }
-
-        void set_fs(FileSystem& fs, Badge<FileSystem>)
-        {
-            m_fs = &fs;
-        }
-
-        void set_inode_number(usize inum, Badge<FileSystem>)
-        {
-            m_inode_number = inum;
-        }
-
-        void set_device(SharedPtr<Device> device, Badge<FileSystem>)
-        {
-            m_device = device;
-        }
-
-        void set_device_id(dev_t id, Badge<FileSystem>)
-        {
-            m_device_id = id;
-        }
-
-        VFS::FileSystem* fs() const override
-        {
-            return m_fs;
-        }
-
-        dev_t device_id() const override
-        {
-            return m_device_id;
-        }
-
-        usize inode_number() const override
-        {
-            return m_inode_number;
-        }
-
-        Result<usize> read(u8* buf, usize offset, usize length) const override
-        {
-            return m_device->read(buf, offset, length);
-        }
-
-        Result<usize> write(const u8* buf, usize offset, usize length) override
-        {
-            return m_device->write(buf, offset, length);
-        }
-
-        Result<void> truncate(usize) override
-        {
-            // POSIX says truncate is for regular files, but doesn't tell us what error to return for non-regular files.
-            return err(EINVAL);
-        }
-
-        Result<u64> ioctl(int request, void* arg) override
-        {
-            return m_device->ioctl(request, arg);
-        }
-
-        bool blocking() const override
-        {
-            return m_device->blocking();
-        }
-
-        usize size() const override
-        {
-            return m_device->size();
-        }
-
-        mode_t mode() const override
-        {
-            return m_mode;
-        }
-
-        u32 uid() const override
-        {
-            return m_uid;
-        }
-
-        u32 gid() const override
-        {
-            return m_gid;
-        }
-
-        nlink_t nlinks() const override
-        {
-            return (nlink_t)m_nlinks;
-        }
-
-        Result<void> chmod(mode_t mode) override
-        {
-            m_mode = mode;
-            return {};
-        }
-
-        Result<void> chown(u32 uid, u32 gid) override
-        {
-            m_uid = uid;
-            m_gid = gid;
-            return {};
-        }
-
-        void did_link() override
-        {
-            m_nlinks++;
-        }
-
-        void did_unlink() override
-        {
-            m_nlinks--;
-        }
-
-        virtual ~DeviceInode() = default;
-
-      private:
-        VFS::FileSystem* m_fs;
-        SharedPtr<Device> m_device;
-        usize m_inode_number;
-        mode_t m_mode;
-        u32 m_uid { 0 };
-        u32 m_gid { 0 };
-        u32 m_nlinks { 0 };
-        dev_t m_device_id { 0 };
-    };
-
-    class DirInode : public VFS::Inode
-    {
-      public:
-        DirInode() = default;
-
-        void set_fs(FileSystem& fs, Badge<FileSystem>)
-        {
-            m_fs = &fs;
-        }
-
-        void set_inode_number(usize inum, Badge<FileSystem>)
-        {
-            m_inode_number = inum;
-        }
-
-        void set_self(SharedPtr<VFS::Inode> self, Badge<FileSystem>)
-        {
-            m_self = self;
-        }
-
-        Result<SharedPtr<VFS::Inode>> find(const char* name) const override;
-        Option<VFS::DirectoryEntry> get(usize index) const override;
-
-        Result<usize> read(u8*, usize, usize) const override
-        {
-            return err(EISDIR);
-        }
-
-        Result<usize> write(const u8*, usize, usize) override
-        {
-            return err(EISDIR);
-        }
-
-        Result<void> truncate(usize) override
-        {
-            return err(EISDIR);
-        }
-
-        bool blocking() const override
-        {
-            return false;
-        }
-
-        usize size() const override
-        {
-            return 0;
-        }
-
-        mode_t mode() const override
-        {
-            return m_mode;
-        }
-
-        u32 uid() const override
-        {
-            return m_uid;
-        }
-
-        u32 gid() const override
-        {
-            return m_gid;
-        }
-
-        Result<void> chmod(mode_t mode) override
-        {
-            m_mode = mode;
-            return {};
-        }
-
-        Result<void> chown(u32 uid, u32 gid) override
-        {
-            m_uid = uid;
-            m_gid = gid;
-            return {};
-        }
-
-        VFS::FileSystem* fs() const override
-        {
-            return m_fs;
-        }
-
-        usize inode_number() const override
-        {
-            return m_inode_number;
-        }
-
-        VFS::InodeType type() const override
-        {
-            return VFS::InodeType::Directory;
-        }
-
-        void did_link() override
-        {
-        }
-
-        void did_unlink() override
-        {
-            m_self = {};
-            m_entries.clear();
-        }
-
-        usize entries() const override
-        {
-            return m_entries.size();
-        }
-
-        Result<void> remove_entry(const char* name) override;
-
-        Result<SharedPtr<VFS::Inode>> create_file(const char* name) override;
-        Result<SharedPtr<VFS::Inode>> create_subdirectory(const char* name) override;
-
-        Result<void> add_entry(SharedPtr<VFS::Inode> inode, const char* name);
-        Result<void> replace_entry(SharedPtr<VFS::Inode> inode, const char* name);
-
-        virtual ~DirInode() = default;
-
-      private:
-        VFS::FileSystem* m_fs;
-        usize m_inode_number;
-        mode_t m_mode;
-        u32 m_uid { 0 };
-        u32 m_gid { 0 };
-
-        SharedPtr<VFS::Inode> m_self;
-
-        Vector<VFS::DirectoryEntry> m_entries;
-    };
 }

kernel/src/fs/tmpfs/Inode.cpp

@@ -0,0 +1,131 @@
+#include "fs/tmpfs/Inode.h"
+
+namespace TmpFS
+{
+    Result<SharedPtr<VFS::Inode>> DirInode::find(const char* name) const
+    {
+        for (const auto& entry : m_entries)
+        {
+            if (!strcmp(name, entry.name.chars())) return entry.inode;
+        }
+
+        return err(ENOENT);
+    }
+
+    Result<void> DirInode::replace_entry(SharedPtr<VFS::Inode> inode, const char* name)
+    {
+        for (auto& entry : m_entries)
+        {
+            if (!strcmp(name, entry.name.chars()))
+            {
+                entry.inode = inode;
+                return {};
+            }
+        }
+
+        return err(ENOENT);
+    }
+
+    Option<VFS::DirectoryEntry> DirInode::get(usize index) const
+    {
+        if (index >= m_entries.size()) return {};
+
+        return m_entries[index];
+    }
+
+    Result<void> DirInode::add_entry(SharedPtr<VFS::Inode> inode, const char* name)
+    {
+        if (find(name).has_value()) return err(EEXIST);
+
+        VFS::DirectoryEntry entry { inode, name };
+
+        TRY(m_entries.try_append(move(entry)));
+
+        inode->did_link();
+
+        return {};
+    }
+
+    Result<void> DirInode::remove_entry(const char* name)
+    {
+        SharedPtr<VFS::Inode> inode = TRY(find(name));
+
+        if (inode->type() == VFS::InodeType::Directory && inode->entries() != 2) return err(ENOTEMPTY);
+
+        if (inode->is_mountpoint()) return err(EBUSY);
+
+        m_entries.remove_first_matching(
+            [&](const VFS::DirectoryEntry& entry) { return !strcmp(entry.name.chars(), name); });
+
+        inode->did_unlink();
+
+        return {};
+    }
+
+    Result<SharedPtr<VFS::Inode>> DirInode::create_file(const char* name)
+    {
+        auto inode = TRY(m_fs->create_file_inode());
+
+        TRY(add_entry(inode, name));
+
+        return inode;
+    }
+
+    Result<SharedPtr<VFS::Inode>> DirInode::create_subdirectory(const char* name)
+    {
+        auto inode = TRY(m_fs->create_dir_inode(m_self));
+
+        TRY(add_entry(inode, name));
+
+        return inode;
+    }
+
+    Result<usize> FileInode::read(u8* buf, usize offset, usize length) const
+    {
+        if (length == 0) return 0;
+
+        if (offset > m_data_buffer.size()) return 0;
+        if (offset + length > m_data_buffer.size()) length = m_data_buffer.size() - offset;
+
+        memcpy(buf, m_data_buffer.data() + offset, length);
+
+        return length;
+    }
+
+    Result<usize> FileInode::write(const u8* buf, usize offset, usize length)
+    {
+        if (length == 0) return 0;
+
+        if (offset > m_data_buffer.size())
+        {
+            // Fill the in-between space with zeroes.
+            usize old_size = m_data_buffer.size();
+            usize zeroes = offset - old_size;
+
+            TRY(m_data_buffer.try_resize(offset));
+
+            memset(m_data_buffer.data() + old_size, 0, zeroes);
+        }
+
+        u8* slice = TRY(m_data_buffer.slice(offset, length));
+        memcpy(slice, buf, length);
+
+        return length;
+    }
+
+    Result<void> FileInode::truncate(usize size)
+    {
+        usize old_size = m_data_buffer.size();
+
+        TRY(m_data_buffer.try_resize(size));
+
+        if (size > old_size) memset(m_data_buffer.data() + old_size, 0, size - old_size);
+
+        return {};
+    }
+
+    usize FileInode::size() const
+    {
+        return m_data_buffer.size();
+    }
+}

kernel/src/fs/tmpfs/Inode.h

@@ -0,0 +1,472 @@
+#pragma once
+#include "fs/tmpfs/FileSystem.h"
+#include <luna/Badge.h>
+#include <luna/Buffer.h>
+#include <luna/StaticString.h>
+#include <luna/String.h>
+#include <luna/Vector.h>
+
+namespace TmpFS
+{
+    class FileInode : public VFS::FileInode
+    {
+      public:
+        FileInode() = default;
+
+        void set_fs(FileSystem& fs, Badge<FileSystem>)
+        {
+            m_fs = &fs;
+        }
+
+        void set_inode_number(usize inum, Badge<FileSystem>)
+        {
+            m_inode_number = inum;
+        }
+
+        VFS::FileSystem* fs() const override
+        {
+            return m_fs;
+        }
+
+        usize inode_number() const override
+        {
+            return m_inode_number;
+        }
+
+        Result<usize> read(u8*, usize, usize) const override;
+
+        Result<usize> write(const u8*, usize, usize) override;
+
+        Result<void> truncate(usize size) override;
+
+        usize size() const override;
+
+        mode_t mode() const override
+        {
+            return m_mode;
+        }
+
+        u32 uid() const override
+        {
+            return m_uid;
+        }
+
+        u32 gid() const override
+        {
+            return m_gid;
+        }
+
+        nlink_t nlinks() const override
+        {
+            return (nlink_t)m_nlinks;
+        }
+
+        Result<void> chmod(mode_t mode) override
+        {
+            m_mode = mode;
+            return {};
+        }
+
+        Result<void> chown(u32 uid, u32 gid) override
+        {
+            m_uid = uid;
+            m_gid = gid;
+            return {};
+        }
+
+        void did_link() override
+        {
+            m_nlinks++;
+        }
+
+        void did_unlink() override
+        {
+            m_nlinks--;
+        }
+
+        virtual ~FileInode() = default;
+
+      private:
+        VFS::FileSystem* m_fs;
+        Buffer m_data_buffer;
+        usize m_inode_number;
+        mode_t m_mode;
+        u32 m_uid { 0 };
+        u32 m_gid { 0 };
+        u32 m_nlinks { 0 };
+    };
+
+    class SymlinkInode : public VFS::FileInode
+    {
+      public:
+        SymlinkInode() = default;
+
+        VFS::InodeType type() const override
+        {
+            return VFS::InodeType::Symlink;
+        }
+
+        void set_fs(FileSystem& fs, Badge<FileSystem>)
+        {
+            m_fs = &fs;
+        }
+
+        void set_inode_number(usize inum, Badge<FileSystem>)
+        {
+            m_inode_number = inum;
+        }
+
+        Result<void> set_link(StringView link, Badge<FileSystem>)
+        {
+            m_link = TRY(String::from_string_view(link));
+            return {};
+        }
+
+        VFS::FileSystem* fs() const override
+        {
+            return m_fs;
+        }
+
+        usize inode_number() const override
+        {
+            return m_inode_number;
+        }
+
+        Result<usize> read(u8*, usize, usize) const override
+        {
+            return err(ENOTSUP);
+        }
+
+        Result<usize> write(const u8*, usize, usize) override
+        {
+            return err(ENOTSUP);
+        }
+
+        Result<void> truncate(usize) override
+        {
+            return err(ENOTSUP);
+        }
+
+        usize size() const override
+        {
+            return m_link.length();
+        }
+
+        mode_t mode() const override
+        {
+            return 0777;
+        }
+
+        u32 uid() const override
+        {
+            return m_uid;
+        }
+
+        u32 gid() const override
+        {
+            return m_gid;
+        }
+
+        nlink_t nlinks() const override
+        {
+            return (nlink_t)m_nlinks;
+        }
+
+        Result<void> chmod(mode_t) override
+        {
+            return {};
+        }
+
+        Result<void> chown(u32 uid, u32 gid) override
+        {
+            m_uid = uid;
+            m_gid = gid;
+            return {};
+        }
+
+        void did_link() override
+        {
+            m_nlinks++;
+        }
+
+        void did_unlink() override
+        {
+            m_nlinks--;
+        }
+
+        Result<StringView> readlink() override
+        {
+            return m_link.view();
+        }
+
+        virtual ~SymlinkInode() = default;
+
+      private:
+        VFS::FileSystem* m_fs;
+        String m_link;
+        usize m_inode_number;
+        u32 m_uid { 0 };
+        u32 m_gid { 0 };
+        u32 m_nlinks { 0 };
+    };
+
+    class DeviceInode : public VFS::DeviceInode
+    {
+      public:
+        DeviceInode() = default;
+
+        VFS::InodeType type() const override
+        {
+            return m_device->is_block_device() ? VFS::InodeType::BlockDevice : VFS::InodeType::CharacterDevice;
+        }
+
+        void set_fs(FileSystem& fs, Badge<FileSystem>)
+        {
+            m_fs = &fs;
+        }
+
+        void set_inode_number(usize inum, Badge<FileSystem>)
+        {
+            m_inode_number = inum;
+        }
+
+        void set_device(SharedPtr<Device> device, Badge<FileSystem>)
+        {
+            m_device = device;
+        }
+
+        void set_device_id(dev_t id, Badge<FileSystem>)
+        {
+            m_device_id = id;
+        }
+
+        VFS::FileSystem* fs() const override
+        {
+            return m_fs;
+        }
+
+        dev_t device_id() const override
+        {
+            return m_device_id;
+        }
+
+        usize inode_number() const override
+        {
+            return m_inode_number;
+        }
+
+        Result<usize> read(u8* buf, usize offset, usize length) const override
+        {
+            return m_device->read(buf, offset, length);
+        }
+
+        Result<usize> write(const u8* buf, usize offset, usize length) override
+        {
+            return m_device->write(buf, offset, length);
+        }
+
+        Result<void> truncate(usize) override
+        {
+            // POSIX says truncate is for regular files, but doesn't tell us what error to return for non-regular files.
+            return err(EINVAL);
+        }
+
+        Result<u64> ioctl(int request, void* arg) override
+        {
+            return m_device->ioctl(request, arg);
+        }
+
+        bool blocking() const override
+        {
+            return m_device->blocking();
+        }
+
+        usize size() const override
+        {
+            return m_device->size();
+        }
+
+        mode_t mode() const override
+        {
+            return m_mode;
+        }
+
+        u32 uid() const override
+        {
+            return m_uid;
+        }
+
+        u32 gid() const override
+        {
+            return m_gid;
+        }
+
+        nlink_t nlinks() const override
+        {
+            return (nlink_t)m_nlinks;
+        }
+
+        Result<void> chmod(mode_t mode) override
+        {
+            m_mode = mode;
+            return {};
+        }
+
+        Result<void> chown(u32 uid, u32 gid) override
+        {
+            m_uid = uid;
+            m_gid = gid;
+            return {};
+        }
+
+        void did_link() override
+        {
+            m_nlinks++;
+        }
+
+        void did_unlink() override
+        {
+            m_nlinks--;
+        }
+
+        virtual ~DeviceInode() = default;
+
+      private:
+        VFS::FileSystem* m_fs;
+        SharedPtr<Device> m_device;
+        usize m_inode_number;
+        mode_t m_mode;
+        u32 m_uid { 0 };
+        u32 m_gid { 0 };
+        u32 m_nlinks { 0 };
+        dev_t m_device_id { 0 };
+    };
+
+    class DirInode : public VFS::Inode
+    {
+      public:
+        DirInode() = default;
+
+        void set_fs(FileSystem& fs, Badge<FileSystem>)
+        {
+            m_fs = &fs;
+        }
+
+        void set_inode_number(usize inum, Badge<FileSystem>)
+        {
+            m_inode_number = inum;
+        }
+
+        void set_self(SharedPtr<VFS::Inode> self, Badge<FileSystem>)
+        {
+            m_self = self;
+        }
+
+        Result<SharedPtr<VFS::Inode>> find(const char* name) const override;
+        Option<VFS::DirectoryEntry> get(usize index) const override;
+
+        Result<usize> read(u8*, usize, usize) const override
+        {
+            return err(EISDIR);
+        }
+
+        Result<usize> write(const u8*, usize, usize) override
+        {
+            return err(EISDIR);
+        }
+
+        Result<void> truncate(usize) override
+        {
+            return err(EISDIR);
+        }
+
+        bool blocking() const override
+        {
+            return false;
+        }
+
+        usize size() const override
+        {
+            return 0;
+        }
+
+        mode_t mode() const override
+        {
+            return m_mode;
+        }
+
+        u32 uid() const override
+        {
+            return m_uid;
+        }
+
+        u32 gid() const override
+        {
+            return m_gid;
+        }
+
+        Result<void> chmod(mode_t mode) override
+        {
+            m_mode = mode;
+            return {};
+        }
+
+        Result<void> chown(u32 uid, u32 gid) override
+        {
+            m_uid = uid;
+            m_gid = gid;
+            return {};
+        }
+
+        VFS::FileSystem* fs() const override
+        {
+            return m_fs;
+        }
+
+        usize inode_number() const override
+        {
+            return m_inode_number;
+        }
+
+        VFS::InodeType type() const override
+        {
+            return VFS::InodeType::Directory;
+        }
+
+        void did_link() override
+        {
+        }
+
+        void did_unlink() override
+        {
+            m_self = {};
+            m_entries.clear();
+        }
+
+        usize entries() const override
+        {
+            return m_entries.size();
+        }
+
+        Result<void> remove_entry(const char* name) override;
+
+        Result<SharedPtr<VFS::Inode>> create_file(const char* name) override;
+        Result<SharedPtr<VFS::Inode>> create_subdirectory(const char* name) override;
+
+        Result<void> add_entry(SharedPtr<VFS::Inode> inode, const char* name);
+        Result<void> replace_entry(SharedPtr<VFS::Inode> inode, const char* name);
+
+        virtual ~DirInode() = default;
+
+      private:
+        VFS::FileSystem* m_fs;
+        usize m_inode_number;
+        mode_t m_mode;
+        u32 m_uid { 0 };
+        u32 m_gid { 0 };
+
+        SharedPtr<VFS::Inode> m_self;
+
+        Vector<VFS::DirectoryEntry> m_entries;
+    };
+}

kernel/src/lib/KMutex.h

@@ -0,0 +1,77 @@
+#pragma once
+#include "Log.h"
+#include "arch/CPU.h"
+#include "thread/Scheduler.h"
+#include "thread/Thread.h"
+#include <luna/CircularQueue.h>
+
+template <usize ConcurrentThreads> class KMutex
+{
+  public:
+    void lock()
+    {
+        int expected = 0;
+        while (!m_lock.compare_exchange_strong(expected, 1))
+        {
+            expected = 0;
+            auto* current = Scheduler::current();
+
+            // We cannot be interrupted between these functions, otherwise we might never exit the loop
+            CPU::disable_interrupts();
+            bool ok = m_blocked_threads.try_push(current);
+            if (!ok) kernel_sleep(10);
+            else
+                kernel_wait_for_event();
+            CPU::enable_interrupts();
+        }
+    };
+
+    void unlock()
+    {
+        int expected = 1;
+        if (!m_lock.compare_exchange_strong(expected, 0))
+        {
+            kwarnln("KMutex::unlock() called on an unlocked lock with value %d", expected);
+        }
+
+        Thread* blocked;
+        if (m_blocked_threads.try_pop(blocked)) blocked->wake_up();
+    }
+
+    bool try_lock()
+    {
+        int expected = 0;
+        return m_lock.compare_exchange_strong(expected, 1);
+    }
+
+  private:
+    CircularQueue<Thread*, ConcurrentThreads> m_blocked_threads;
+    Atomic<int> m_lock;
+};
+
+template <usize ConcurrentThreads> class ScopedKMutexLock
+{
+  public:
+    ScopedKMutexLock(KMutex<ConcurrentThreads>& lock) : m_lock(lock)
+    {
+        m_lock.lock();
+    }
+
+    ~ScopedKMutexLock()
+    {
+        if (!m_taken_over) m_lock.unlock();
+    }
+
+    ScopedKMutexLock(const ScopedKMutexLock&) = delete;
+    ScopedKMutexLock(ScopedKMutexLock&&) = delete;
+
+    KMutex<ConcurrentThreads>& take_over()
+    {
+        m_taken_over = true;
+        return m_lock;
+    }
+
+  private:
+    KMutex<ConcurrentThreads>& m_lock;
+    bool m_taken_over { false };
+};

kernel/src/main.cpp

@@ -1,6 +1,5 @@
 #include "Log.h"
 #include "arch/CPU.h"
-#include "arch/PCI.h"
 #include "arch/Timer.h"
 #include "boot/Init.h"
 #include "config.h"
@@ -11,6 +10,10 @@
 #include "thread/Scheduler.h"
 #include <luna/Units.h>
 
+#ifdef ARCH_X86_64
+#include "arch/x86_64/disk/ATA.h"
+#endif
+
 extern void set_host_name(StringView);
 
 void reap_thread()
@@ -27,7 +30,7 @@ void reap_thread()
     }
 }
 
-Result<void> init()
+[[noreturn]] void init()
 {
     kinfoln("Starting Moon %s, built on %s at %s", MOON_VERSION, __DATE__, __TIME__);
 
@@ -42,23 +45,22 @@ Result<void> init()
     kinfoln("Used memory:     %s", to_dynamic_unit(MemoryManager::used()).release_value().chars());
     kinfoln("Reserved memory: %s", to_dynamic_unit(MemoryManager::reserved()).release_value().chars());
 
-    auto root = TRY(TmpFS::FileSystem::create());
-    TRY(VFS::mount_root(root));
-    TRY(InitRD::populate_vfs());
-    TRY(DeviceRegistry::init());
+    auto root = mark_critical(TmpFS::FileSystem::create(), "Failed to create initial ramfs");
+    mark_critical(VFS::mount_root(root), "Failed to mount the initial ramfs as the root filesystem");
+    mark_critical(InitRD::populate_vfs(), "Failed to load files from the initial ramdisk");
+    mark_critical(DeviceRegistry::init(), "Failed to register initial devices");
 
-    auto init = TRY(VFS::resolve_path("/bin/init", Credentials {}));
-    auto init_thread = TRY(Scheduler::new_userspace_thread(init, "/bin/init"));
+    auto init = mark_critical(VFS::resolve_path("/bin/init", Credentials {}), "Can't find init in the initial ramfs!");
+    auto init_thread =
+        mark_critical(Scheduler::new_userspace_thread(init, "/bin/init"), "Failed to create PID 1 process for init");
 
-    auto reap = Scheduler::new_kernel_thread(reap_thread, "[reap]").release_value();
+    auto reap = mark_critical(Scheduler::new_kernel_thread(reap_thread, "[reap]"),
+                              "Failed to create the process reaper kernel thread");
     Scheduler::set_reap_thread(reap);
 
-    PCI::scan(
-        [](const PCI::Device& device) {
-            kinfoln("Found PCI mass storage device %.4x:%.4x, at address %u:%u:%u", device.id.vendor, device.id.device,
-                    device.address.bus, device.address.slot, device.address.function);
-        },
-        { .klass = 1 });
+#ifdef ARCH_X86_64
+    ATA::Controller::scan();
+#endif
 
     // Disable console logging before transferring control to userspace.
     setup_log(log_debug_enabled(), log_serial_enabled(), false);
@@ -68,13 +70,6 @@ Result<void> init()
     kernel_exit();
 }
 
-[[noreturn]] void init_wrapper()
-{
-    auto rc = init();
-    if (rc.has_error()) kerrorln("Runtime error: %s", rc.error_string());
-    kernel_exit();
-}
-
 extern "C" [[noreturn]] void _start()
 {
     Init::check_magic();
@@ -85,7 +80,7 @@ extern "C" [[noreturn]] void _start()
     Thread::init();
     Scheduler::init();
 
-    Scheduler::new_kernel_thread(init_wrapper, "[kinit]");
+    Scheduler::new_kernel_thread(init, "[kinit]");
 
     CPU::platform_finish_init();
 

kernel/src/memory/MemoryManager.cpp

@@ -267,8 +267,9 @@ namespace MemoryManager
 
         memset((void*)address, 0, count * ARCH_PAGE_SIZE);
 
-        remap(address, count, flags)
-            .expect_value("Wait... we just mapped something but it doesn't exist anymore? Confused.");
+        // This should never fail (we just mapped memory at that address) but we don't want to crash the kernel if it
+        // does.
+        TRY(remap(address, count, flags));
 
         return address;
     }

kernel/src/memory/UserVM.cpp

@@ -5,118 +5,197 @@
 #include <luna/CString.h>
 #include <luna/ScopeGuard.h>
 
-static constexpr u64 VM_BASE = 0x10000000;
-
-static constexpr usize INITIAL_VM_SIZE = 80;
-static constexpr usize MAX_VM_SIZE = 1024 * 1024 * 16;
+static constexpr u64 VM_START = ARCH_PAGE_SIZE;
+static constexpr u64 VM_END = 0x0000800000000000;
 
 Result<OwnedPtr<UserVM>> UserVM::try_create()
 {
-    void* const base = TRY(kmalloc(INITIAL_VM_SIZE));
+    OwnedPtr<UserVM> ptr = TRY(make_owned<UserVM>());
 
-    auto guard = make_scope_guard([&] { kfree(base); });
-
-    OwnedPtr<UserVM> ptr = TRY(make_owned<UserVM>(base, INITIAL_VM_SIZE));
-
-    guard.deactivate();
+    TRY(ptr->create_null_region());
+    TRY(ptr->create_default_region());
 
     return move(ptr);
 }
 
+Result<void> UserVM::create_null_region()
+{
+    // Create a small region at the start of the address space to prevent anyone from mapping page 0.
+    auto* region = TRY(make<VMRegion>());
+    region->start = 0;
+    region->end = VM_START;
+    region->count = 1;
+    region->used = true;
+    region->persistent = true;
+    m_regions.append(region);
+    return {};
+}
+
+Result<void> UserVM::create_default_region()
+{
+    // Create a free region covering the rest of the address space.
+    auto* region = TRY(make<VMRegion>());
+    region->start = VM_START;
+    region->end = VM_END;
+    region->count = (VM_END / ARCH_PAGE_SIZE) - 1;
+    region->used = false;
+    m_regions.append(region);
+    return {};
+}
+
 Result<OwnedPtr<UserVM>> UserVM::clone()
 {
-    void* const base = TRY(kmalloc(m_bitmap.size_in_bytes()));
+    OwnedPtr<UserVM> ptr = TRY(make_owned<UserVM>());
 
-    auto guard = make_scope_guard([&] { kfree(base); });
-
-    OwnedPtr<UserVM> ptr = TRY(make_owned<UserVM>(base, m_bitmap.size_in_bytes()));
-
-    memcpy(ptr->m_bitmap.location(), m_bitmap.location(), m_bitmap.size_in_bytes());
-
-    guard.deactivate();
+    for (const auto* region : m_regions)
+    {
+        auto* new_region = TRY(make<VMRegion>());
+        memcpy(new_region, region, sizeof(*region));
+        ptr->m_regions.append(new_region);
+    }
 
     return move(ptr);
 }
 
-UserVM::UserVM(void* base, usize size)
+UserVM::UserVM()
 {
-    m_bitmap.initialize(base, size);
-    m_bitmap.clear(false);
 }
 
-Result<bool> UserVM::try_expand(usize size)
+Result<u64> UserVM::alloc_region(usize count, bool persistent)
 {
-    if (m_bitmap.size_in_bytes() == MAX_VM_SIZE) { return false; }
-
-    const usize old_size = m_bitmap.size_in_bytes();
-    usize new_size = old_size + size;
-
-    if (new_size > MAX_VM_SIZE) new_size = MAX_VM_SIZE;
-
-    m_bitmap.resize(new_size);
-    m_bitmap.clear_region(old_size * 8, (new_size - old_size) * 8, false);
-
-    return true;
-}
-
-Result<u64> UserVM::alloc_one_page()
-{
-    u64 index;
-    bool ok = m_bitmap.find_and_toggle(false).try_set_value(index);
-    if (!ok)
+    for (auto* region = m_regions.expect_last(); region; region = m_regions.previous(region).value_or(nullptr))
     {
-        bool success = TRY(try_expand());
-        if (!success) return err(ENOMEM);
-        index = TRY(Result<u64>::from_option(m_bitmap.find_and_toggle(false), ENOMEM));
+        if (!region->used)
+        {
+            if (region->count < count) continue;
+            if (region->count == count)
+            {
+                region->used = true;
+                region->persistent = persistent;
+                u64 address = region->start;
+                try_merge_region_with_neighbors(region);
+                return address;
+            }
+
+            u64 boundary = region->end - (count * ARCH_PAGE_SIZE);
+
+            auto* new_region = TRY(split_region(region, boundary));
+            new_region->used = true;
+            new_region->persistent = persistent;
+            try_merge_region_with_neighbors(new_region);
+
+            return boundary;
+        }
     }
 
-    return VM_BASE + index * ARCH_PAGE_SIZE;
+    return err(ENOMEM);
 }
 
-Result<u64> UserVM::alloc_several_pages(usize count)
+Result<bool> UserVM::set_region(u64 address, usize count, bool used, bool persistent)
 {
-    u64 index;
-    bool ok = m_bitmap.find_and_toggle_region(false, count).try_set_value(index);
-    if (!ok)
+    if (address >= VM_END) return err(EINVAL);
+
+    u64 end = address + (count * ARCH_PAGE_SIZE);
+
+    for (auto* region : m_regions)
     {
-        bool success = TRY(try_expand((count / 8) + INITIAL_VM_SIZE));
-        if (!success) return err(ENOMEM);
-        index = TRY(Result<u64>::from_option(m_bitmap.find_and_toggle_region(false, count), ENOMEM));
+        if (region->end < address) continue;
+        if (region->start > end) return false;
+
+        if (region->persistent) return false;
+        if (region->used == used)
+        {
+            if (used) return false;
+            continue;
+        }
+
+        if (region->start >= address && region->end <= end)
+        {
+            region->used = used;
+            region->persistent = persistent;
+            if (region->start == address && region->end == end)
+            {
+                try_merge_region_with_neighbors(region);
+                return true;
+            }
+            continue;
+        }
+
+        if (region->end > end && region->start < address)
+        {
+            auto* middle_region = TRY(split_region(region, address));
+            TRY(split_region(middle_region, end));
+            middle_region->used = used;
+            middle_region->persistent = persistent;
+            return true;
+        }
+
+        if (region->start < address)
+        {
+            bool finished = region->end == end;
+            auto* split = TRY(split_region(region, address));
+            split->used = used;
+            split->persistent = persistent;
+            try_merge_region_with_neighbors(split);
+            if (!finished) continue;
+            return true;
+        }
+
+        if (region->end > end)
+        {
+            TRY(split_region(region, end));
+            region->used = used;
+            region->persistent = persistent;
+            try_merge_region_with_neighbors(region);
+            return true;
+        }
     }
 
-    return VM_BASE + index * ARCH_PAGE_SIZE;
-}
-
-Result<bool> UserVM::free_one_page(u64 address)
-{
-    if (address < VM_BASE) return err(EINVAL);
-    const u64 index = (address - VM_BASE) / ARCH_PAGE_SIZE;
-    if (index > (MAX_VM_SIZE * 8)) return err(EINVAL);
-
-    // NOTE: POSIX says munmap() should silently do nothing if the address is not mapped, instead of throwing an error
-    // like EFAULT.
-    if (!m_bitmap.get(index)) return false;
-
-    m_bitmap.set(index, false);
-
     return true;
 }
 
-Result<bool> UserVM::free_several_pages(u64 address, usize count)
+void UserVM::merge_contiguous_regions(VMRegion* a, VMRegion* b)
 {
-    if (address < VM_BASE) return err(EINVAL);
-    const u64 index = (address - VM_BASE) / ARCH_PAGE_SIZE;
-    if ((index + count) > (MAX_VM_SIZE * 8)) return err(EINVAL);
+    a->end = b->end;
+    a->count += b->count;
+    m_regions.remove(b);
+    delete b;
+}
 
-    // NOTE: Same as above.
-    if (!TRY(m_bitmap.try_match_region(index, count, true))) return false;
+void UserVM::try_merge_region_with_neighbors(VMRegion* region)
+{
+    auto prev = m_regions.previous(region);
+    if (prev.has_value() && (*prev)->used == region->used && (*prev)->persistent == region->persistent)
+    {
+        merge_contiguous_regions(*prev, region);
+        region = *prev;
+    }
 
-    m_bitmap.clear_region(index, count, false);
+    auto next = m_regions.next(region);
+    if (next.has_value() && (*next)->used == region->used && (*next)->persistent == region->persistent)
+    {
+        merge_contiguous_regions(region, *next);
+    }
+}
 
-    return true;
+Result<VMRegion*> UserVM::split_region(VMRegion* parent, u64 boundary)
+{
+    auto* region = TRY(make<VMRegion>());
+
+    region->start = boundary;
+    region->end = parent->end;
+    region->count = (region->end - region->start) / ARCH_PAGE_SIZE;
+    region->used = parent->used;
+    region->persistent = parent->persistent;
+    m_regions.add_after(parent, region);
+
+    parent->end = boundary;
+    parent->count -= region->count;
+
+    return region;
 }
 
 UserVM::~UserVM()
 {
-    m_bitmap.deallocate();
+    m_regions.consume([](VMRegion* region) { delete region; });
 }

kernel/src/memory/UserVM.h

@@ -1,25 +1,46 @@
 #pragma once
-#include <luna/Bitmap.h>
+#include <luna/LinkedList.h>
 #include <luna/OwnedPtr.h>
 #include <luna/Result.h>
 
+class VMRegion : LinkedListNode<VMRegion>
+{
+  public:
+    u64 start;
+    u64 end;
+    usize count;
+    bool used { true };
+    bool persistent { false };
+};
+
 class UserVM
 {
   public:
-    UserVM(void* base, usize size);
+    UserVM();
     ~UserVM();
 
-    Result<u64> alloc_one_page();
-    Result<u64> alloc_several_pages(usize count);
+    Result<u64> alloc_region(usize count, bool persistent = false);
 
-    Result<bool> free_one_page(u64 address);
-    Result<bool> free_several_pages(u64 address, usize count);
+    Result<bool> test_and_alloc_region(u64 address, usize count, bool persistent = false)
+    {
+        return set_region(address, count, true, persistent);
+    }
+
+    Result<bool> free_region(u64 address, usize count)
+    {
+        return set_region(address, count, false, false);
+    }
 
     static Result<OwnedPtr<UserVM>> try_create();
 
     Result<OwnedPtr<UserVM>> clone();
 
   private:
-    Result<bool> try_expand(usize size = 160);
-    Bitmap m_bitmap;
+    Result<bool> set_region(u64 address, usize count, bool used, bool persistent);
+    Result<void> create_default_region();
+    Result<void> create_null_region();
+    void try_merge_region_with_neighbors(VMRegion* region);
+    void merge_contiguous_regions(VMRegion* a, VMRegion* b);
+    Result<VMRegion*> split_region(VMRegion* parent, u64 boundary);
+    LinkedList<VMRegion> m_regions;
 };

kernel/src/sys/clock_gettime.cpp

@@ -12,19 +12,11 @@ Result<u64> sys_clock_gettime(Registers*, SyscallArgs args)
     switch (id)
     {
     case CLOCK_MONOTONIC: {
-        usize ticks = Timer::ticks_ns();
-        struct timespec kernel_ts;
-        kernel_ts.tv_sec = (time_t)(ticks / NS_PER_SECOND);
-        kernel_ts.tv_nsec = (long)(ticks % NS_PER_SECOND);
-        if (!MemoryManager::copy_to_user_typed(ts, &kernel_ts)) return err(EFAULT);
+        if (!MemoryManager::copy_to_user_typed(ts, Timer::monotonic_clock())) return err(EFAULT);
         break;
     }
     case CLOCK_REALTIME: {
-        usize clock = Timer::clock_ns();
-        struct timespec kernel_ts;
-        kernel_ts.tv_sec = (time_t)(clock / NS_PER_SECOND);
-        kernel_ts.tv_nsec = (long)(clock % NS_PER_SECOND);
-        if (!MemoryManager::copy_to_user_typed(ts, &kernel_ts)) return err(EFAULT);
+        if (!MemoryManager::copy_to_user_typed(ts, Timer::realtime_clock())) return err(EFAULT);
         break;
     }
     default: return err(EINVAL);

kernel/src/sys/exec.cpp

@@ -66,7 +66,9 @@ Result<u64> sys_execve(Registers* regs, SyscallArgs args)
 
     if (!VFS::can_execute(inode, current->auth)) return err(EACCES);
 
-    kinfoln("exec: attempting to replace current image with %s", path.chars());
+#ifdef EXEC_DEBUG
+    kdbgln("exec: attempting to replace current image with %s", path.chars());
+#endif
 
     auto guard = make_scope_guard([current] { MMU::switch_page_directory(current->directory); });
 
@@ -80,7 +82,9 @@ Result<u64> sys_execve(Registers* regs, SyscallArgs args)
 
     // From now on, nothing should fail.
 
-    kinfoln("exec: image load ok, will now replace existing process image");
+#ifdef EXEC_DEBUG
+    kdbgln("exec: image load ok, will now replace existing process image");
+#endif
 
     guard.deactivate();
 
@@ -110,6 +114,8 @@ Result<u64> sys_execve(Registers* regs, SyscallArgs args)
 
     memcpy(regs, &current->regs, sizeof(*regs));
 
+    kinfoln("exec: thread %lu was replaced with %s", current->id, path.chars());
+
     return 0;
 }
 
@@ -151,7 +157,9 @@ Result<u64> sys_fork(Registers* regs, SyscallArgs)
 
     Scheduler::add_thread(thread);
 
-    kinfoln("fork: thread %lu forked into child %lu", current->id, thread->id);
+#ifdef FORK_DEBUG
+    kdbgln("fork: thread %lu forked into child %lu", current->id, thread->id);
+#endif
 
     return thread->id;
 }

kernel/src/sys/exit.cpp

@@ -7,25 +7,5 @@ Result<u64> sys_exit(Registers*, SyscallArgs args)
 
     Thread* current = Scheduler::current();
 
-    Scheduler::for_each_child(current, [](Thread* child) {
-        child->parent = Scheduler::init_thread();
-        return true;
-    });
-
-    auto* parent = current->parent;
-    if (parent && parent->state == ThreadState::Waiting)
-    {
-        auto child = *parent->child_being_waited_for;
-        if (child == -1 || child == (pid_t)current->id)
-        {
-            parent->child_being_waited_for = (pid_t)current->id;
-            parent->wake_up();
-        }
-    }
-
-    current->status = status;
-    current->state = ThreadState::Exited;
-
-    kernel_yield();
-    unreachable();
+    current->exit_and_signal_parent(status);
 }

kernel/src/sys/file.cpp

@@ -16,6 +16,8 @@ Result<u64> sys_read(Registers*, SyscallArgs args)
     u8* buf = (u8*)args[1];
     usize size = (usize)args[2];
 
+    if (!size) return 0;
+
     if (!MemoryManager::validate_user_write(buf, size)) return err(EFAULT);
 
     Thread* current = Scheduler::current();
@@ -44,6 +46,8 @@ Result<u64> sys_write(Registers*, SyscallArgs args)
     const u8* buf = (const u8*)args[1];
     usize size = (usize)args[2];
 
+    if (!size) return 0;
+
     if (!MemoryManager::validate_user_read(buf, size)) return err(EFAULT);
 
     Thread* current = Scheduler::current();

kernel/src/sys/link.cpp

@@ -13,10 +13,8 @@ Result<u64> sys_unlinkat(Registers*, SyscallArgs args)
 
     Thread* current = Scheduler::current();
 
-    PathParser parser = TRY(PathParser::create(path.chars()));
-
-    auto dirname = TRY(parser.dirname());
-    auto basename = TRY(parser.basename());
+    auto dirname = TRY(PathParser::dirname(path.view()));
+    auto basename = TRY(PathParser::basename(path.view()));
 
     if (basename.view() == ".") return err(EINVAL);
 
@@ -25,11 +23,12 @@ Result<u64> sys_unlinkat(Registers*, SyscallArgs args)
     auto inode = TRY(current->resolve_atfile(dirfd, dirname, false, false));
     if (!VFS::can_write(inode, current->auth)) return err(EACCES);
 
-    if (flags > 0)
-    {
-        auto child = TRY(inode->find(basename.chars()));
-        if (child->type() != VFS::InodeType::Directory) return err(ENOTDIR);
-    }
+    auto child = TRY(inode->find(basename.chars()));
+    if (flags == AT_REMOVEDIR && child->type() != VFS::InodeType::Directory) return err(ENOTDIR);
+
+    if (current->auth.euid != 0 && VFS::is_sticky(inode) && current->auth.euid != inode->uid() &&
+        current->auth.euid != child->uid())
+        return err(EACCES);
 
     TRY(inode->remove_entry(basename.chars()));
 
@@ -46,14 +45,13 @@ Result<u64> sys_symlinkat(Registers*, SyscallArgs args)
 
     auto* current = Scheduler::current();
 
-    auto parser = TRY(PathParser::create(linkpath.chars()));
-    auto parent = TRY(parser.dirname());
+    auto parent = TRY(PathParser::dirname(linkpath.view()));
 
     auto parent_inode = TRY(current->resolve_atfile(dirfd, parent, false, true));
 
     if (!VFS::can_write(parent_inode, current->auth)) return err(EACCES);
 
-    auto child_name = TRY(parser.basename());
+    auto child_name = TRY(PathParser::basename(linkpath.view()));
 
     TRY(VFS::validate_filename(child_name.view()));
 
@@ -100,8 +98,7 @@ Result<u64> sys_linkat(Registers*, SyscallArgs args)
 
     auto* current = Scheduler::current();
 
-    auto parser = TRY(PathParser::create(newpath.chars()));
-    auto parent = TRY(parser.dirname());
+    auto parent = TRY(PathParser::dirname(newpath.view()));
 
     // FIXME: Use AT_SYMLINK_FOLLOW.
     auto target = TRY(current->resolve_atfile(olddirfd, oldpath, flags & AT_EMPTY_PATH, false));
@@ -114,7 +111,7 @@ Result<u64> sys_linkat(Registers*, SyscallArgs args)
 
     if (!VFS::can_write(parent_inode, current->auth)) return err(EACCES);
 
-    auto child_name = TRY(parser.basename());
+    auto child_name = TRY(PathParser::basename(newpath.view()));
 
     TRY(VFS::validate_filename(child_name.view()));
 

kernel/src/sys/mmap.cpp

@@ -37,11 +37,13 @@ Result<u64> sys_mmap(Registers*, SyscallArgs args)
     Thread* current = Scheduler::current();
 
     u64 address;
-    if (!addr) address = TRY(current->vm_allocator->alloc_several_pages(get_blocks_from_size(len, ARCH_PAGE_SIZE)));
+    if (!addr) address = TRY(current->vm_allocator->alloc_region(get_blocks_from_size(len, ARCH_PAGE_SIZE)));
     else
     {
-        kwarnln("mmap: FIXME: tried to mmap at a given address, instead of letting us choose");
-        return err(ENOTSUP);
+        // FIXME: We should be more flexible if MAP_FIXED was not specified.
+        address = align_down<ARCH_PAGE_SIZE>((u64)addr);
+        if (!TRY(current->vm_allocator->test_and_alloc_region(address, get_blocks_from_size(len, ARCH_PAGE_SIZE))))
+            return err(ENOMEM);
     }
 
     int mmu_flags = MMU::User | MMU::NoExecute;
@@ -53,6 +55,7 @@ Result<u64> sys_mmap(Registers*, SyscallArgs args)
     kdbgln("mmap: mapping memory at %#lx, size=%zu", address, len);
 #endif
 
+    // FIXME: This leaks VM if it fails.
     return MemoryManager::alloc_at_zeroed(address, get_blocks_from_size(len, ARCH_PAGE_SIZE), mmu_flags);
 }
 
@@ -66,7 +69,7 @@ Result<u64> sys_munmap(Registers*, SyscallArgs args)
 
     Thread* current = Scheduler::current();
 
-    bool ok = TRY(current->vm_allocator->free_several_pages(address, get_blocks_from_size(size, ARCH_PAGE_SIZE)));
+    bool ok = TRY(current->vm_allocator->free_region(address, get_blocks_from_size(size, ARCH_PAGE_SIZE)));
 
     // POSIX says munmap should silently do nothing if the memory was not already mapped.
     if (!ok) return 0;

kernel/src/sys/mount.cpp

@@ -12,10 +12,14 @@ Result<u64> sys_mount(Registers*, SyscallArgs args)
     auto* current = Scheduler::current();
     if (current->auth.euid != 0) return err(EPERM);
 
-    // Right now we only support one file system.
-    if (fstype.view() != "tmpfs") return err(ENODEV);
+    SharedPtr<VFS::FileSystem> fs;
+
+    if (fstype.view() == "tmpfs") fs = TRY(TmpFS::FileSystem::create());
+    else if (fstype.view() == "devfs")
+        fs = TRY(DeviceRegistry::create_devfs_instance());
+    else
+        return err(ENODEV);
 
-    auto fs = TRY(TmpFS::FileSystem::create());
     TRY(VFS::mount(target.chars(), fs, current->auth, current->current_directory));
 
     return 0;
@@ -32,3 +36,16 @@ Result<u64> sys_umount(Registers*, SyscallArgs args)
 
     return 0;
 }
+
+Result<u64> sys_pivot_root(Registers*, SyscallArgs args)
+{
+    auto new_root = TRY(MemoryManager::strdup_from_user(args[0]));
+    auto put_old = TRY(MemoryManager::strdup_from_user(args[1]));
+
+    auto* current = Scheduler::current();
+    if (current->auth.euid != 0) return err(EPERM);
+
+    TRY(VFS::pivot_root(new_root.chars(), put_old.chars(), current->current_directory));
+
+    return 0;
+}

kernel/src/sys/open.cpp

@@ -74,7 +74,9 @@ Result<u64> sys_openat(Registers*, SyscallArgs args)
 
     int fd = TRY(current->allocate_fd(0));
 
-    kinfoln("openat: opening file %s from dirfd %d, flags %d, mode %#o = fd %d", path.chars(), dirfd, flags, mode, fd);
+#ifdef OPEN_DEBUG
+    kdbgln("openat: opening file %s from dirfd %d, flags %d, mode %#o = fd %d", path.chars(), dirfd, flags, mode, fd);
+#endif
 
     inode->add_handle();
 

kernel/src/sys/usleep.cpp

@@ -6,6 +6,9 @@ Result<u64> sys_usleep(Registers*, SyscallArgs args)
 {
     useconds_t us = (useconds_t)args[0];
 
+    // FIXME: Allow usleep() to use a more precise resolution.
+    if (us < 1000) return 0;
+
     kernel_sleep(us / 1000);
 
     return 0;

kernel/src/thread/ELF.cpp

@@ -25,7 +25,7 @@ static bool can_write_segment(u32 flags)
 
 namespace ELFLoader
 {
-    Result<ELFData> load(SharedPtr<VFS::Inode> inode)
+    Result<ELFData> load(SharedPtr<VFS::Inode> inode, UserVM* vm)
     {
         Elf64_Ehdr elf_header;
         usize nread = TRY(inode->read((u8*)&elf_header, 0, sizeof elf_header));
@@ -73,7 +73,9 @@ namespace ELFLoader
             return err(ENOEXEC);
         }
 
-        kinfoln("ELF: Loading ELF with entry=%#.16lx", elf_header.e_entry);
+#ifdef ELF_DEBUG
+        kdbgln("ELF: Loading ELF with entry=%#.16lx", elf_header.e_entry);
+#endif
 
         usize i;
         Elf64_Phdr program_header;
@@ -100,6 +102,10 @@ namespace ELFLoader
                 if (can_write_segment(program_header.p_flags)) flags |= MMU::ReadWrite;
                 if (can_execute_segment(program_header.p_flags)) flags &= ~MMU::NoExecute;
 
+                if (!TRY(vm->test_and_alloc_region(
+                        base_vaddr, get_blocks_from_size(program_header.p_memsz + vaddr_diff, ARCH_PAGE_SIZE), true)))
+                    return err(ENOMEM);
+
                 // Allocate physical memory for the segment
                 TRY(MemoryManager::alloc_at(
                     base_vaddr, get_blocks_from_size(program_header.p_memsz + vaddr_diff, ARCH_PAGE_SIZE), flags));

kernel/src/thread/ELF.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "fs/VFS.h"
+#include "memory/UserVM.h"
 #include <luna/Types.h>
 
 #define ELFMAG "\177ELF"
@@ -53,5 +54,5 @@ struct ELFData
 
 namespace ELFLoader
 {
-    Result<ELFData> load(SharedPtr<VFS::Inode> inode);
+    Result<ELFData> load(SharedPtr<VFS::Inode> inode, UserVM* vm);
 };

kernel/src/thread/Scheduler.cpp

@@ -31,8 +31,8 @@ namespace Scheduler
         g_idle.ticks_left = 1;
 
         // Map some stack for the idle task
-        u64 idle_stack_vm = MemoryManager::alloc_for_kernel(1, MMU::NoExecute | MMU::ReadWrite)
-                                .expect_value("Error while setting up the idle task, cannot continue");
+        u64 idle_stack_vm = mark_critical(MemoryManager::alloc_for_kernel(1, MMU::NoExecute | MMU::ReadWrite),
+                                          "Failed to allocate stack memory for the CPU idle thread");
 
         Stack idle_stack { idle_stack_vm, ARCH_PAGE_SIZE };
         g_idle.set_sp(idle_stack.top());
@@ -171,7 +171,9 @@ namespace Scheduler
     {
         CPU::disable_interrupts();
 
-        kinfoln("reap: reaping thread with id %zu", thread->id);
+#ifdef REAP_DEBUG
+        kdbgln("reap: reaping thread with id %zu", thread->id);
+#endif
 
         if (thread->is_kernel)
         {
@@ -233,9 +235,11 @@ namespace Scheduler
         {
             switch_context(old_thread, new_thread, regs);
             if (!old_thread->is_kernel) old_thread->fp_data.save();
+            if (old_thread->is_kernel && MMU::get_page_directory() != MMU::kernel_page_directory())
+                old_thread->directory = MMU::get_page_directory();
+            if (new_thread->directory) MMU::switch_page_directory(new_thread->directory);
             if (!new_thread->is_kernel)
             {
-                MMU::switch_page_directory(new_thread->directory);
                 CPU::switch_kernel_stack(new_thread->kernel_stack.top());
                 new_thread->fp_data.restore();
             }
@@ -271,7 +275,7 @@ namespace Scheduler
         {
             if (thread->state == ThreadState::Sleeping)
             {
-                if (--thread->sleep_ticks_left == 0) thread->wake_up();
+                if (thread->sleep_ticks_left == 0 || --thread->sleep_ticks_left == 0) thread->wake_up();
             }
         }
 

kernel/src/thread/Thread.cpp

@@ -1,5 +1,6 @@
 #include "thread/Thread.h"
 #include "memory/MemoryManager.h"
+#include "thread/Scheduler.h"
 #include <bits/atfile.h>
 #include <bits/open-flags.h>
 #include <luna/Alloc.h>
@@ -68,6 +69,33 @@ Result<SharedPtr<VFS::Inode>> Thread::resolve_atfile(int dirfd, const String& pa
     return VFS::resolve_path(path.chars(), this->auth, descriptor->inode, follow_last_symlink);
 }
 
+[[noreturn]] void Thread::exit_and_signal_parent(u8 _status)
+{
+    if (is_kernel) state = ThreadState::Dying;
+    else
+    {
+        Scheduler::for_each_child(this, [](Thread* child) {
+            child->parent = Scheduler::init_thread();
+            return true;
+        });
+
+        if (parent && parent->state == ThreadState::Waiting)
+        {
+            auto child = *parent->child_being_waited_for;
+            if (child == -1 || child == (pid_t)id)
+            {
+                parent->child_being_waited_for = (pid_t)id;
+                parent->wake_up();
+            }
+        }
+
+        state = ThreadState::Exited;
+    }
+    status = _status;
+    kernel_yield();
+    unreachable();
+}
+
 bool FileDescriptor::should_append()
 {
     return flags & O_APPEND;

kernel/src/thread/Thread.h

@@ -99,6 +99,8 @@ struct Thread : public LinkedListNode<Thread>
 
     PageDirectory* directory;
 
+    [[noreturn]] void exit_and_signal_parent(u8 status);
+
     bool is_idle()
     {
         return state == ThreadState::Idle;

kernel/src/thread/ThreadImage.cpp

@@ -1,15 +1,18 @@
 #include "thread/ThreadImage.h"
 #include "memory/MemoryManager.h"
 #include "thread/Thread.h"
+#include <luna/Alignment.h>
 #include <luna/CString.h>
 
 static constexpr usize DEFAULT_USER_STACK_PAGES = 6;
 static constexpr usize DEFAULT_USER_STACK_SIZE = DEFAULT_USER_STACK_PAGES * ARCH_PAGE_SIZE;
 
-static Result<void> create_stacks(Stack& user_stack, Stack& kernel_stack)
+static Result<void> create_stacks(Stack& user_stack, Stack& kernel_stack, UserVM* vm)
 {
     const u64 THREAD_STACK_BASE = 0x10000;
 
+    if (!TRY(vm->test_and_alloc_region(THREAD_STACK_BASE, DEFAULT_USER_STACK_PAGES, true))) return err(ENOMEM);
+
     TRY(MemoryManager::alloc_at_zeroed(THREAD_STACK_BASE, DEFAULT_USER_STACK_PAGES,
                                        MMU::ReadWrite | MMU::NoExecute | MMU::User));
 
@@ -42,11 +45,11 @@ Result<OwnedPtr<ThreadImage>> ThreadImage::try_load_from_elf(SharedPtr<VFS::Inod
         MMU::switch_page_directory(old_directory);
     });
 
-    const ELFData data = TRY(ELFLoader::load(inode));
+    const ELFData data = TRY(ELFLoader::load(inode, vm_allocator.ptr()));
 
     Stack user_stack;
     Stack kernel_stack;
-    TRY(create_stacks(user_stack, kernel_stack));
+    TRY(create_stacks(user_stack, kernel_stack, vm_allocator.ptr()));
 
     guard.deactivate();
 
@@ -119,7 +122,7 @@ void ThreadImage::apply(Thread* thread)
 
     thread->kernel_stack = m_kernel_stack;
     thread->stack = m_user_stack;
-    thread->set_sp(m_sp);
+    thread->set_sp(align_down<16>(m_sp));
 
     thread->directory = m_directory;
 

kernel/src/video/TextConsole.cpp

@@ -127,12 +127,12 @@ namespace TextConsole
     {
         auto guard = make_scope_guard([] { utf8_decoder.reset(); });
 
-        const Option<wchar_t> maybe_wchar = TRY(utf8_decoder.feed(c));
+        bool is_ready = TRY(utf8_decoder.feed(c));
+
+        if (is_ready) putwchar(TRY(utf8_decoder.extract()));
 
         guard.deactivate();
 
-        if (maybe_wchar.has_value()) putwchar(maybe_wchar.value());
-
         return {};
     }
 

libc/CMakeLists.txt

@@ -19,6 +19,8 @@ set(SOURCES
     src/env.cpp
     src/pwd.cpp
     src/grp.cpp
+    src/locale.cpp
+    src/scanf.cpp
     src/sys/stat.cpp
     src/sys/mman.cpp
     src/sys/wait.cpp

libc/include/bits/errno-return.h

@@ -5,6 +5,7 @@
 
 #include <errno.h>
 
+// Set errno after a failed system call, otherwise extract the successful value.
 #define __errno_return(value, type)                                                                                    \
     do {                                                                                                               \
         if (value < 0)                                                                                                 \
@@ -15,4 +16,18 @@
         return (type)value;                                                                                            \
     } while (0)
 
+// If expr has a value, evaluates to that value. Otherwise, short-circuits, sets errno to expr's error, and returns
+// failval (casted to rtype) from the calling function. Similar to TRY() in luna/Result.h, but used for transforming
+// errors from luna functions to libc functions.
+#define TRY_OR_SET_ERRNO(expr, rtype, failval)                                                                         \
+    ({                                                                                                                 \
+        auto _expr_rc = (expr);                                                                                        \
+        if (!_expr_rc.has_value())                                                                                     \
+        {                                                                                                              \
+            errno = _expr_rc.error();                                                                                  \
+            return (rtype)failval;                                                                                     \
+        }                                                                                                              \
+        _expr_rc.release_value();                                                                                      \
+    })
+
 #endif

libc/include/bits/locale-cat.h

@@ -0,0 +1,18 @@
+/* bits/locale-cat.h: Locale categories. */
+
+#ifndef _BITS_LOCALE_CAT_H
+#define _BITS_LOCALE_CAT_H
+
+enum __libc_locale_category
+{
+    LC_ALL,      // Controls all locales.
+    LC_CTYPE,    // Character classification and case conversion.
+    LC_COLLATE,  // Collation order.
+    LC_MONETARY, // Monetary formatting.
+    LC_NUMERIC,  // Numeric, non-monetary formatting.
+    LC_TIME,     // Date and time formats.
+    LC_MESSAGES, // Formats of informative and diagnostic messages and interactive responses.
+    __num_locale_categories,
+};
+
+#endif

libc/include/bits/timespec.h

@@ -18,3 +18,19 @@ struct timeval
 };
 
 #endif
+
+#if defined(IN_MOON) || defined(_INCLUDE_TIMESPEC_MACROS)
+#ifndef _TIMESPEC_MACROS_INCLUDED
+#define _TIMESPEC_MACROS_INCLUDED
+#define timespecadd(a, b, res)                                                                                         \
+    do {                                                                                                               \
+        (res)->tv_sec = (a)->tv_sec + (b)->tv_sec;                                                                     \
+        (res)->tv_nsec = (a)->tv_nsec + (b)->tv_nsec;                                                                  \
+        while ((res)->tv_nsec >= 1'000'000'000)                                                                        \
+        {                                                                                                              \
+            (res)->tv_sec++;                                                                                           \
+            (res)->tv_nsec -= 1'000'000'000;                                                                           \
+        }                                                                                                              \
+    } while (0);
+#endif
+#endif

libc/include/locale.h

@@ -0,0 +1,20 @@
+/* locale.h: Locale category macros. */
+
+#ifndef _LOCALE_H
+#define _LOCALE_H
+
+#include <bits/locale-cat.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    // Query or set the current locale.
+    char* setlocale(int category, const char* locale);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

libc/include/stdio.h

@@ -27,6 +27,9 @@ extern FILE* stderr;
 #define stderr stderr
 
 #define BUFSIZ 1024
+#define FILENAME_MAX                                                                                                   \
+    1024 // As Luna does not impose a limit on this, this is the recommended size for character arrays holding a file
+         // name.
 
 #ifdef __cplusplus
 extern "C"
@@ -41,6 +44,9 @@ extern "C"
     /* Bind a stream to a file descriptor. */
     FILE* fdopen(int fd, const char* mode);
 
+    /* Change the underlying file and mode of a stream. */
+    FILE* freopen(const char* path, const char* mode, FILE* stream);
+
     /* Close a file and frees up its stream. */
     int fclose(FILE* stream);
 
@@ -95,6 +101,9 @@ extern "C"
     /* Read a character from standard input. */
     int getchar(void);
 
+    /* Push a character back to stream so that it can be read again. */
+    int ungetc(int c, FILE* stream);
+
     /* Read a line from stream. */
     char* fgets(char* buf, size_t size, FILE* stream);
 
@@ -122,16 +131,34 @@ extern "C"
     int snprintf(char* buf, size_t max, const char* format, ...);
 
     /* Write formatted output into a buffer. */
-    int vsprintf(char*, const char*, va_list);
+    int vsprintf(char* buf, const char* format, va_list ap);
 
     /* Write up to max bytes of formatted output into a buffer. */
-    int vsnprintf(char*, size_t, const char*, va_list);
+    int vsnprintf(char* buf, size_t max, const char* format, va_list ap);
 
     /* Write formatted output to standard output. */
-    int vprintf(const char*, va_list ap);
+    int vprintf(const char* format, va_list ap);
 
     /* Write formatted output to standard output. */
-    int printf(const char*, ...);
+    int printf(const char* format, ...);
+
+    /* Scan formatted input from a string. */
+    int vsscanf(const char* str, const char* format, va_list ap);
+
+    /* Scan formatted input from a string. */
+    int sscanf(const char* str, const char* format, ...);
+
+    /* Scan formatted input from a file. */
+    int vfscanf(FILE* stream, const char* format, va_list ap);
+
+    /* Scan formatted input from a file. */
+    int fscanf(FILE* stream, const char* format, ...);
+
+    /* Scan formatted input from standard input. */
+    int vscanf(const char* format, va_list ap);
+
+    /* Scan formatted input from standard input. */
+    int scanf(const char* format, ...);
 
     /* Write a string followed by a newline to standard output. */
     int puts(const char* s);

libc/include/stdlib.h

@@ -31,6 +31,9 @@ typedef struct
 
 #define RAND_MAX 32767
 
+#define EXIT_SUCCESS 0
+#define EXIT_FAILURE 1
+
 #ifdef __cplusplus
 extern "C"
 {
@@ -93,6 +96,9 @@ extern "C"
      * endptr if nonnull. */
     unsigned long strtoul(const char* str, char** endptr, int base);
 
+#define strtoll strtol
+#define strtoull strtoul
+
     /* Return the next pseudorandom number. */
     int rand();
 

libc/include/sys/param.h

@@ -0,0 +1,16 @@
+/* sys/param.h: Old-style BSD macros. */
+
+#ifndef _SYS_PARAM_H
+#define _SYS_PARAM_H
+
+#include <limits.h>
+
+#ifndef MIN
+#define MIN(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
+
+#endif

libc/include/sys/time.h

@@ -3,6 +3,8 @@
 #ifndef _SYS_TIME_H
 #define _SYS_TIME_H
 
+#define _INCLUDE_TIMESPEC_MACROS
+
 #include <bits/attrs.h>
 #include <bits/timespec.h>
 

libc/src/env.cpp

@@ -1,3 +1,4 @@
+#include <bits/errno-return.h>
 #include <errno.h>
 #include <luna/ScopeGuard.h>
 #include <luna/Vector.h>
@@ -83,12 +84,8 @@ static Result<void> _try_move_env()
 
 static int _move_env()
 {
-    auto rc = _try_move_env();
-    if (rc.has_error())
-    {
-        errno = rc.error();
-        return -1;
-    }
+    TRY_OR_SET_ERRNO(_try_move_env(), int, -1);
+
     return 0;
 }
 
@@ -178,12 +175,7 @@ extern "C"
         // Add a new NULL at the end of the array and replace the previous one with our string.
         index = g_dynamic_env.size() - 1;
 
-        auto rc = g_dynamic_env.try_append(nullptr);
-        if (rc.has_error())
-        {
-            errno = rc.error();
-            return -1;
-        }
+        TRY_OR_SET_ERRNO(g_dynamic_env.try_append(nullptr), int, -1);
         guard.deactivate();
         _update_env();
 

libc/src/grp.cpp

@@ -1,3 +1,4 @@
+#include <bits/errno-return.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <grp.h>
@@ -67,13 +68,7 @@ extern "C"
 
     struct group* getgrent()
     {
-        auto rc = try_getgrent();
-        if (rc.has_error())
-        {
-            errno = rc.error();
-            return nullptr;
-        }
-        return rc.value();
+        return TRY_OR_SET_ERRNO(try_getgrent(), group*, nullptr);
     }
 
     struct group* getgrnam(const char* name)

libc/src/locale.cpp

@@ -0,0 +1,12 @@
+#include <locale.h>
+
+static char s_default_locale[] = "C";
+
+extern "C"
+{
+    char* setlocale(int, const char*)
+    {
+        // FIXME: Set the current locale if <locale> is not NULL.
+        return s_default_locale;
+    }
+}

libc/src/scanf.cpp

@@ -0,0 +1,263 @@
+#include <errno.h>
+#include <luna/CType.h>
+#include <luna/NumberParsing.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define FLAG_DISCARD (1 << 0)
+#define FLAG_ALLOC (1 << 1)
+#define FLAG_WIDTH (1 << 2)
+#define FLAG_LONG (1 << 3)
+#define FLAG_LONG_LONG (1 << 4)
+#define FLAG_SHORT (1 << 5)
+#define FLAG_CHAR (1 << 6)
+
+static int parse_flags(const char** format)
+{
+    int result = 0;
+
+    while (true)
+    {
+        switch (**format)
+        {
+        case '*':
+            result |= FLAG_DISCARD;
+            (*format)++;
+            break;
+        case 'm':
+            result |= FLAG_ALLOC;
+            (*format)++;
+            break;
+        default: return result;
+        }
+    }
+}
+
+static size_t parse_width(const char** format, int& flags)
+{
+    size_t result = 0;
+
+    if (_isdigit(**format))
+    {
+        result = scan_unsigned_integer(format);
+        flags |= FLAG_WIDTH;
+    }
+
+    return result;
+}
+
+static void parse_type(const char** format, int& flags)
+{
+    // FIXME: Support %j (intmax_t/uintmax_t)
+    switch (**format)
+    {
+    case 'h':
+        flags |= FLAG_SHORT;
+        (*format)++;
+        if (**format == 'h')
+        {
+            flags |= FLAG_CHAR;
+            (*format)++;
+        }
+        break;
+    case 'l':
+        flags |= FLAG_LONG;
+        (*format)++;
+        if (**format == 'l')
+        {
+            flags |= FLAG_LONG_LONG;
+            (*format)++;
+        }
+        break;
+    case 't':
+        flags |= (sizeof(ptrdiff_t) == sizeof(long)) ? FLAG_LONG : FLAG_LONG_LONG;
+        (*format)++;
+        break;
+    case 'z':
+        flags |= (sizeof(size_t) == sizeof(long)) ? FLAG_LONG : FLAG_LONG_LONG;
+        (*format)++;
+        break;
+    default: break;
+    }
+}
+
+static void write_parsed_signed_integer(ssize_t value, int flags, va_list ap)
+{
+    if (flags & FLAG_LONG_LONG) *va_arg(ap, signed long long*) = (signed long long)value;
+    else if (flags & FLAG_LONG)
+        *va_arg(ap, signed long*) = (signed long)value;
+    else if (flags & FLAG_SHORT)
+        *va_arg(ap, signed int*) = (signed short)value;
+    else if (flags & FLAG_CHAR)
+        *va_arg(ap, signed int*) = (signed char)value;
+    else
+        *va_arg(ap, signed int*) = (signed int)value;
+}
+
+static void write_parsed_unsigned_integer(size_t value, int flags, va_list ap)
+{
+    if (flags & FLAG_LONG_LONG) *va_arg(ap, unsigned long long*) = (unsigned long long)value;
+    else if (flags & FLAG_LONG)
+        *va_arg(ap, unsigned long*) = (unsigned long)value;
+    else if (flags & FLAG_SHORT)
+        *va_arg(ap, unsigned int*) = (unsigned short)value;
+    else if (flags & FLAG_CHAR)
+        *va_arg(ap, unsigned int*) = (unsigned char)value;
+    else
+        *va_arg(ap, unsigned int*) = (unsigned int)value;
+}
+
+#define WHITESPACE_CHARACTERS " \t\f\r\n\v"
+
+static void skip_whitespace(const char** str)
+{
+    *str += strspn(*str, WHITESPACE_CHARACTERS);
+}
+
+extern "C"
+{
+    int vsscanf(const char* str, const char* format, va_list ap)
+    {
+        int parsed = 0;
+        const char* s = str; // Keep a pointer to the beginning of the string for %n
+
+        if (*str == 0) return EOF;
+
+        while (*format)
+        {
+            if (*format != '%')
+            {
+            normal:
+                if (!_isspace(*format))
+                {
+                    if (*str != *format) return parsed;
+                    str++;
+                    format++;
+                    if (*str == 0) return parsed;
+                    continue;
+                }
+
+                skip_whitespace(&format);
+                skip_whitespace(&str);
+                if (*str == 0) return parsed;
+                continue;
+            }
+            else
+            {
+                format++;
+                if (*format == '%')
+                {
+                    skip_whitespace(&str);
+                    goto normal;
+                }
+
+                int flags = parse_flags(&format);
+                size_t width = parse_width(&format, flags);
+                parse_type(&format, flags);
+                char specifier = *format++;
+                if (!specifier) return parsed;
+
+                switch (specifier)
+                {
+                case 's': {
+                    skip_whitespace(&str);
+                    size_t chars = strcspn(str, WHITESPACE_CHARACTERS);
+                    if (!chars) return parsed;
+                    if ((flags & FLAG_WIDTH) && chars > width) chars = width;
+                    if (!(flags & FLAG_DISCARD))
+                    {
+                        char* ptr;
+                        if (flags & FLAG_ALLOC)
+                        {
+                            ptr = (char*)malloc(chars + 1);
+                            if (!ptr) return parsed;
+                            *va_arg(ap, char**) = ptr;
+                        }
+                        else
+                            ptr = va_arg(ap, char*);
+                        memcpy(ptr, str, chars);
+                        ptr[chars] = 0;
+                    }
+                    str += chars;
+                    parsed++;
+                    break;
+                }
+                case 'c': {
+                    if (strlen(str) < width) return parsed;
+                    if (!(flags & FLAG_WIDTH)) width = 1;
+                    if (!(flags & FLAG_DISCARD))
+                    {
+                        char* ptr;
+                        if (flags & FLAG_ALLOC)
+                        {
+                            ptr = (char*)malloc(width);
+                            if (!ptr) return parsed;
+                            *va_arg(ap, char**) = ptr;
+                        }
+                        else
+                            ptr = va_arg(ap, char*);
+                        memcpy(ptr, str, width);
+                    }
+                    str += width;
+                    parsed++;
+                    break;
+                }
+                case 'd': {
+                    skip_whitespace(&str);
+                    ssize_t value = scan_signed_integer(&str, 10);
+                    if (!(flags & FLAG_DISCARD)) write_parsed_signed_integer(value, flags, ap);
+                    parsed++;
+                    break;
+                }
+                case 'i': {
+                    skip_whitespace(&str);
+                    ssize_t value = scan_signed_integer(&str, 0);
+                    if (!(flags & FLAG_DISCARD)) write_parsed_signed_integer(value, flags, ap);
+                    parsed++;
+                    break;
+                }
+                case 'o': {
+                    skip_whitespace(&str);
+                    size_t value = scan_unsigned_integer(&str, 8);
+                    if (!(flags & FLAG_DISCARD)) write_parsed_unsigned_integer(value, flags, ap);
+                    parsed++;
+                    break;
+                }
+                case 'u': {
+                    skip_whitespace(&str);
+                    size_t value = scan_unsigned_integer(&str, 10);
+                    if (!(flags & FLAG_DISCARD)) write_parsed_unsigned_integer(value, flags, ap);
+                    parsed++;
+                    break;
+                }
+                case 'X':
+                case 'x': {
+                    skip_whitespace(&str);
+                    size_t value = scan_unsigned_integer(&str, 16);
+                    if (!(flags & FLAG_DISCARD)) write_parsed_unsigned_integer(value, flags, ap);
+                    parsed++;
+                    break;
+                }
+                case 'p': {
+                    skip_whitespace(&str);
+                    size_t value = scan_unsigned_integer(&str, 16);
+                    if (!(flags & FLAG_DISCARD)) *va_arg(ap, void**) = (void*)value;
+                    parsed++;
+                    break;
+                }
+                case 'n': {
+                    if (!(flags & FLAG_DISCARD)) *va_arg(ap, int*) = (int)(str - s);
+                    break;
+                }
+                default: {
+                    fprintf(stderr, "vsscanf: unknown conversion specifier: %%%c\n", specifier);
+                    return parsed;
+                }
+                }
+            }
+        }
+
+        return parsed;
+    }
+}

libc/src/stdio.cpp

@@ -11,6 +11,13 @@ FILE* stdin = nullptr;
 FILE* stderr = nullptr;
 FILE* stdout = nullptr;
 
+static const char* read_tmpdir()
+{
+    const char* tmpdir = getenv("TMPDIR");
+    if (!tmpdir) return "/tmp";
+    return tmpdir;
+}
+
 static int fopen_parse_mode(const char* mode)
 {
     int result = 0;
@@ -46,6 +53,12 @@ static int fdopen_check_compatible_mode(int fd, int new_flags)
 
 extern "C"
 {
+    int fflush(FILE*)
+    {
+        // FIXME: Files are not buffered right now.
+        return 0;
+    }
+
     FILE* fopen(const char* path, const char* mode)
     {
         int flags;
@@ -85,6 +98,25 @@ extern "C"
         return f;
     }
 
+    FILE* freopen(const char* path, const char* mode, FILE* stream)
+    {
+        int flags;
+
+        if ((flags = fopen_parse_mode(mode)) < 0) return nullptr;
+
+        close(stream->_fd);
+
+        if (!path) { fail("FIXME: freopen() called with path=nullptr"); }
+
+        int fd = open(path, flags, 0666);
+        if (fd < 0) { return nullptr; }
+
+        stream->_fd = fd;
+        clearerr(stream);
+
+        return stream;
+    }
+
     int fclose(FILE* stream)
     {
         if (close(stream->_fd) < 0) return EOF;
@@ -381,6 +413,54 @@ extern "C"
         return rc;
     }
 
+    int sscanf(const char* str, const char* format, ...)
+    {
+        va_list ap;
+        va_start(ap, format);
+
+        int rc = vsscanf(str, format, ap);
+
+        va_end(ap);
+
+        return rc;
+    }
+
+    int vfscanf(FILE* stream, const char* format, va_list ap)
+    {
+        char buf[BUFSIZ];
+        if (!fgets(buf, sizeof(buf), stream)) return EOF;
+        return vsscanf(buf, format, ap);
+    }
+
+    int fscanf(FILE* stream, const char* format, ...)
+    {
+        va_list ap;
+        va_start(ap, format);
+
+        int rc = vfscanf(stream, format, ap);
+
+        va_end(ap);
+
+        return rc;
+    }
+
+    int vscanf(const char* format, va_list ap)
+    {
+        return vfscanf(stdin, format, ap);
+    }
+
+    int scanf(const char* format, ...)
+    {
+        va_list ap;
+        va_start(ap, format);
+
+        int rc = vfscanf(stdin, format, ap);
+
+        va_end(ap);
+
+        return rc;
+    }
+
     int puts(const char* s)
     {
         if (fputs(s, stdout) < 0) return -1;
@@ -404,18 +484,16 @@ extern "C"
 
     FILE* tmpfile()
     {
-        // FIXME: use /tmp as the directory when the tmpfs is mounted only there.
-        int fd = open("/", O_RDWR | O_TMPFILE, 0600);
+        int fd = open(read_tmpdir(), O_RDWR | O_TMPFILE, 0600);
         if (fd < 0) return nullptr;
 
         FILE* f = fdopen(fd, "w+b");
         if (!f) close(fd);
         return f;
     }
-}
 
-void debug_log_impl(const char* format, va_list ap)
-{
-    vfprintf(stderr, format, ap);
-    fputc('\n', stderr);
+    int ungetc(int, FILE*)
+    {
+        fail("FIXME: ungetc: not implemented");
+    }
 }

libc/src/stdlib.cpp

@@ -1,3 +1,4 @@
+#include <bits/errno-return.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -144,35 +145,17 @@ extern "C"
 
     void* malloc(size_t size)
     {
-        auto rc = malloc_impl(size);
-        if (rc.has_error())
-        {
-            errno = rc.error();
-            return nullptr;
-        }
-        return rc.value();
+        return TRY_OR_SET_ERRNO(malloc_impl(size), void*, nullptr);
     }
 
     void* calloc(size_t nmemb, size_t size)
     {
-        auto rc = calloc_impl(nmemb, size);
-        if (rc.has_error())
-        {
-            errno = rc.error();
-            return nullptr;
-        }
-        return rc.value();
+        return TRY_OR_SET_ERRNO(calloc_impl(nmemb, size), void*, nullptr);
     }
 
     void* realloc(void* ptr, size_t size)
     {
-        auto rc = realloc_impl(ptr, size);
-        if (rc.has_error())
-        {
-            errno = rc.error();
-            return nullptr;
-        }
-        return rc.value();
+        return TRY_OR_SET_ERRNO(realloc_impl(ptr, size), void*, nullptr);
     }
 
     void free(void* ptr)
@@ -228,7 +211,7 @@ extern "C"
     static void generate_random_character(char* ptr)
     {
         constexpr const char chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_";
-        *ptr = chars[rand() % sizeof(chars)];
+        *ptr = chars[rand() % (sizeof(chars) - 1)];
     }
 
     static int check_template(char* _template, size_t* len)

libc/src/sys/mman.cpp

@@ -18,17 +18,3 @@ extern "C"
         __errno_return(rc, int);
     }
 }
-
-Result<void*> allocate_pages_impl(usize count)
-{
-    long rc = syscall(SYS_mmap, nullptr, count * PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS);
-    if (rc < 0) { return err((int)-rc); }
-    return (void*)rc;
-}
-
-Result<void> release_pages_impl(void* address, usize count)
-{
-    long rc = syscall(SYS_munmap, address, count * PAGE_SIZE);
-    if (rc < 0) { return err((int)-rc); }
-    return {};
-}

libc/src/unistd.cpp

@@ -42,7 +42,8 @@ static Result<int> try_execvpe(const char* name, char* const* argv, char* const*
         {
             Vector<char*> shell_argv;
             TRY(shell_argv.try_append(const_cast<char*>("sh")));
-            char* const* arg = argv;
+            TRY(shell_argv.try_append(file.mutable_data()));
+            char* const* arg = argv + 1;
             do {
                 TRY(shell_argv.try_append(*arg));
             } while (*(arg++));
@@ -168,13 +169,7 @@ extern "C"
 
     int execvpe(const char* name, char* const* argv, char* const* envp)
     {
-        auto rc = try_execvpe(name, argv, envp);
-        if (rc.has_error())
-        {
-            errno = rc.error();
-            return -1;
-        }
-        return rc.value();
+        return TRY_OR_SET_ERRNO(try_execvpe(name, argv, envp), int, -1);
     }
 
     int execvp(const char* name, char* const* argv)