kernel+apps+base+tools: Use Ext2 for the root partition file system

init is now split into two parts: preinit, which lives in the initrd and prepares the root file system for init,
and the actual /usr/bin/init, which lives in the root partition and starts services and reaps zombies.

The kernel now looks for /bin/preinit instead of /bin/init as the executable for the init process.

All configuration files in initrd/etc have been moved to base/etc. (The plan is to have only moon and preinit in the initrd.)

Since the current Ext2 implementation is read-only (and it's on a CDROM so it would be read-only anyways),
/home/selene is a tmpfs (as well as /tmp), to allow for a writable home directory.

The system is slower now, but that's to expect since the Ext2 code doesn't use caching and the ATA code still uses PIO.

.gitignore

@@ -4,7 +4,6 @@ build/
 initrd/boot/moon
 env-local.sh
 initrd/bin/**
-initrd/tests/**
-base/
+base/usr/**
 .fakeroot
 kernel/config.cmake

README.md

@@ -6,9 +6,10 @@ A very basic POSIX-based operating system for personal computers, written in C++
 
 ## Features
 - x86_64-compatible lightweight [kernel](kernel/).
-- Preemptive multitasking, with a round-robin [scheduler](kernel/src/thread/) that can switch between tasks.
+- Preemptive multitasking, with a round-robin [scheduler](kernel/src/thread/).
 - [Virtual file system](kernel/src/fs/) with a simple [tmpfs](kernel/src/fs/tmpfs/) and read-only [ext2](kernel/src/fs/ext2/) support.
 - Can [load ELF programs](kernel/src/thread/ELF.cpp) from the file system as userspace tasks.
+- Boots from an [ext2](apps/preinit.cpp) root filesystem (a bit slow for now).
 - [System call](kernel/src/sys/) interface and [C Library](libc/), aiming to be mostly POSIX-compatible.
 - Designed to be [portable](kernel/src/arch), no need to be restricted to x86_64.
 - Designed around [UTF-8](libluna/include/luna/Utf8.h).

apps/CMakeLists.txt

@@ -4,9 +4,15 @@ function(luna_app SOURCE_FILE APP_NAME)
     add_dependencies(${APP_NAME} libc)
     target_include_directories(${APP_NAME} PRIVATE ${LUNA_BASE}/usr/include)
     target_link_libraries(${APP_NAME} PRIVATE os)
-    install(TARGETS ${APP_NAME} DESTINATION ${LUNA_ROOT}/initrd/bin)
+    install(TARGETS ${APP_NAME} DESTINATION ${LUNA_BASE}/usr/bin)
 endfunction()
 
+add_executable(preinit preinit.cpp)
+target_compile_options(preinit PRIVATE -Os ${COMMON_FLAGS} -Wno-write-strings)
+add_dependencies(preinit libc)
+target_include_directories(preinit PRIVATE ${LUNA_BASE}/usr/include)
+install(TARGETS preinit DESTINATION ${LUNA_ROOT}/initrd/bin)
+
 luna_app(init.cpp init)
 luna_app(env.cpp env)
 luna_app(su.cpp su)

apps/init.cpp

@@ -16,7 +16,7 @@
 #include <sys/sysmacros.h>
 #include <unistd.h>
 
-FILE* g_init_log;
+FILE* g_init_log = nullptr;
 
 struct Service
 {
@@ -33,6 +33,16 @@ struct Service
 
 Vector<Service> g_services;
 
+static void do_log(const char* format, ...)
+{
+    va_list ap;
+    va_start(ap, format);
+
+    if (g_init_log) vfprintf(g_init_log, format, ap);
+
+    va_end(ap);
+}
+
 static Result<void> service_child(const Service& service, SharedPtr<os::File> output, SharedPtr<os::File> error,
                                   SharedPtr<os::File> input)
 {
@@ -82,23 +92,23 @@ static Result<void> try_start_service(Service& service)
         auto rc = service_child(service, new_stdout, new_stderr, new_stdin);
         if (rc.has_error())
         {
-            fprintf(g_init_log, "[child %d] failed to start service %s due to error: %s\n", getpid(),
-                    service.name.chars(), rc.error_string());
+            do_log("[child %d] failed to start service %s due to error: %s\n", getpid(), service.name.chars(),
+                   rc.error_string());
         }
         fclose(g_init_log);
         exit(127);
     }
 
-    fprintf(g_init_log, "[init] created new child process %d for service %s\n", pid, service.name.chars());
+    do_log("[init] created new child process %d for service %s\n", pid, service.name.chars());
 
     if (service.wait)
     {
-        fprintf(g_init_log, "[init] waiting for child process %d to finish\n", pid);
+        do_log("[init] waiting for child process %d to finish\n", pid);
 
         int status;
         TRY(os::Process::wait(pid, &status));
 
-        fprintf(g_init_log, "[init] child process %d exited with code %d\n", pid, WEXITSTATUS(status));
+        do_log("[init] child process %d exited with code %d\n", pid, WEXITSTATUS(status));
     }
     else
         service.pid = pid;
@@ -111,14 +121,13 @@ static void start_service(Service& service)
     auto rc = try_start_service(service);
     if (rc.has_error())
     {
-        fprintf(g_init_log, "[init] failed to start service %s due to error: %s\n", service.name.chars(),
-                rc.error_string());
+        do_log("[init] failed to start service %s due to error: %s\n", service.name.chars(), rc.error_string());
     }
 }
 
 static Result<void> load_service(const os::Path& path)
 {
-    fprintf(g_init_log, "[init] reading service file: %s\n", path.name().chars());
+    do_log("[init] reading service file: %s\n", path.name().chars());
 
     auto file = TRY(os::File::open(path, os::File::ReadOnly));
 
@@ -135,7 +144,7 @@ static Result<void> load_service(const os::Path& path)
         auto parts = TRY(line.split_once('='));
         if (parts.size() < 2 || parts[0].is_empty() || parts[1].is_empty())
         {
-            fprintf(g_init_log, "[init] file contains invalid line, aborting: '%s'\n", line.chars());
+            do_log("[init] file contains invalid line, aborting: '%s'\n", line.chars());
             return {};
         }
 
@@ -149,8 +158,8 @@ static Result<void> load_service(const os::Path& path)
         {
             if (!service.command.is_empty())
             {
-                fprintf(g_init_log, "[init] 'Command' cannot be specified after 'Script' has already been set! (%s)\n",
-                        line.chars());
+                do_log("[init] 'Command' cannot be specified after 'Script' has already been set! (%s)\n",
+                       line.chars());
                 return {};
             }
             service.command = move(parts[1]);
@@ -161,8 +170,8 @@ static Result<void> load_service(const os::Path& path)
         {
             if (!service.command.is_empty())
             {
-                fprintf(g_init_log, "[init] 'Script' cannot be specified after 'Command' has already been set! (%s)\n",
-                        line.chars());
+                do_log("[init] 'Script' cannot be specified after 'Command' has already been set! (%s)\n",
+                       line.chars());
                 return {};
             }
             service.command = TRY(String::format("/bin/sh -- %s"_sv, parts[1].chars()));
@@ -215,22 +224,22 @@ static Result<void> load_service(const os::Path& path)
             continue;
         }
 
-        fprintf(g_init_log, "[init] skipping unknown entry name %s\n", parts[0].chars());
+        do_log("[init] skipping unknown entry name %s\n", parts[0].chars());
     }
 
     if (service.name.is_empty())
     {
-        fprintf(g_init_log, "[init] service file is missing 'Name' entry, aborting!\n");
+        do_log("[init] service file is missing 'Name' entry, aborting!\n");
         return {};
     }
 
     if (service.command.is_empty())
     {
-        fprintf(g_init_log, "[init] service file is missing 'Command' or 'Script' entry, aborting!\n");
+        do_log("[init] service file is missing 'Command' or 'Script' entry, aborting!\n");
         return {};
     }
 
-    fprintf(g_init_log, "[init] loaded service %s into memory\n", service.name.chars());
+    do_log("[init] loaded service %s into memory\n", service.name.chars());
 
     TRY(g_services.try_append(move(service)));
 
@@ -254,7 +263,7 @@ static Result<void> start_services()
     TRY(load_services());
     for (auto& service : g_services)
     {
-        fprintf(g_init_log, "[init] starting service %s\n", service.name.chars());
+        do_log("[init] starting service %s\n", service.name.chars());
         start_service(service);
     }
 
@@ -270,16 +279,16 @@ static Result<void> set_hostname()
 
     if (sethostname(hostname.chars(), hostname.length()) < 0) return {};
 
-    fprintf(g_init_log, "[init] successfully set system hostname to '%s'\n", hostname.chars());
+    do_log("[init] successfully set system hostname to '%s'\n", hostname.chars());
 
     return {};
 }
 
-static void mount_devfs()
+static void mount_tmpfs()
 {
-    if (mkdir("/dev", 0755) < 0 && errno != EEXIST) exit(255);
+    if (mount("/tmp", "tmpfs", "tmpfs") < 0) exit(255);
 
-    if (mount("/dev", "devfs", "devfs") < 0) exit(255);
+    if (chmod("/tmp", 01777) < 0) exit(255);
 }
 
 int main()
@@ -290,17 +299,17 @@ int main()
         return 1;
     }
 
-    mount_devfs();
-
     // Before this point, we don't even have an stdin, stdout and stderr. Set it up now so that child processes (and us)
     // can print stuff.
     stdin = fopen("/dev/console", "r");
     stdout = fopen("/dev/console", "w");
     stderr = fopen("/dev/console", "w");
 
+    mount_tmpfs();
+
     umask(022);
 
-    g_init_log = fopen("/init.log", "w+");
+    g_init_log = fopen("/tmp/init.log", "w+");
     fcntl(fileno(g_init_log), F_SETFD, FD_CLOEXEC);
 
     set_hostname();
@@ -319,12 +328,11 @@ int main()
         {
             if (service.pid.has_value() && service.pid.value() == child)
             {
-                fprintf(g_init_log, "[init] service %s exited with status %d\n", service.name.chars(),
-                        WEXITSTATUS(status));
+                do_log("[init] service %s exited with status %d\n", service.name.chars(), WEXITSTATUS(status));
 
                 if (service.restart)
                 {
-                    fprintf(g_init_log, "[init] restarting service %s\n", service.name.chars());
+                    do_log("[init] restarting service %s\n", service.name.chars());
 
                     start_service(service);
                 }

apps/preinit.cpp

@@ -0,0 +1,86 @@
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+// FIXME: Make this configurable.
+#define LUNA_ROOT_PARTITION "/dev/cd0p2"
+
+static void mount_devfs()
+{
+    if (mkdir("/dev", 0755) < 0 && errno != EEXIST && errno != EROFS) exit(255);
+
+    if (mount("/dev", "devfs", "devfs") < 0) exit(255);
+}
+
+static void open_std_streams()
+{
+    stdin = fopen("/dev/console", "r");
+    stdout = fopen("/dev/console", "w");
+    stderr = fopen("/dev/console", "w");
+}
+
+static void fail(const char* message)
+{
+    open_std_streams();
+    fprintf(stderr, "preinit: fatal error: %s\n", message);
+    exit(255);
+}
+
+static void fail_errno(const char* message)
+{
+    int err = errno;
+    open_std_streams();
+    fprintf(stderr, "preinit: fatal error: %s (%s)\n", message, strerror(err));
+    exit(255);
+}
+
+static void mount_rootfs()
+{
+    if (mkdir("/osroot", 0755) < 0 && errno != EEXIST) exit(255);
+
+    if (mount("/osroot", "ext2", LUNA_ROOT_PARTITION) < 0) fail_errno("Cannot mount the root partition");
+}
+
+int main()
+{
+    if (getpid() != 1)
+    {
+        fprintf(stderr, "error: preinit must be run as the init process.\n");
+        return 1;
+    }
+
+    mount_devfs();
+    mount_rootfs();
+
+    struct stat st;
+    if (stat("/osroot/mnt", &st) < 0 || !S_ISDIR(st.st_mode)) fail("No suitable temporary mountpoint for pivot_root");
+
+    // Now that we have mounted the root file system, remove the /dev mount on the current ramfs root.
+    umount("/dev");
+
+    long rc = syscall(SYS_pivot_root, "/osroot", "/osroot/mnt");
+    if (rc < 0) exit(255);
+
+    chdir("/");
+    umount("/mnt");
+
+    // Now, mount the /dev file system on the new root.
+    mount_devfs();
+
+    /*setenv("PATH", "/bin:/usr/bin", 1);
+    char* argv[] = { "init", nullptr };
+    char* envp[] = { nullptr };
+    execvpe(argv[0], argv, envp);*/
+
+    char* argv[] = { "/usr/bin/init", nullptr };
+    execv(argv[0], argv);
+
+    fail_errno("Failed to execute init");
+
+    return 255;
+}

base/bin

@@ -0,0 +1 @@
+usr/bin

base/etc/init/00-motd

@@ -0,0 +1,3 @@
+Name=motd
+Command=/usr/bin/cat /etc/motd
+Wait=true

base/etc/init/01-selene

@@ -0,0 +1,3 @@
+Name=selene
+Script=/etc/startup/selene-home.sh
+Wait=true

base/etc/init/99-login

@@ -1,3 +1,3 @@
 Name=login
-Command=/bin/login
+Command=/usr/bin/login
 Restart=true

base/etc/startup/selene-home.sh

@@ -0,0 +1,2 @@
+mount -t tmpfs tmpfs /home/selene
+chown selene:selene /home/selene

initrd/etc/init/00-tmpfs

@@ -1,3 +0,0 @@
-Name=tmpfs
-Script=/sbin/mount-tmpfs
-Wait=true

initrd/etc/init/01-motd

@@ -1,3 +0,0 @@
-Name=motd
-Command=/bin/cat /etc/motd
-Wait=true

initrd/etc/init/02-ext2fs

@@ -1,3 +0,0 @@
-Name=ext2fs
-Script=/sbin/mount-ext2fs
-Wait=true

initrd/sbin/mount-tmpfs

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-mkdir -p /tmp
-mount -t tmpfs tmpfs /tmp
-chmod 1777 /tmp

kernel/CMakeLists.txt

@@ -105,7 +105,8 @@ if(MOON_DEBUG)
     include(debug.cmake)
 endif()
 
-if(EXISTS config.cmake)
+if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/config.cmake)
+    message(STATUS "Using custom config.cmake file")
     include(config.cmake)
 endif()
 

kernel/debug.cmake

@@ -10,4 +10,5 @@ target_compile_definitions(moon PRIVATE PCI_DEBUG)
 target_compile_definitions(moon PRIVATE EXT2_DEBUG)
 target_compile_definitions(moon PRIVATE DEVICE_REGISTRY_DEBUG)
 target_compile_definitions(moon PRIVATE FORK_DEBUG)
+target_compile_definitions(moon PRIVATE MOUNT_DEBUG)
 target_compile_options(moon PRIVATE -fsanitize=undefined)

kernel/src/arch/x86_64/disk/ATA.cpp

@@ -65,9 +65,12 @@ namespace ATA
 
         if (command_new != command_old) PCI::write16(m_device.address, PCI::Command, command_new);
 
-        if (!m_primary_channel.initialize()) return false;
+        bool success = false;
 
-        return m_secondary_channel.initialize();
+        if (m_primary_channel.initialize()) success = true;
+        if (m_secondary_channel.initialize()) success = true;
+
+        return success;
     }
 
     void Controller::irq_handler(Registers* regs)

kernel/src/fs/VFS.cpp

@@ -233,7 +233,9 @@ namespace VFS
         auto parent_path = TRY(PathParser::dirname(path));
         auto child = TRY(PathParser::basename(path));
 
-        kinfoln("vfs: Mounting filesystem on target %s", path);
+#ifdef MOUNT_DEBUG
+        kdbgln("vfs: Mounting filesystem on target %s", path);
+#endif
 
         auto parent_inode = TRY(resolve_path(parent_path.chars(), auth, working_directory));
 
@@ -245,6 +247,8 @@ namespace VFS
 
         TRY(parent_inode->replace_entry(mount, child.chars()));
 
+        kinfoln("vfs: Successfully mounted filesystem on target %s", path);
+
         return {};
     }
 

kernel/src/main.cpp

@@ -45,9 +45,10 @@ void reap_thread()
     mark_critical(InitRD::populate_vfs(), "Failed to load files from the initial ramdisk");
     mark_critical(DeviceRegistry::init(), "Failed to register initial devices");
 
-    auto init = mark_critical(VFS::resolve_path("/bin/init", Credentials {}), "Can't find init in the initial ramfs!");
+    auto init =
+        mark_critical(VFS::resolve_path("/bin/preinit", Credentials {}), "Can't find init in the initial ramfs!");
     auto init_thread =
-        mark_critical(Scheduler::new_userspace_thread(init, "/bin/init"), "Failed to create PID 1 process for init");
+        mark_critical(Scheduler::new_userspace_thread(init, "/bin/preinit"), "Failed to create PID 1 process for init");
 
     auto reap = mark_critical(Scheduler::new_kernel_thread(reap_thread, "[reap]"),
                               "Failed to create the process reaper kernel thread");

kernel/src/sys/uname.cpp

@@ -1,3 +1,4 @@
+#include "Log.h"
 #include "arch/CPU.h"
 #include "config.h"
 #include "memory/MemoryManager.h"
@@ -51,5 +52,7 @@ Result<u64> sys_sethostname(Registers*, SyscallArgs args)
 
     s_hostname.adopt(new_hostname);
 
+    kinfoln("System hostname updated to '%s'", s_hostname.chars());
+
     return 0;
 }

kernel/src/thread/Scheduler.cpp

@@ -27,6 +27,7 @@ namespace Scheduler
         g_idle.is_kernel = true;
         g_idle.parent = nullptr;
         g_idle.name = "[idle]";
+        g_idle.active_directory = nullptr;
 
         g_idle.ticks_left = 1;
 
@@ -86,6 +87,7 @@ namespace Scheduler
         thread->name = name;
 
         thread->is_kernel = true;
+        thread->active_directory = MMU::kernel_page_directory();
 
         thread->auth = Credentials { .uid = 0, .euid = 0, .suid = 0, .gid = 0, .egid = 0, .sgid = 0 };
 
@@ -233,7 +235,7 @@ namespace Scheduler
         {
             switch_context(old_thread, new_thread, regs);
             if (!old_thread->is_kernel) old_thread->fp_data.save();
-            if (MMU::get_page_directory() != MMU::kernel_page_directory())
+            if (old_thread->state != ThreadState::Idle && MMU::get_page_directory() != MMU::kernel_page_directory())
                 old_thread->active_directory = MMU::get_page_directory();
             if (new_thread->active_directory) MMU::switch_page_directory(new_thread->active_directory);
             if (!new_thread->is_kernel)

kernel/src/thread/Thread.cpp

@@ -71,6 +71,7 @@ Result<SharedPtr<VFS::Inode>> Thread::resolve_atfile(int dirfd, const String& pa
 
 [[noreturn]] void Thread::exit_and_signal_parent(u8 _status)
 {
+    if (this->id == 1) fail("the init process exited");
     if (is_kernel) state = ThreadState::Dying;
     else
     {

luna.json

@@ -1,6 +1,6 @@
 {
     "diskguid": "00000000-0000-0000-0000-000000000000",
-    "disksize": 20,
+    "disksize": 32,
     "align": 1024,
     "iso9660": true,
     "config": "initrd/sys/config",
@@ -12,7 +12,7 @@
     "partitions": [
         {
             "type": "boot",
-            "size": 16
+            "size": 8
         },
         {
             "type": "ext2",

tests/CMakeLists.txt

@@ -9,7 +9,7 @@ function(luna_test SOURCE_FILE APP_NAME)
     add_dependencies(${APP_NAME} libc)
     target_include_directories(${APP_NAME} PRIVATE ${LUNA_BASE}/usr/include)
     target_link_libraries(${APP_NAME} PRIVATE test os)
-    install(TARGETS ${APP_NAME} DESTINATION ${LUNA_ROOT}/initrd/bin/tests)
+    install(TARGETS ${APP_NAME} DESTINATION ${LUNA_BASE}/usr/bin/tests)
 endfunction()
 
 if(BUILD_TESTS)

tools/install.sh

@@ -17,14 +17,17 @@ then
     exit 1
 fi
 
-chown -R root:root initrd
+chown -R root:root base
 
 cmake --install $LUNA_BUILD_DIR
 
-chmod 400 initrd/boot/moon
-chmod a+s initrd/bin/su
+chmod a+s base/usr/bin/su
 
-mkdir -p initrd/home/selene
-chown 1000:1000 initrd/home/selene
+mkdir -p base/home/selene
 
-mkdir -p initrd/dev
+mkdir -p base/dev
+mkdir -p base/mnt
+mkdir -p base/tmp
+
+rm -f base/bin
+ln -s usr/bin base/bin

tools/make-iso.sh

@@ -7,6 +7,6 @@ cd $LUNA_ROOT
 
 fakeroot -u -s $LUNA_ROOT/.fakeroot -- tools/install.sh
 
-genext2fs -d base -B 4096 -b 1024 -L luna-rootfs -U -N 1024 build/ext2fs.bin
+fakeroot -u -i $LUNA_ROOT/.fakeroot -- genext2fs -d base -B 4096 -b 1024 -L luna-rootfs -N 1024 build/ext2fs.bin
 
-fakeroot -u -i $LUNA_ROOT/.fakeroot mkbootimg luna.json Luna.iso
+mkbootimg luna.json Luna.iso