apps: Add free

We can finally show memory usage in userspace.

This could have been done using sysfs, but I'm lazy and don't want to implement that. Maybe in the next release?

apps/CMakeLists.txt

@@ -47,3 +47,4 @@ luna_app(socket-client.cpp socket-client)
 luna_app(input.cpp input)
 luna_app(shmem-test.cpp shmem-test)
 luna_app(touch.cpp touch)
+luna_app(free.cpp free)

apps/free.cpp

@@ -0,0 +1,50 @@
+#include <luna/Units.h>
+#include <os/ArgumentParser.h>
+#include <os/File.h>
+#include <sys/memstat.h>
+
+Result<int> luna_main(int argc, char** argv)
+{
+    bool reserved { false };
+    bool human_readable { false };
+
+    os::ArgumentParser parser;
+    parser.add_description("Query the amount of memory available and used in the system.");
+    parser.add_system_program_info("free"_sv);
+    parser.add_switch_argument(reserved, 'r', "show-reserved"_sv, "show reserved system memory"_sv);
+    parser.add_switch_argument(human_readable, 'h', "human-readable"_sv, "show output in human-readable units"_sv);
+    parser.parse(argc, argv);
+
+    struct membuf buf;
+    memstat(&buf);
+
+    if (human_readable)
+    {
+        auto total = TRY(to_dynamic_unit(buf.mem_total, 10, false, Unit::Binary, true));
+        auto free = TRY(to_dynamic_unit(buf.mem_free, 10, false, Unit::Binary, true));
+        auto used = TRY(to_dynamic_unit(buf.mem_used, 10, false, Unit::Binary, true));
+
+        os::println("%s total memory, out of which:", total.chars());
+        os::println("%s used", used.chars());
+        os::println("%s free", free.chars());
+    }
+    else
+    {
+        os::println("%lu total memory, out of which:", buf.mem_total);
+        os::println("%lu used", buf.mem_used);
+        os::println("%lu free", buf.mem_free);
+    }
+
+    if (reserved)
+    {
+        if (human_readable)
+        {
+            auto reserved_string = TRY(to_dynamic_unit(buf.mem_reserved, 10, false, Unit::Binary, true));
+
+            os::println("%s reserved", reserved_string.chars());
+        }
+        else { os::println("%lu reserved", buf.mem_reserved); }
+    }
+
+    return 0;
+}

kernel/CMakeLists.txt

@@ -47,6 +47,7 @@ set(SOURCES
     src/sys/poll.cpp
     src/sys/alarm.cpp
     src/sys/pledge.cpp
+    src/sys/memstat.cpp
     src/fs/VFS.cpp
     src/fs/Pipe.cpp
     src/fs/Mount.cpp

kernel/src/fs/StorageCache.cpp

@@ -1,5 +1,6 @@
 #include "fs/StorageCache.h"
 #include "Log.h"
+#include <luna/Heap.h>
 #include <luna/ScopeGuard.h>
 
 static LinkedList<StorageCache> g_storage_caches;
@@ -23,9 +24,13 @@ Result<StorageCache::CacheEntry*> StorageCache::fetch_entry(u64 block)
 
 void StorageCache::clear()
 {
-    ScopedKMutexLock<100> lock(m_mutex);
+    m_mutex.lock();
 
+    kdbgln("cache: clearing %lu entries, out of %lu buckets", m_cache_entries.size(), m_cache_entries.capacity());
     m_cache_entries.clear();
+    kdbgln("cache: done");
+
+    m_mutex.unlock();
 }
 
 StorageCache::StorageCache()

kernel/src/main.cpp

@@ -31,6 +31,16 @@ void reap_thread()
     }
 }
 
+void oom_thread()
+{
+    while (true)
+    {
+        kernel_wait_for_event();
+        // OOM! Do everything we can to recover memory.
+        StorageCache::clear_caches();
+    }
+}
+
 [[noreturn]] void init()
 {
     {
@@ -58,6 +68,10 @@ void reap_thread()
                                   "Failed to create the process reaper kernel thread");
         Scheduler::set_reap_thread(reap);
 
+        auto oom = mark_critical(Scheduler::new_kernel_thread(oom_thread, "[oom]"),
+                                 "Failed to create the out-of-memory kernel thread");
+        Scheduler::set_oom_thread(oom);
+
 #ifdef ARCH_X86_64
         ATA::Controller::scan();
 #endif

kernel/src/memory/MemoryManager.cpp

@@ -146,9 +146,9 @@ namespace MemoryManager
         bool ok = frame_bitmap->find_and_toggle(false, start_index).try_set_value(index);
         if (!ok)
         {
-            kwarnln("OOM alert! Trying to free caches...");
+            kwarnln("OOM alert! Scheduling the OOM thread...");
-            StorageCache::clear_caches();
+            Scheduler::signal_oom_thread();
-            if (!frame_bitmap->find_and_toggle(false, start_index).try_set_value(index)) return err(ENOMEM);
+            return err(ENOMEM);
         }
 
         start_index = index + 1;

kernel/src/sys/memstat.cpp

@@ -0,0 +1,21 @@
+#include "Pledge.h"
+#include "memory/MemoryManager.h"
+#include "sys/Syscall.h"
+#include "thread/Scheduler.h"
+#include <bits/memstat.h>
+
+Result<u64> sys_memstat(Registers*, SyscallArgs args)
+{
+    auto* current = Scheduler::current();
+    TRY(check_pledge(current, Promise::p_stdio));
+
+    struct membuf buf;
+    buf.mem_total = MemoryManager::total();
+    buf.mem_used = MemoryManager::used();
+    buf.mem_free = MemoryManager::free();
+    buf.mem_reserved = MemoryManager::reserved();
+
+    if (!MemoryManager::copy_to_user_typed((struct membuf*)args[0], &buf)) return err(EFAULT);
+
+    return 0;
+}

kernel/src/thread/Scheduler.cpp

@@ -13,6 +13,7 @@ static Thread g_idle;
 static Thread* g_current = nullptr;
 static Thread* g_init = nullptr;
 static Thread* g_reap = nullptr;
+static Thread* g_oom = nullptr;
 
 static const usize TICKS_PER_TIMESLICE = 20;
 
@@ -70,6 +71,17 @@ namespace Scheduler
         if (g_reap) g_reap->wake_up();
     }
 
+    void set_oom_thread(Thread* thread)
+    {
+        g_oom = thread;
+        g_oom->unrestricted_task = true;
+    }
+
+    void signal_oom_thread()
+    {
+        if (g_oom) g_oom->wake_up();
+    }
+
     Result<Thread*> new_kernel_thread_impl(Thread* thread, const char* name)
     {
         // If anything fails, make sure to clean up.
@@ -154,8 +166,13 @@ namespace Scheduler
         u64 argv = TRY(image->push_string_vector_on_stack(args));
         u64 envp = TRY(image->push_string_vector_on_stack(env));
 
+        const u64 kernel_stack_base = TRY(MemoryManager::alloc_for_kernel(4, MMU::ReadWrite | MMU::NoExecute));
+        Stack kernel_stack { kernel_stack_base, 4 * ARCH_PAGE_SIZE };
+
         guard.deactivate();
 
+        thread->kernel_stack = kernel_stack;
+
         image->apply(thread);
         thread->set_arguments(args.size(), argv, env.size(), envp);
 
@@ -254,8 +271,12 @@ namespace Scheduler
             new_thread->ticks_left = 1; // The idle task only runs for 1 tick so we can check for new runnable tasks
                                         // as fast as possible.
         }
-        else
+        else if (new_thread->unrestricted_task)
-            new_thread->ticks_left = TICKS_PER_TIMESLICE;
+        {
+            check(new_thread->is_kernel);
+            new_thread->ticks_left = -1;
+        }
+        else { new_thread->ticks_left = TICKS_PER_TIMESLICE; }
     }
 
     void switch_task(Registers* regs)

kernel/src/thread/Scheduler.h

@@ -13,6 +13,9 @@ namespace Scheduler
     void set_reap_thread(Thread*);
     void signal_reap_thread();
 
+    void set_oom_thread(Thread*);
+    void signal_oom_thread();
+
     Result<Thread*> new_kernel_thread(u64 address, const char* name);
     Result<Thread*> new_kernel_thread(void (*func)(void), const char* name);
     Result<Thread*> new_kernel_thread(void (*func)(void*), void* arg, const char* name);

kernel/src/thread/Thread.h

@@ -109,6 +109,8 @@ struct Thread : public LinkedListNode<Thread>
     sigset_t pending_signals { 0 };
     bool interrupted { false };
 
+    bool unrestricted_task { false };
+
     FPData fp_data;
 
     ThreadState state = ThreadState::Runnable;

libc/CMakeLists.txt

@@ -33,6 +33,7 @@ set(SOURCES
     src/sys/utsname.cpp
     src/sys/mount.cpp
     src/sys/pstat.cpp
+    src/sys/memstat.cpp
     src/sys/resource.cpp
     src/sys/socket.cpp
     src/sys/poll.cpp

libc/include/bits/memstat.h

@@ -0,0 +1,14 @@
+/* bits/memstat.h: The membuf structure for memstat(). */
+
+#ifndef _BITS_MEMSTAT_H
+#define _BITS_MEMSTAT_H
+
+struct membuf
+{
+    unsigned long mem_total;
+    unsigned long mem_used;
+    unsigned long mem_free;
+    unsigned long mem_reserved;
+};
+
+#endif

libc/include/sys/memstat.h

@@ -0,0 +1,20 @@
+/* sys/memstat.h: Memory usage querying. */
+
+#ifndef _SYS_MEMSTAT_H
+#define _SYS_MEMSTAT_H
+
+#include <bits/memstat.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    /* Query system memory usage. */
+    int memstat(struct membuf* mem);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

libc/src/sys/memstat.cpp

@@ -0,0 +1,13 @@
+#include <bits/errno-return.h>
+#include <sys/memstat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+extern "C"
+{
+    int memstat(struct membuf* buf)
+    {
+        long rc = syscall(SYS_memstat, buf);
+        __errno_return(rc, int);
+    }
+}

libluna/include/luna/HashMap.h

@@ -49,6 +49,16 @@ template <typename K, typename V> struct HashMap
         return m_table.try_remove(HashPair<K, V> { key, {} });
     }
 
+    usize capacity() const
+    {
+        return m_table.capacity();
+    }
+
+    usize size() const
+    {
+        return m_table.size();
+    }
+
     void clear()
     {
         m_table.clear();

libluna/include/luna/HashTable.h

@@ -100,12 +100,25 @@ template <typename T> class HashTable
     }
 
     void clear()
+    {
+        if (m_capacity)
         {
             for (usize i = 0; i < m_capacity; i++) m_buckets[i].~Option();
 
             free_impl(m_buckets);
             m_capacity = m_size = 0;
         }
+    }
+
+    usize capacity() const
+    {
+        return m_capacity;
+    }
+
+    usize size() const
+    {
+        return m_size;
+    }
 
     ~HashTable()
     {

libluna/include/luna/Syscall.h

@@ -8,7 +8,7 @@
                     _e(umount) _e(pstat) _e(getrusage) _e(symlinkat) _e(readlinkat) _e(umask) _e(linkat) _e(faccessat) \
                         _e(pivot_root) _e(sigreturn) _e(sigaction) _e(kill) _e(sigprocmask) _e(setpgid) _e(isatty)     \
                             _e(getpgid) _e(socket) _e(bind) _e(connect) _e(listen) _e(accept) _e(poll) _e(msync)       \
-                                _e(truncate) _e(ftruncate) _e(utimensat) _e(alarm) _e(pledge)
+                                _e(truncate) _e(ftruncate) _e(utimensat) _e(alarm) _e(pledge) _e(memstat)
 
 enum Syscalls
 {