kernel: Improve Your Disk IO performance by 500% with this One Trick!

The trick being caching lol.

kernel/CMakeLists.txt

@@ -52,12 +52,14 @@ set(SOURCES
     src/fs/Mount.cpp
     src/fs/MBR.cpp
     src/fs/GPT.cpp
+    src/fs/StorageCache.cpp
     src/net/UnixSocket.cpp
     src/fs/tmpfs/FileSystem.cpp
     src/fs/tmpfs/Inode.cpp
     src/fs/ext2/FileSystem.cpp
     src/fs/ext2/Inode.cpp
     src/fs/devices/DeviceRegistry.cpp
+    src/fs/devices/BlockDevice.cpp
     src/fs/devices/NullDevice.cpp
     src/fs/devices/ZeroDevice.cpp
     src/fs/devices/FullDevice.cpp

kernel/debug.cmake

@@ -11,4 +11,5 @@ target_compile_definitions(moon PRIVATE EXT2_DEBUG)
 target_compile_definitions(moon PRIVATE DEVICE_REGISTRY_DEBUG)
 target_compile_definitions(moon PRIVATE FORK_DEBUG)
 target_compile_definitions(moon PRIVATE MOUNT_DEBUG)
+target_compile_definitions(moon PRIVATE CACHE_DEBUG)
 target_compile_options(moon PRIVATE -fsanitize=undefined)

kernel/src/arch/x86_64/disk/ATA.cpp

@@ -739,66 +739,26 @@ Result<String> ATA::Drive::create_drive_name(ATA::Drive* drive)
 
 Result<SharedPtr<Device>> ATADevice::create(ATA::Drive* drive)
 {
-    auto device = TRY(adopt_shared_if_nonnull(new (std::nothrow) ATADevice()));
-    device->m_drive = drive;
+    auto device = TRY(adopt_shared_if_nonnull(new (std::nothrow) ATADevice(drive)));
     device->m_device_path = TRY(ATA::Drive::create_drive_name(drive));
     TRY(DeviceRegistry::register_special_device(DeviceRegistry::Disk, next_minor++, device, 0400));
     return (SharedPtr<Device>)device;
 }
 
-Result<u64> ATADevice::read(u8* buf, usize offset, usize size) const
+ATADevice::ATADevice(ATA::Drive* drive) : BlockDevice(drive->block_size(), drive->block_count()), m_drive(drive)
 {
-    if (size == 0) return 0;
-
-    if (offset > m_drive->capacity()) return 0;
-    if (offset + size > m_drive->capacity()) size = m_drive->capacity() - offset;
-
-    usize length = size;
-
-    auto block_size = m_drive->block_size();
+}
 
+Result<void> ATADevice::read_block(Buffer& buf, u64 block) const
+{
     ScopedKMutexLock<100> lock(m_drive->channel()->lock());
 
-    Buffer temp;
-
-    if (offset % block_size)
+    if (buf.size() != m_drive->block_size())
     {
-        // The size we need to read to round up to a block.
-        usize extra_size = block_size - (offset % block_size);
-        // Maybe we don't even want enough to get to the next block?
-        if (extra_size > size) extra_size = size;
-
-        TRY(temp.try_resize(block_size));
-
-        TRY(m_drive->read_lba(offset / block_size, temp.data(), 1));
-        memcpy(buf, temp.data() + (offset % block_size), extra_size);
-        offset += extra_size;
-        size -= extra_size;
-        buf += extra_size;
+        kwarnln("ata: error while reading block %lu: cache entry size mismatch (%lu), data=%p", block, buf.size(),
+                buf.data());
+        fail("Cache entry size mismatch");
     }
 
-    while (size >= ARCH_PAGE_SIZE)
-    {
-        TRY(m_drive->read_lba(offset / block_size, buf, ARCH_PAGE_SIZE / block_size));
-        offset += ARCH_PAGE_SIZE;
-        size -= ARCH_PAGE_SIZE;
-        buf += ARCH_PAGE_SIZE;
-    }
-
-    while (size >= block_size)
-    {
-        TRY(m_drive->read_lba(offset / block_size, buf, 1));
-        offset += block_size;
-        size -= block_size;
-        buf += block_size;
-    }
-
-    if (size)
-    {
-        TRY(temp.try_resize(block_size));
-        TRY(m_drive->read_lba(offset / block_size, temp.data(), 1));
-        memcpy(buf, temp.data(), size);
-    }
-
-    return length;
+    return m_drive->read_lba(block, buf.data(), 1);
 }

kernel/src/arch/x86_64/disk/ATA.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "arch/PCI.h"
+#include "fs/devices/BlockDevice.h"
 #include "fs/devices/DeviceRegistry.h"
 #include "lib/KMutex.h"
 #include <luna/Atomic.h>
@@ -256,7 +257,6 @@ namespace ATA
         Controller* m_controller;
         u8 m_channel_index;
         bool m_is_pci_native_mode;
-
         u8 m_interrupt_line;
 
         KMutex<100> m_lock {};
@@ -297,44 +297,25 @@ namespace ATA
 
 }
 
-class ATADevice : public Device
+class ATADevice : public BlockDevice
 {
   public:
     // Initializer for DeviceRegistry.
     static Result<SharedPtr<Device>> create(ATA::Drive* drive);
 
-    Result<usize> read(u8*, usize, usize) const override;
-
-    Result<usize> write(const u8*, usize, usize) override
-    {
-        return err(ENOTSUP);
-    }
+    Result<void> read_block(Buffer& buf, u64 block) const override;
 
     bool will_block_if_read() const override
     {
         return false;
     }
 
-    bool is_block_device() const override
-    {
-        return true;
-    }
-
-    usize size() const override
-    {
-        return m_drive->capacity();
-    }
-
-    Result<usize> block_size() const override
-    {
-        return m_drive->block_size();
-    }
-
     StringView device_path() const override
     {
         return m_device_path.view();
     }
 
+    ATADevice(ATA::Drive* drive);
     virtual ~ATADevice() = default;
 
   private:

kernel/src/fs/StorageCache.cpp

@@ -0,0 +1,44 @@
+#include "fs/StorageCache.h"
+#include "Log.h"
+#include <luna/ScopeGuard.h>
+
+static LinkedList<StorageCache> g_storage_caches;
+
+Result<StorageCache::CacheEntry*> StorageCache::fetch_entry(u64 block)
+{
+    {
+        CacheEntry* entry = m_cache_entries.try_get_ref(block);
+        if (entry && !entry->buffer.is_empty()) return entry;
+    }
+
+    CacheEntry entry {};
+    TRY(m_cache_entries.try_set(block, move(entry)));
+
+#ifdef CACHE_DEBUG
+    kdbgln("cache: Created new cache entry for block %lu", block);
+#endif
+
+    return m_cache_entries.try_get_ref(block);
+}
+
+void StorageCache::clear()
+{
+    ScopedKMutexLock<100> lock(m_mutex);
+
+    m_cache_entries.clear();
+}
+
+StorageCache::StorageCache()
+{
+    g_storage_caches.append(this);
+}
+
+StorageCache::~StorageCache()
+{
+    g_storage_caches.remove(this);
+}
+
+void StorageCache::clear_caches()
+{
+    for (auto* cache : g_storage_caches) { cache->clear(); }
+}

kernel/src/fs/StorageCache.h

@@ -0,0 +1,37 @@
+#pragma once
+#include "lib/KMutex.h"
+#include <luna/Buffer.h>
+#include <luna/HashMap.h>
+#include <luna/LinkedList.h>
+
+class StorageCache : public LinkedListNode<StorageCache>
+{
+  public:
+    struct CacheEntry
+    {
+        Buffer buffer {};
+    };
+
+    void lock()
+    {
+        return m_mutex.lock();
+    }
+
+    void unlock()
+    {
+        return m_mutex.unlock();
+    }
+
+    Result<CacheEntry*> fetch_entry(u64 block);
+
+    void clear();
+
+    static void clear_caches();
+
+    StorageCache();
+    ~StorageCache();
+
+  private:
+    HashMap<u64, CacheEntry> m_cache_entries;
+    KMutex<100> m_mutex;
+};

kernel/src/fs/devices/BlockDevice.cpp

@@ -0,0 +1,51 @@
+#include "fs/devices/BlockDevice.h"
+#include "arch/MMU.h"
+#include <luna/Common.h>
+
+BlockDevice::BlockDevice(u64 block_size, u64 num_blocks) : m_cache(), m_block_size(block_size), m_num_blocks(num_blocks)
+{
+}
+
+Result<usize> BlockDevice::read(u8* buf, usize offset, usize length) const
+{
+    if (length == 0) return 0;
+
+    if (offset > size()) return 0;
+    if (offset + length > size()) length = size() - offset;
+
+    const usize saved = length;
+
+    auto read_data = [&](u64 off, u64 len) -> Result<void> {
+        const u64 block = offset / m_block_size;
+
+        m_cache.lock();
+        auto guard = make_scope_guard([&] { m_cache.unlock(); });
+
+        auto* entry = TRY(m_cache.fetch_entry(block));
+        if (entry->buffer.is_empty())
+        {
+            // TODO: What if the cache needs clearing in the middle of try_resize()? That could lead to some weird
+            // state...
+            TRY(entry->buffer.try_resize(m_block_size));
+            TRY(read_block(entry->buffer, block));
+        }
+        memcpy(buf, entry->buffer.data() + off, len);
+        offset += len;
+        length -= len;
+        buf += len;
+        return {};
+    };
+
+    if (offset % m_block_size)
+    {
+        const u64 off = offset % m_block_size;
+        const u64 len = min(m_block_size - off, length);
+        TRY(read_data(off, len));
+    }
+
+    while (length >= m_block_size) { TRY(read_data(0, m_block_size)); }
+
+    if (length) { TRY(read_data(0, length)); }
+
+    return saved;
+}

kernel/src/fs/devices/BlockDevice.h

@@ -0,0 +1,40 @@
+#pragma once
+#include "fs/StorageCache.h"
+#include "fs/devices/Device.h"
+
+class BlockDevice : public Device
+{
+  public:
+    BlockDevice(u64 block_size, u64 num_blocks);
+
+    Result<usize> read(u8* buf, usize offset, usize length) const override;
+
+    virtual Result<void> read_block(Buffer& buf, u64 block) const = 0;
+
+    Result<usize> write(const u8*, usize, usize) override
+    {
+        return err(ENOTSUP);
+    }
+
+    usize size() const override
+    {
+        return m_block_size * m_num_blocks;
+    }
+
+    bool is_block_device() const override
+    {
+        return true;
+    }
+
+    Result<usize> block_size() const override
+    {
+        return m_block_size;
+    }
+
+    virtual ~BlockDevice() = default;
+
+  protected:
+    mutable StorageCache m_cache;
+    u64 m_block_size;
+    u64 m_num_blocks;
+};

kernel/src/fs/devices/Device.h

@@ -2,6 +2,8 @@
 #include "Log.h"
 #include <luna/Result.h>
 #include <luna/SharedPtr.h>
+#include <luna/StringView.h>
+#include <sys/types.h>
 
 class Device : public Shareable
 {

kernel/src/memory/MemoryManager.cpp

@@ -1,6 +1,7 @@
 #include "memory/MemoryManager.h"
 #include "Log.h"
 #include "arch/MMU.h"
+#include "fs/StorageCache.h"
 #include "memory/KernelVM.h"
 #include "memory/MemoryMap.h"
 #include <luna/Alignment.h>
@@ -143,7 +144,12 @@ namespace MemoryManager
 
         usize index;
         bool ok = frame_bitmap->find_and_toggle(false, start_index).try_set_value(index);
-        if (!ok) return err(ENOMEM);
+        if (!ok)
+        {
+            kwarnln("OOM alert! Trying to free caches...");
+            StorageCache::clear_caches();
+            if (!frame_bitmap->find_and_toggle(false, start_index).try_set_value(index)) return err(ENOMEM);
+        }
 
         start_index = index + 1;
 

libluna/include/luna/HashMap.h

@@ -49,6 +49,11 @@ template <typename K, typename V> struct HashMap
         return m_table.try_remove(HashPair<K, V> { key, {} });
     }
 
+    void clear()
+    {
+        m_table.clear();
+    }
+
   private:
     HashTable<HashPair<K, V>> m_table;
 };