From d43590e68cfa74335ce6f12f3dd3b5a72df696d4 Mon Sep 17 00:00:00 2001 From: apio Date: Wed, 16 Aug 2023 14:54:13 +0200 Subject: [PATCH] kernel: Improve Your Disk IO performance by 500% with this One Trick! The trick being caching lol. --- kernel/CMakeLists.txt | 2 + kernel/debug.cmake | 1 + kernel/src/arch/x86_64/disk/ATA.cpp | 60 +++++---------------------- kernel/src/arch/x86_64/disk/ATA.h | 27 ++---------- kernel/src/fs/StorageCache.cpp | 44 ++++++++++++++++++++ kernel/src/fs/StorageCache.h | 37 +++++++++++++++++ kernel/src/fs/devices/BlockDevice.cpp | 51 +++++++++++++++++++++++ kernel/src/fs/devices/BlockDevice.h | 40 ++++++++++++++++++ kernel/src/fs/devices/Device.h | 2 + kernel/src/memory/MemoryManager.cpp | 8 +++- libluna/include/luna/HashMap.h | 5 +++ 11 files changed, 203 insertions(+), 74 deletions(-) create mode 100644 kernel/src/fs/StorageCache.cpp create mode 100644 kernel/src/fs/StorageCache.h create mode 100644 kernel/src/fs/devices/BlockDevice.cpp create mode 100644 kernel/src/fs/devices/BlockDevice.h diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 1fd13dec..fbf11b11 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -52,12 +52,14 @@ set(SOURCES src/fs/Mount.cpp src/fs/MBR.cpp src/fs/GPT.cpp + src/fs/StorageCache.cpp src/net/UnixSocket.cpp src/fs/tmpfs/FileSystem.cpp src/fs/tmpfs/Inode.cpp src/fs/ext2/FileSystem.cpp src/fs/ext2/Inode.cpp src/fs/devices/DeviceRegistry.cpp + src/fs/devices/BlockDevice.cpp src/fs/devices/NullDevice.cpp src/fs/devices/ZeroDevice.cpp src/fs/devices/FullDevice.cpp diff --git a/kernel/debug.cmake b/kernel/debug.cmake index 10db3d17..c272cd71 100644 --- a/kernel/debug.cmake +++ b/kernel/debug.cmake @@ -11,4 +11,5 @@ target_compile_definitions(moon PRIVATE EXT2_DEBUG) target_compile_definitions(moon PRIVATE DEVICE_REGISTRY_DEBUG) target_compile_definitions(moon PRIVATE FORK_DEBUG) target_compile_definitions(moon PRIVATE MOUNT_DEBUG) +target_compile_definitions(moon PRIVATE CACHE_DEBUG) target_compile_options(moon PRIVATE -fsanitize=undefined) diff --git a/kernel/src/arch/x86_64/disk/ATA.cpp b/kernel/src/arch/x86_64/disk/ATA.cpp index e759f533..4b601c77 100644 --- a/kernel/src/arch/x86_64/disk/ATA.cpp +++ b/kernel/src/arch/x86_64/disk/ATA.cpp @@ -739,66 +739,26 @@ Result ATA::Drive::create_drive_name(ATA::Drive* drive) Result> ATADevice::create(ATA::Drive* drive) { - auto device = TRY(adopt_shared_if_nonnull(new (std::nothrow) ATADevice())); - device->m_drive = drive; + auto device = TRY(adopt_shared_if_nonnull(new (std::nothrow) ATADevice(drive))); device->m_device_path = TRY(ATA::Drive::create_drive_name(drive)); TRY(DeviceRegistry::register_special_device(DeviceRegistry::Disk, next_minor++, device, 0400)); return (SharedPtr)device; } -Result ATADevice::read(u8* buf, usize offset, usize size) const +ATADevice::ATADevice(ATA::Drive* drive) : BlockDevice(drive->block_size(), drive->block_count()), m_drive(drive) { - if (size == 0) return 0; - - if (offset > m_drive->capacity()) return 0; - if (offset + size > m_drive->capacity()) size = m_drive->capacity() - offset; - - usize length = size; - - auto block_size = m_drive->block_size(); +} +Result ATADevice::read_block(Buffer& buf, u64 block) const +{ ScopedKMutexLock<100> lock(m_drive->channel()->lock()); - Buffer temp; - - if (offset % block_size) + if (buf.size() != m_drive->block_size()) { - // The size we need to read to round up to a block. - usize extra_size = block_size - (offset % block_size); - // Maybe we don't even want enough to get to the next block? - if (extra_size > size) extra_size = size; - - TRY(temp.try_resize(block_size)); - - TRY(m_drive->read_lba(offset / block_size, temp.data(), 1)); - memcpy(buf, temp.data() + (offset % block_size), extra_size); - offset += extra_size; - size -= extra_size; - buf += extra_size; + kwarnln("ata: error while reading block %lu: cache entry size mismatch (%lu), data=%p", block, buf.size(), + buf.data()); + fail("Cache entry size mismatch"); } - while (size >= ARCH_PAGE_SIZE) - { - TRY(m_drive->read_lba(offset / block_size, buf, ARCH_PAGE_SIZE / block_size)); - offset += ARCH_PAGE_SIZE; - size -= ARCH_PAGE_SIZE; - buf += ARCH_PAGE_SIZE; - } - - while (size >= block_size) - { - TRY(m_drive->read_lba(offset / block_size, buf, 1)); - offset += block_size; - size -= block_size; - buf += block_size; - } - - if (size) - { - TRY(temp.try_resize(block_size)); - TRY(m_drive->read_lba(offset / block_size, temp.data(), 1)); - memcpy(buf, temp.data(), size); - } - - return length; + return m_drive->read_lba(block, buf.data(), 1); } diff --git a/kernel/src/arch/x86_64/disk/ATA.h b/kernel/src/arch/x86_64/disk/ATA.h index 5d9adc45..9d5e19d7 100644 --- a/kernel/src/arch/x86_64/disk/ATA.h +++ b/kernel/src/arch/x86_64/disk/ATA.h @@ -1,5 +1,6 @@ #pragma once #include "arch/PCI.h" +#include "fs/devices/BlockDevice.h" #include "fs/devices/DeviceRegistry.h" #include "lib/KMutex.h" #include @@ -256,7 +257,6 @@ namespace ATA Controller* m_controller; u8 m_channel_index; bool m_is_pci_native_mode; - u8 m_interrupt_line; KMutex<100> m_lock {}; @@ -297,44 +297,25 @@ namespace ATA } -class ATADevice : public Device +class ATADevice : public BlockDevice { public: // Initializer for DeviceRegistry. static Result> create(ATA::Drive* drive); - Result read(u8*, usize, usize) const override; - - Result write(const u8*, usize, usize) override - { - return err(ENOTSUP); - } + Result read_block(Buffer& buf, u64 block) const override; bool will_block_if_read() const override { return false; } - bool is_block_device() const override - { - return true; - } - - usize size() const override - { - return m_drive->capacity(); - } - - Result block_size() const override - { - return m_drive->block_size(); - } - StringView device_path() const override { return m_device_path.view(); } + ATADevice(ATA::Drive* drive); virtual ~ATADevice() = default; private: diff --git a/kernel/src/fs/StorageCache.cpp b/kernel/src/fs/StorageCache.cpp new file mode 100644 index 00000000..95e0b012 --- /dev/null +++ b/kernel/src/fs/StorageCache.cpp @@ -0,0 +1,44 @@ +#include "fs/StorageCache.h" +#include "Log.h" +#include + +static LinkedList g_storage_caches; + +Result StorageCache::fetch_entry(u64 block) +{ + { + CacheEntry* entry = m_cache_entries.try_get_ref(block); + if (entry && !entry->buffer.is_empty()) return entry; + } + + CacheEntry entry {}; + TRY(m_cache_entries.try_set(block, move(entry))); + +#ifdef CACHE_DEBUG + kdbgln("cache: Created new cache entry for block %lu", block); +#endif + + return m_cache_entries.try_get_ref(block); +} + +void StorageCache::clear() +{ + ScopedKMutexLock<100> lock(m_mutex); + + m_cache_entries.clear(); +} + +StorageCache::StorageCache() +{ + g_storage_caches.append(this); +} + +StorageCache::~StorageCache() +{ + g_storage_caches.remove(this); +} + +void StorageCache::clear_caches() +{ + for (auto* cache : g_storage_caches) { cache->clear(); } +} diff --git a/kernel/src/fs/StorageCache.h b/kernel/src/fs/StorageCache.h new file mode 100644 index 00000000..9767246d --- /dev/null +++ b/kernel/src/fs/StorageCache.h @@ -0,0 +1,37 @@ +#pragma once +#include "lib/KMutex.h" +#include +#include +#include + +class StorageCache : public LinkedListNode +{ + public: + struct CacheEntry + { + Buffer buffer {}; + }; + + void lock() + { + return m_mutex.lock(); + } + + void unlock() + { + return m_mutex.unlock(); + } + + Result fetch_entry(u64 block); + + void clear(); + + static void clear_caches(); + + StorageCache(); + ~StorageCache(); + + private: + HashMap m_cache_entries; + KMutex<100> m_mutex; +}; diff --git a/kernel/src/fs/devices/BlockDevice.cpp b/kernel/src/fs/devices/BlockDevice.cpp new file mode 100644 index 00000000..112b65b5 --- /dev/null +++ b/kernel/src/fs/devices/BlockDevice.cpp @@ -0,0 +1,51 @@ +#include "fs/devices/BlockDevice.h" +#include "arch/MMU.h" +#include + +BlockDevice::BlockDevice(u64 block_size, u64 num_blocks) : m_cache(), m_block_size(block_size), m_num_blocks(num_blocks) +{ +} + +Result BlockDevice::read(u8* buf, usize offset, usize length) const +{ + if (length == 0) return 0; + + if (offset > size()) return 0; + if (offset + length > size()) length = size() - offset; + + const usize saved = length; + + auto read_data = [&](u64 off, u64 len) -> Result { + const u64 block = offset / m_block_size; + + m_cache.lock(); + auto guard = make_scope_guard([&] { m_cache.unlock(); }); + + auto* entry = TRY(m_cache.fetch_entry(block)); + if (entry->buffer.is_empty()) + { + // TODO: What if the cache needs clearing in the middle of try_resize()? That could lead to some weird + // state... + TRY(entry->buffer.try_resize(m_block_size)); + TRY(read_block(entry->buffer, block)); + } + memcpy(buf, entry->buffer.data() + off, len); + offset += len; + length -= len; + buf += len; + return {}; + }; + + if (offset % m_block_size) + { + const u64 off = offset % m_block_size; + const u64 len = min(m_block_size - off, length); + TRY(read_data(off, len)); + } + + while (length >= m_block_size) { TRY(read_data(0, m_block_size)); } + + if (length) { TRY(read_data(0, length)); } + + return saved; +} diff --git a/kernel/src/fs/devices/BlockDevice.h b/kernel/src/fs/devices/BlockDevice.h new file mode 100644 index 00000000..8a411770 --- /dev/null +++ b/kernel/src/fs/devices/BlockDevice.h @@ -0,0 +1,40 @@ +#pragma once +#include "fs/StorageCache.h" +#include "fs/devices/Device.h" + +class BlockDevice : public Device +{ + public: + BlockDevice(u64 block_size, u64 num_blocks); + + Result read(u8* buf, usize offset, usize length) const override; + + virtual Result read_block(Buffer& buf, u64 block) const = 0; + + Result write(const u8*, usize, usize) override + { + return err(ENOTSUP); + } + + usize size() const override + { + return m_block_size * m_num_blocks; + } + + bool is_block_device() const override + { + return true; + } + + Result block_size() const override + { + return m_block_size; + } + + virtual ~BlockDevice() = default; + + protected: + mutable StorageCache m_cache; + u64 m_block_size; + u64 m_num_blocks; +}; diff --git a/kernel/src/fs/devices/Device.h b/kernel/src/fs/devices/Device.h index df251536..c1eab3ae 100644 --- a/kernel/src/fs/devices/Device.h +++ b/kernel/src/fs/devices/Device.h @@ -2,6 +2,8 @@ #include "Log.h" #include #include +#include +#include class Device : public Shareable { diff --git a/kernel/src/memory/MemoryManager.cpp b/kernel/src/memory/MemoryManager.cpp index 4f925bcd..d4ae70b1 100644 --- a/kernel/src/memory/MemoryManager.cpp +++ b/kernel/src/memory/MemoryManager.cpp @@ -1,6 +1,7 @@ #include "memory/MemoryManager.h" #include "Log.h" #include "arch/MMU.h" +#include "fs/StorageCache.h" #include "memory/KernelVM.h" #include "memory/MemoryMap.h" #include @@ -143,7 +144,12 @@ namespace MemoryManager usize index; bool ok = frame_bitmap->find_and_toggle(false, start_index).try_set_value(index); - if (!ok) return err(ENOMEM); + if (!ok) + { + kwarnln("OOM alert! Trying to free caches..."); + StorageCache::clear_caches(); + if (!frame_bitmap->find_and_toggle(false, start_index).try_set_value(index)) return err(ENOMEM); + } start_index = index + 1; diff --git a/libluna/include/luna/HashMap.h b/libluna/include/luna/HashMap.h index 3a276ae2..e88a1790 100644 --- a/libluna/include/luna/HashMap.h +++ b/libluna/include/luna/HashMap.h @@ -49,6 +49,11 @@ template struct HashMap return m_table.try_remove(HashPair { key, {} }); } + void clear() + { + m_table.clear(); + } + private: HashTable> m_table; };