kvdb: Implement FileKeyValueStore/Cache

include/stratosphere/kvdb/kvdb_auto_buffer.hpp

@@ -24,14 +24,19 @@ namespace sts::kvdb {
     class AutoBuffer {
         NON_COPYABLE(AutoBuffer);
         private:
-            std::unique_ptr<u8[]> buffer;
+            u8 *buffer;
             size_t size;
         public:
-            AutoBuffer() : size(0) { /* ... */ }
+            AutoBuffer() : buffer(nullptr), size(0) { /* ... */ }
+
+            ~AutoBuffer() {
+                this->Reset();
+            }
 
             AutoBuffer(AutoBuffer &&rhs) {
-                this->buffer = std::move(rhs.buffer);
+                this->buffer = rhs.buffer;
                 this->size = rhs.size;
+                rhs.buffer = nullptr;
                 rhs.size = 0;
             }
 
@@ -46,12 +51,15 @@ namespace sts::kvdb {
             }
 
             void Reset() {
-                this->buffer.reset();
+                if (this->buffer != nullptr) {
-                this->size = 0;
+                    std::free(this->buffer);
+                    this->buffer = nullptr;
+                    this->size = 0;
+                }
             }
 
             u8 *Get() const {
-                return this->buffer.get();
+                return this->buffer;
             }
 
             size_t GetSize() const {
@@ -65,12 +73,10 @@ namespace sts::kvdb {
                 }
 
                 /* Allocate a buffer. */
-                auto mem = new (std::nothrow) u8[size];
+                this->buffer = static_cast<u8 *>(std::malloc(size));
-                if (mem == nullptr) {
+                if (this->buffer == nullptr) {
                     return ResultKvdbAllocationFailed;
                 }
-
-                this->buffer.reset(mem);
                 this->size = size;
                 return ResultSuccess;
             }
@@ -80,7 +86,7 @@ namespace sts::kvdb {
                 R_TRY(this->Initialize(size));
 
                 /* Copy the input data in. */
-                std::memcpy(this->buffer.get(), buf, size);
+                std::memcpy(this->buffer, buf, size);
 
                 return ResultSuccess;
             }

include/stratosphere/kvdb/kvdb_bounded_string.hpp

@@ -38,20 +38,20 @@ namespace sts::kvdb {
             }
         public:
             /* Constructors. */
-            BoundedString() {
+            constexpr BoundedString() {
                 buffer[0] = 0;
             }
 
-            explicit BoundedString(const char *s) {
+            explicit constexpr BoundedString(const char *s) {
                 this->Set(s);
             }
 
             /* Static constructors. */
-            static BoundedString<N> Make(const char *s) {
+            static constexpr BoundedString<N> Make(const char *s) {
                 return BoundedString<N>(s);
             }
 
-            static BoundedString<N> MakeFormat(const char *format, ...) __attribute__((format (printf, 1, 2)))  {
+            static constexpr BoundedString<N> MakeFormat(const char *format, ...) __attribute__((format (printf, 1, 2)))  {
                 BoundedString<N> string;
 
                 std::va_list args;
@@ -142,7 +142,7 @@ namespace sts::kvdb {
                 return std::strncmp(this->buffer + offset, s, N - offset) == 0;
             }
 
-            bool EndsWith(const char *s) {
+            bool EndsWith(const char *s) const {
                 const size_t suffix_length = strnlen(s, N);
                 const size_t length = GetLength();
                 return suffix_length <= length && EndsWith(s, length - suffix_length);

include/stratosphere/kvdb/kvdb_file_key_value_cache.hpp

@@ -0,0 +1,422 @@
+/*
+ * Copyright (c) 2018-2019 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <sys/stat.h>
+#include <switch.h>
+#include "kvdb_bounded_string.hpp"
+#include "kvdb_file_key_value_store.hpp"
+
+namespace sts::kvdb {
+
+    namespace impl {
+
+        template<class Key, size_t Capacity>
+        class LruList {
+            private:
+                /* Subtypes. */
+                struct LruHeader {
+                    u32 entry_count;
+                };
+            public:
+                static constexpr size_t BufferSize = sizeof(Key) * Capacity;
+                static constexpr size_t FileSize = sizeof(LruHeader) + BufferSize;
+                using Path = FileKeyValueStore::Path;
+            private:
+                Path file_path;
+                Key *keys;
+                LruHeader header;
+            public:
+                static Result CreateNewList(const char *path) {
+                    /* Create new lru_list.dat. */
+                    R_TRY(fsdevCreateFile(path, FileSize, 0));
+
+                    /* Open the file. */
+                    FILE *fp = fopen(path, "r+b");
+                    if (fp == nullptr) {
+                        return fsdevGetLastResult();
+                    }
+                    ON_SCOPE_EXIT { fclose(fp); };
+
+                    /* Write new header with zero entries to the file. */
+                    LruHeader new_header = { .entry_count = 0, };
+                    if (fwrite(&new_header, sizeof(new_header), 1, fp) != 1) {
+                        return fsdevGetLastResult();
+                    }
+
+                    return ResultSuccess;
+                }
+            private:
+                void RemoveIndex(size_t i) {
+                    if (i >= this->GetCount()) {
+                        std::abort();
+                    }
+                    std::memmove(this->keys + i, this->keys + i + 1, sizeof(*this->keys) * (this->GetCount() - (i + 1)));
+                    this->DecrementCount();
+                }
+
+                void IncrementCount() {
+                    this->header.entry_count++;
+                }
+
+                void DecrementCount() {
+                    this->header.entry_count--;
+                }
+            public:
+                LruList() : keys(nullptr), header({}) { /* ... */ }
+
+                Result Initialize(const char *path, void *buf, size_t size) {
+                    /* Only initialize once, and ensure we have sufficient memory. */
+                    if (this->keys != nullptr || size < BufferSize) {
+                        std::abort();
+                    }
+
+                    /* Setup member variables. */
+                    this->keys = static_cast<Key *>(buf);
+                    this->file_path.Set(path);
+                    std::memset(this->keys, 0, BufferSize);
+
+                    /* Open file. */
+                    FILE *fp = fopen(this->file_path, "rb");
+                    if (fp == nullptr) {
+                        return fsdevGetLastResult();
+                    }
+                    ON_SCOPE_EXIT { fclose(fp); };
+
+                    /* Read header. */
+                    if (fread(&this->header, sizeof(this->header), 1, fp) != 1) {
+                        return fsdevGetLastResult();
+                    }
+
+                    /* Read entries. */
+                    const size_t count = this->GetCount();
+                    if (count > 0) {
+                        if (fread(this->keys, std::min(BufferSize, sizeof(Key) * count), 1, fp) != 1) {
+                            return fsdevGetLastResult();
+                        }
+                    }
+
+                    return ResultSuccess;
+                }
+
+                Result Save() {
+                    /* Open file. */
+                    FILE *fp = fopen(this->file_path, "r+b");
+                    if (fp == nullptr) {
+                        return fsdevGetLastResult();
+                    }
+                    ON_SCOPE_EXIT { fclose(fp); };
+
+                    /* Write header. */
+                    if (fwrite(&this->header, sizeof(this->header), 1, fp) != 1) {
+                        return fsdevGetLastResult();
+                    }
+
+                    /* Write entries. */
+                    if (fwrite(this->keys, BufferSize, 1, fp) != 1) {
+                        return fsdevGetLastResult();
+                    }
+
+                    /* Flush. */
+                    fflush(fp);
+
+                    return ResultSuccess;
+                }
+
+                size_t GetCount() const {
+                    return this->header.entry_count;
+                }
+
+                bool IsEmpty() const {
+                    return this->GetCount() == 0;
+                }
+
+                bool IsFull() const {
+                    return this->GetCount() >= Capacity;
+                }
+
+                Key Get(size_t i) const {
+                    if (i >= this->GetCount()) {
+                        std::abort();
+                    }
+                    return this->keys[i];
+                }
+
+                Key Peek() const {
+                    if (this->IsEmpty()) {
+                        std::abort();
+                    }
+                    return this->Get(0);
+                }
+
+                void Push(const Key &key) {
+                    if (this->IsFull()) {
+                        std::abort();
+                    }
+                    this->keys[this->GetCount()] = key;
+                    this->IncrementCount();
+                }
+
+                Key Pop() {
+                    if (this->IsEmpty()) {
+                        std::abort();
+                    }
+                    this->RemoveIndex(0);
+                }
+
+                bool Remove(const Key &key) {
+                    const size_t count = this->GetCount();
+
+                    /* Iterate over the list, removing the last entry that matches the key. */
+                    for (size_t i = 0; i < count; i++) {
+                        if (this->keys[count - 1 - i] == key) {
+                            this->RemoveIndex(count - 1 - i);
+                            return true;
+                        }
+                    }
+
+                    return false;
+                }
+
+                bool Contains(const Key &key) const {
+                    const size_t count = this->GetCount();
+
+                    /* Iterate over the list, checking to see if we have the key. */
+                    for (size_t i = 0; i < count; i++) {
+                        if (this->keys[count - 1 - i] == key) {
+                            return true;
+                        }
+                    }
+
+                    return false;
+                }
+
+                bool Update(const Key &key) {
+                    if (this->Remove(key)) {
+                        this->Push(key);
+                        return true;
+                    }
+
+                    return false;
+                }
+        };
+
+    }
+
+    template<class Key, size_t Capacity>
+    class FileKeyValueCache {
+        static_assert(std::is_pod<Key>::value, "FileKeyValueCache Key must be pod!");
+        static_assert(sizeof(Key) <= FileKeyValueStore::MaxKeySize, "FileKeyValueCache Key is too big!");
+        public:
+            using LeastRecentlyUsedList = impl::LruList<Key, Capacity>;
+            /* Note: Nintendo code in NS uses Path = BoundedString<0x180> here. */
+            /* It's unclear why, since they use 0x300 everywhere else. */
+            /* We'll just use 0x300, since it shouldn't make a difference, */
+            /* as FileKeyValueStore paths are limited to 0x100 anyway. */
+            using Path = typename LeastRecentlyUsedList::Path;
+        private:
+            FileKeyValueStore kvs;
+            LeastRecentlyUsedList lru_list;
+        private:
+            static constexpr Path GetLeastRecentlyUsedListPath(const char *dir) {
+                return Path::MakeFormat("%s/%s", dir, "lru_list.dat");
+            }
+
+            static constexpr Path GetFileKeyValueStorePath(const char *dir) {
+                return Path::MakeFormat("%s/%s", dir, "kvs");
+            }
+
+            static Result Exists(bool *out, const char *path, bool is_dir) {
+                /* Check if the path exists. */
+                struct stat st;
+                if (stat(path, &st) != 0) {
+                    R_TRY_CATCH(fsdevGetLastResult()) {
+                        R_CATCH(ResultFsPathNotFound) {
+                            /* If the path doesn't exist, nothing has gone wrong. */
+                            *out = false;
+                            return ResultSuccess;
+                        }
+                    } R_END_TRY_CATCH;
+                }
+
+                /* Check that our entry type is correct. */
+                if ((is_dir && !(S_ISDIR(st.st_mode))) || (!is_dir && !(S_ISREG(st.st_mode)))) {
+                    return ResultKvdbInvalidFilesystemState;
+                }
+
+                *out = true;
+                return ResultSuccess;
+            }
+
+            static Result DirectoryExists(bool *out, const char *path) {
+                return Exists(out, path, true);
+            }
+
+            static Result FileExists(bool *out, const char *path) {
+                return Exists(out, path, false);
+            }
+        public:
+            static Result CreateNewCache(const char *dir) {
+                /* Make a new key value store filesystem, and a new lru_list.dat. */
+                R_TRY(LeastRecentlyUsedList::CreateNewList(GetLeastRecentlyUsedListPath(dir)));
+                if (mkdir(GetFileKeyValueStorePath(dir), 0) != 0) {
+                    return fsdevGetLastResult();
+                }
+
+                return ResultSuccess;
+            }
+
+            static Result ValidateExistingCache(const char *dir) {
+                /* Check for existence. */
+                bool has_lru = false, has_kvs = false;
+                R_TRY(FileExists(&has_lru, GetLeastRecentlyUsedListPath(dir)));
+                R_TRY(DirectoryExists(&has_kvs, GetFileKeyValueStorePath(dir)));
+
+                /* If neither exists, CreateNewCache was never called. */
+                if (!has_lru && !has_kvs) {
+                    return ResultKvdbNotCreated;
+                }
+
+                /* If one exists but not the other, we have an invalid state. */
+                if (has_lru ^ has_kvs) {
+                    return ResultKvdbInvalidFilesystemState;
+                }
+
+                return ResultSuccess;
+            }
+        private:
+            void RemoveOldestKey() {
+                const Key &oldest_key = this->lru_list.Peek();
+                this->lru_list.Pop();
+                this->kvs.Remove(oldest_key);
+            }
+        public:
+            Result Initialize(const char *dir, void *buf, size_t size) {
+                /* Initialize list. */
+                R_TRY(this->lru_list.Initialize(GetLeastRecentlyUsedListPath(dir), buf, size));
+
+                /* Initialize kvs. */
+                /* NOTE: Despite creating the kvs folder and returning an error if it does not exist, */
+                /* Nintendo does not use the kvs folder, and instead uses the passed dir. */
+                /* This causes lru_list.dat to be in the same directory as the store's .val files */
+                /* instead of in the same directory as a folder containing the store's .val files. */
+                /* This is probably a Nintendo bug, but because system saves contain data in the wrong */
+                /* layout it can't really be fixed without breaking existing devices... */
+                R_TRY(this->kvs.Initialize(dir));
+
+                return ResultSuccess;
+            }
+
+            size_t GetCount() const {
+                return this->lru_list.GetCount();
+            }
+
+            size_t GetCapacity() const {
+                return Capacity;
+            }
+
+            Key GetKey(size_t i) const {
+                return this->lru_list.Get(i);
+            }
+
+            bool Contains(const Key &key) const {
+                return this->lru_list.Contains(key);
+            }
+
+            Result Get(size_t *out_size, void *out_value, size_t max_out_size, const Key &key) {
+                /* Note that we accessed the key. */
+                this->lru_list.Update(key);
+                return this->kvs.Get(out_size, out_value, max_out_size, key);
+            }
+
+            template<typename Value>
+            Result Get(Value *out_value, const Key &key) {
+                /* Note that we accessed the key. */
+                this->lru_list.Update(key);
+                return this->kvs.Get(out_value, key);
+            }
+
+            Result GetSize(size_t *out_size, const Key &key) {
+                return this->kvs.GetSize(out_size, key);
+            }
+
+            Result Set(const Key &key, const void *value, size_t value_size) {
+                if (this->lru_list.Update(key)) {
+                    /* If an entry for the key exists, delete the existing value file. */
+                    this->kvs.Remove(key);
+                } else {
+                    /* If the list is full, we need to remove the oldest key. */
+                    if (this->lru_list.IsFull()) {
+                        this->RemoveOldestKey();
+                    }
+
+                    /* Add the key to the list. */
+                    this->lru_list.Push(key);
+                }
+
+                /* Loop, trying to save the new value to disk. */
+                while (true) {
+                    /* Try to set the key. */
+                    R_TRY_CATCH(this->kvs.Set(key, value, value_size)) {
+                        R_CATCH_RANGE(ResultFsNotEnoughFreeSpace) {
+                            /* If our entry is the only thing in the Lru list, remove it. */
+                            if (this->lru_list.GetCount() == 1) {
+                                this->lru_list.Pop();
+                                R_TRY(this->lru_list.Save());
+                                return R_TRY_CATCH_RESULT;
+                            }
+
+                            /* Otherwise, remove the oldest element from the cache and try again. */
+                            this->RemoveOldestKey();
+                            continue;
+                        }
+                    } R_END_TRY_CATCH;
+
+                    /* If we got here, we succeeded. */
+                    break;
+                }
+
+                /* Save the list. */
+                R_TRY(this->lru_list.Save());
+
+                return ResultSuccess;
+            }
+
+            template<typename Value>
+            Result Set(const Key &key, const Value &value) {
+                return this->Set(key, &value, sizeof(Value));
+            }
+
+            Result Remove(const Key &key) {
+                /* Remove the key. */
+                this->lru_list.Remove(key);
+                R_TRY(this->kvs.Remove(key));
+                R_TRY(this->lru_list.Save());
+
+                return ResultSuccess;
+            }
+
+            Result RemoveAll() {
+                /* TODO: Nintendo doesn't check errors here. Should we? */
+                while (!this->lru_list.IsEmpty()) {
+                    this->RemoveOldestKey();
+                }
+                R_TRY(this->lru_list.Save());
+
+                return ResultSuccess;
+            }
+    };
+
+}

include/stratosphere/kvdb/kvdb_file_key_value_store.hpp

@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2018-2019 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <optional>
+#include <switch.h>
+
+#include "kvdb_bounded_string.hpp"
+
+namespace sts::kvdb {
+
+    class FileKeyValueStore {
+        NON_COPYABLE(FileKeyValueStore);
+        NON_MOVEABLE(FileKeyValueStore);
+        public:
+            static constexpr size_t MaxPathLength = 0x300; /* TODO: FS_MAX_PATH - 1? */
+            static constexpr size_t MaxFileLength = 0xFF;
+            static constexpr char FileExtension[5] = ".val";
+            static constexpr size_t FileExtensionLength = sizeof(FileExtension) - 1;
+            static constexpr size_t MaxKeySize = (MaxFileLength - FileExtensionLength) / 2;
+            using Path = kvdb::BoundedString<MaxPathLength>;
+            using FileName = kvdb::BoundedString<MaxFileLength>;
+        private:
+            /* Subtypes. */
+            struct Entry {
+                u8 key[MaxKeySize];
+                void *value;
+                size_t key_size;
+                size_t value_size;
+            };
+            static_assert(std::is_pod<Entry>::value, "FileKeyValueStore::Entry definition!");
+
+            class Cache {
+                private:
+                    u8 *backing_buffer = nullptr;
+                    size_t backing_buffer_size = 0;
+                    size_t backing_buffer_free_offset = 0;
+                    Entry *entries = nullptr;
+                    size_t count = 0;
+                    size_t capacity = 0;
+                private:
+                    void *Allocate(size_t size);
+
+                    bool HasEntries() const {
+                        return this->entries != nullptr && this->capacity != 0;
+                    }
+                public:
+                    Result Initialize(void *buffer, size_t buffer_size, size_t capacity);
+                    void Invalidate();
+                    std::optional<size_t> TryGet(void *out_value, size_t max_out_size, const void *key, size_t key_size);
+                    std::optional<size_t> TryGetSize(const void *key, size_t key_size);
+                    void Set(const void *key, size_t key_size, const void *value, size_t value_size);
+                    bool Contains(const void *key, size_t key_size);
+            };
+        private:
+            HosMutex lock;
+            Path dir_path;
+            Cache cache;
+        private:
+            Path GetPath(const void *key, size_t key_size);
+            Result GetKey(size_t *out_size, void *out_key, size_t max_out_size, const FileName &file_name);
+        public:
+            FileKeyValueStore() { /* ... */ }
+
+            /* Basic accessors. */
+            Result Initialize(const char *dir);
+            Result InitializeWithCache(const char *dir, void *cache_buffer, size_t cache_buffer_size, size_t cache_capacity);
+            Result Get(size_t *out_size, void *out_value, size_t max_out_size, const void *key, size_t key_size);
+            Result GetSize(size_t *out_size, const void *key, size_t key_size);
+            Result Set(const void *key, size_t key_size, const void *value, size_t value_size);
+            Result Remove(const void *key, size_t key_size);
+
+            /* Niceties. */
+            template<typename Key>
+            Result Get(size_t *out_size, void *out_value, size_t max_out_size, const Key &key) {
+                static_assert(std::is_pod<Key>::value && sizeof(Key) <= MaxKeySize, "Invalid FileKeyValueStore Key!");
+                return this->Get(out_size, out_value, max_out_size, &key, sizeof(Key));
+            }
+
+            template<typename Key, typename Value>
+            Result Get(Value *out_value, const Key &key) {
+                static_assert(std::is_pod<Value>::value && !std::is_pointer<Value>::value, "Invalid FileKeyValueStore Value!");
+                size_t size = 0;
+                R_TRY(this->Get(&size, out_value, sizeof(Value), key));
+                if (size < sizeof(Value)) {
+                    std::abort();
+                }
+                return ResultSuccess;
+            }
+
+            template<typename Key>
+            Result GetSize(size_t *out_size, const Key &key) {
+                return this->GetSize(out_size, &key, sizeof(Key));
+            }
+
+            template<typename Key>
+            Result Set(const Key &key, const void *value, size_t value_size) {
+                static_assert(std::is_pod<Key>::value && sizeof(Key) <= MaxKeySize, "Invalid FileKeyValueStore Key!");
+                return this->Set(&key, sizeof(Key), value, value_size);
+            }
+
+            template<typename Key, typename Value>
+            Result Set(const Key &key, const Value &value) {
+                static_assert(std::is_pod<Value>::value && !std::is_pointer<Value>::value, "Invalid FileKeyValueStore Value!");
+                return this->Set(key, &value, sizeof(Value));
+            }
+
+            template<typename Key>
+            Result Remove(const Key &key) {
+                return this->Remove(&key, sizeof(Key));
+            }
+    };
+
+}

include/stratosphere/kvdb/kvdb_memory_key_value_store.hpp

@@ -150,7 +150,7 @@ namespace sts::kvdb {
                             /* We need to add a new entry. Check we have room, move future keys forward. */
                             if (this->count >= this->capacity) {
                                 std::free(new_value);
-                                return ResultKvdbTooManyKeys;
+                                return ResultKvdbKeyCapacityInsufficient;
                             }
                             std::memmove(it + 1, it, sizeof(*it) * (this->end() - it));
                             this->count++;
@@ -163,7 +163,7 @@ namespace sts::kvdb {
 
                     Result AddUnsafe(const Key &key, void *value, size_t value_size) {
                         if (this->count >= this->capacity) {
-                            return ResultKvdbTooManyKeys;
+                            return ResultKvdbKeyCapacityInsufficient;
                         }
 
                         this->entries[this->count++] = Entry(key, value, value_size);
@@ -506,9 +506,12 @@ namespace sts::kvdb {
                 /* Try to delete temporary archive, but allow deletion failure (it may not exist). */
                 std::remove(this->temp_path.Get());
 
+                /* Create new temporary archive. */
+                R_TRY(fsdevCreateFile(this->temp_path.Get(), buffer.GetSize(), 0));
+
                 /* Write data to the temporary archive. */
                 {
-                    FILE *f = fopen(this->temp_path, "wb");
+                    FILE *f = fopen(this->temp_path, "r+b");
                     if (f == nullptr) {
                         return fsdevGetLastResult();
                     }

include/stratosphere/results/fs_results.hpp

@@ -25,6 +25,16 @@ static constexpr Result ResultFsPathAlreadyExists  = MAKERESULT(Module_Fs, 2);
 static constexpr Result ResultFsTargetLocked       = MAKERESULT(Module_Fs, 7);
 static constexpr Result ResultFsDirectoryNotEmpty  = MAKERESULT(Module_Fs, 8);
 
+static constexpr Result ResultFsNotEnoughFreeSpaceRangeStart             = MAKERESULT(Module_Fs, 30);
+    static constexpr Result ResultFsNotEnoughFreeSpaceBisRangeStart      = MAKERESULT(Module_Fs, 34);
+        static constexpr Result ResultFsNotEnoughFreeSpaceBisCalibration = MAKERESULT(Module_Fs, 35);
+        static constexpr Result ResultFsNotEnoughFreeSpaceBisSafe        = MAKERESULT(Module_Fs, 36);
+        static constexpr Result ResultFsNotEnoughFreeSpaceBisUser        = MAKERESULT(Module_Fs, 37);
+        static constexpr Result ResultFsNotEnoughFreeSpaceBisSystem      = MAKERESULT(Module_Fs, 38);
+    static constexpr Result ResultFsNotEnoughFreeSpaceBisRangeEnd        = MAKERESULT(Module_Fs, 39);
+    static constexpr Result ResultFsNotEnoughFreeSpaceSdCard             = MAKERESULT(Module_Fs, 39);
+static constexpr Result ResultFsNotEnoughFreeSpaceRangeEnd               = MAKERESULT(Module_Fs, 45);
+
 static constexpr Result ResultFsMountNameAlreadyExists = MAKERESULT(Module_Fs, 60);
 
 static constexpr Result ResultFsTargetNotFound   = MAKERESULT(Module_Fs, 1002);

include/stratosphere/results/kvdb_results.hpp

@@ -19,8 +19,11 @@
 
 static constexpr u32 Module_Kvdb = 20;
 
-static constexpr Result ResultKvdbTooManyKeys        = MAKERESULT(Module_Kvdb, 1);
+static constexpr Result ResultKvdbKeyCapacityInsufficient = MAKERESULT(Module_Kvdb, 1);
-static constexpr Result ResultKvdbKeyNotFound        = MAKERESULT(Module_Kvdb, 2);
+static constexpr Result ResultKvdbKeyNotFound             = MAKERESULT(Module_Kvdb, 2);
-static constexpr Result ResultKvdbAllocationFailed   = MAKERESULT(Module_Kvdb, 4);
+static constexpr Result ResultKvdbAllocationFailed        = MAKERESULT(Module_Kvdb, 4);
-static constexpr Result ResultKvdbInvalidKeyValue    = MAKERESULT(Module_Kvdb, 5);
+static constexpr Result ResultKvdbInvalidKeyValue         = MAKERESULT(Module_Kvdb, 5);
-static constexpr Result ResultKvdbBufferInsufficient = MAKERESULT(Module_Kvdb, 6);
+static constexpr Result ResultKvdbBufferInsufficient      = MAKERESULT(Module_Kvdb, 6);
+
+static constexpr Result ResultKvdbInvalidFilesystemState  = MAKERESULT(Module_Kvdb, 8);
+static constexpr Result ResultKvdbNotCreated              = MAKERESULT(Module_Kvdb, 9);

include/stratosphere/results/utilities.h

@@ -37,40 +37,48 @@ extern "C" {
     })
 
 /// Helpers for pattern-matching on a result expression, if the result would fail.
+#define R_TRY_CATCH_RESULT _tmp_r_try_catch_rc
+
 #define R_TRY_CATCH(res_expr) \
     ({ \
-        const Result _tmp_r_try_catch_rc = res_expr; \
+        const Result R_TRY_CATCH_RESULT = res_expr; \
-        if (R_FAILED(_tmp_r_try_catch_rc)) { \
+        if (R_FAILED(R_TRY_CATCH_RESULT)) { \
             if (false)
 
 #define R_CATCH(catch_result) \
-            } else if (_tmp_r_try_catch_rc == catch_result) { \
+            } else if (R_TRY_CATCH_RESULT == catch_result) { \
                 _Static_assert(R_FAILED(catch_result), "Catch result must be constexpr error Result!"); \
                 if (false) { } \
                 else
 
-#define R_CATCH_RANGE(catch_result_start, catch_result_end) \
+#define R_GET_CATCH_RANGE_IMPL(_1, _2, NAME, ...) NAME
-            } else if (catch_result_start <= _tmp_r_try_catch_rc && _tmp_r_try_catch_rc <= catch_result_end) { \
+
+#define R_CATCH_RANGE_IMPL_2(catch_result_start, catch_result_end) \
+            } else if (catch_result_start <= R_TRY_CATCH_RESULT && R_TRY_CATCH_RESULT <= catch_result_end) { \
                 _Static_assert(R_FAILED(catch_result_start), "Catch start result must be constexpr error Result!"); \
                 _Static_assert(R_FAILED(catch_result_end), "Catch end result must be constexpr error Result!");  \
                 _Static_assert(R_MODULE(catch_result_start) == R_MODULE(catch_result_end), "Catch range modules must be equal!"); \
                 if (false) { } \
                 else
 
+#define R_CATCH_RANGE_IMPL_1(catch_result) R_CATCH_RANGE_IMPL_2(catch_result##RangeStart, catch_result##RangeEnd)
+
+#define R_CATCH_RANGE(...) R_GET_CATCH_RANGE_IMPL(__VA_ARGS__, R_CATCH_RANGE_IMPL_2, R_CATCH_RANGE_IMPL_1)(__VA_ARGS__)
+
 #define R_CATCH_MODULE(module) \
-            } else if (R_MODULE(_tmp_r_try_catch_rc) == module) { \
+            } else if (R_MODULE(R_TRY_CATCH_RESULT) == module) { \
                 _Static_assert(module != 0, "Catch module must be error!"); \
                 if (false) { } \
                 else
 
 #define R_CATCH_ALL() \
-            } else if (R_FAILED(_tmp_r_try_catch_rc)) { \
+            } else if (R_FAILED(R_TRY_CATCH_RESULT)) { \
                 if (false) { } \
                 else
 
 #define R_END_TRY_CATCH \
-            else if (R_FAILED(_tmp_r_try_catch_rc)) { \
+            else if (R_FAILED(R_TRY_CATCH_RESULT)) { \
-                return _tmp_r_try_catch_rc; \
+                return R_TRY_CATCH_RESULT; \
             } \
         } \
     })

source/kvdb/kvdb_file_key_value_store.cpp

@@ -0,0 +1,351 @@
+/*
+ * Copyright (c) 2018-2019 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <sys/stat.h>
+#include <stratosphere.hpp>
+#include <stratosphere/kvdb/kvdb_file_key_value_store.hpp>
+
+namespace sts::kvdb {
+
+    /* Cache implementation. */
+    void *FileKeyValueStore::Cache::Allocate(size_t size) {
+        if (this->backing_buffer_size - this->backing_buffer_free_offset < size) {
+            return nullptr;
+        }
+        ON_SCOPE_EXIT { this->backing_buffer_free_offset += size; };
+        return this->backing_buffer + this->backing_buffer_free_offset;
+    }
+
+    Result FileKeyValueStore::Cache::Initialize(void *buffer, size_t buffer_size, size_t capacity) {
+        this->backing_buffer = static_cast<u8 *>(buffer);
+        this->backing_buffer_size = buffer_size;
+        this->backing_buffer_free_offset = 0;
+        this->entries = nullptr;
+        this->count = 0;
+        this->capacity = capacity;
+
+        /* If we have memory to work with, ensure it's at least enough for the cache entries. */
+        if (this->backing_buffer != nullptr) {
+            this->entries = static_cast<decltype(this->entries)>(this->Allocate(sizeof(*this->entries) * this->capacity));
+            if (this->entries == nullptr) {
+                return ResultKvdbBufferInsufficient;
+            }
+        }
+
+        return ResultSuccess;
+    }
+
+    void FileKeyValueStore::Cache::Invalidate() {
+        if (!this->HasEntries()) {
+            return;
+        }
+
+        /* Reset the allocation pool. */
+        this->backing_buffer_free_offset = 0;
+        this->count = 0;
+        this->entries = static_cast<decltype(this->entries)>(this->Allocate(sizeof(*this->entries) * this->capacity));
+        if (this->entries == nullptr) {
+            std::abort();
+        }
+    }
+
+    std::optional<size_t> FileKeyValueStore::Cache::TryGet(void *out_value, size_t max_out_size, const void *key, size_t key_size) {
+        if (!this->HasEntries()) {
+            return std::nullopt;
+        }
+
+        /* Try to find the entry. */
+        for (size_t i = 0; i < this->count; i++) {
+            const auto &entry = this->entries[i];
+            if (entry.key_size == key_size && std::memcmp(entry.key, key, key_size) == 0) {
+                /* If we don't have enough space, fail to read from cache. */
+                if (max_out_size < entry.value_size) {
+                    return std::nullopt;
+                }
+
+                std::memcpy(out_value, entry.value, entry.value_size);
+                return entry.value_size;
+            }
+        }
+
+        return std::nullopt;
+    }
+
+    std::optional<size_t> FileKeyValueStore::Cache::TryGetSize(const void *key, size_t key_size) {
+        if (!this->HasEntries()) {
+            return std::nullopt;
+        }
+
+        /* Try to find the entry. */
+        for (size_t i = 0; i < this->count; i++) {
+            const auto &entry = this->entries[i];
+            if (entry.key_size == key_size && std::memcmp(entry.key, key, key_size) == 0) {
+                return entry.value_size;
+            }
+        }
+
+        return std::nullopt;
+    }
+
+    void FileKeyValueStore::Cache::Set(const void *key, size_t key_size, const void *value, size_t value_size) {
+        if (!this->HasEntries()) {
+            return;
+        }
+
+        /* Ensure key size is small enough. */
+        if (key_size > MaxKeySize) {
+            std::abort();
+        }
+
+        /* If we're at capacity, invalidate the cache. */
+        if (this->count == this->capacity) {
+            this->Invalidate();
+        }
+
+        /* Allocate memory for the value. */
+        void *value_buf = this->Allocate(value_size);
+        if (value_buf == nullptr) {
+            /* We didn't have enough memory for the value. Invalidating might get us enough memory. */
+            this->Invalidate();
+            value_buf = this->Allocate(value_size);
+            if (value_buf == nullptr) {
+                /* If we still don't have enough memory, just fail to put the value in the cache. */
+                return;
+            }
+        }
+
+        auto &entry = this->entries[this->count++];
+        std::memcpy(entry.key, key, key_size);
+        entry.key_size = key_size;
+        entry.value = value_buf;
+        std::memcpy(entry.value, value, value_size);
+        entry.value_size = value_size;
+    }
+
+    bool FileKeyValueStore::Cache::Contains(const void *key, size_t key_size) {
+        return this->TryGetSize(key, key_size).has_value();
+    }
+
+    /* Store functionality. */
+    FileKeyValueStore::Path FileKeyValueStore::GetPath(const void *_key, size_t key_size) {
+        /* Format is "<dir>/<hex formatted key>.val" */
+        FileKeyValueStore::Path key_path(this->dir_path.Get());
+        key_path.Append('/');
+
+        /* Append hex formatted key. */
+        const u8 *key = static_cast<const u8 *>(_key);
+        for (size_t i = 0; i < key_size; i++) {
+            key_path.AppendFormat("%02x", key[i]);
+        }
+
+        /* Append extension. */
+        key_path.Append(FileExtension);
+
+        return key_path;
+    }
+
+    Result FileKeyValueStore::GetKey(size_t *out_size, void *_out_key, size_t max_out_size, const FileKeyValueStore::FileName &file_name) {
+        /* Validate that the filename can be converted to a key. */
+        /* TODO: Nintendo does not validate that the key is valid hex. Should we do this? */
+        const size_t file_name_len = file_name.GetLength();
+        const size_t key_name_len = file_name_len - FileExtensionLength;
+        if (file_name_len < FileExtensionLength + 2 || !file_name.EndsWith(FileExtension) || key_name_len % 2 != 0) {
+            return ResultKvdbInvalidKeyValue;
+        }
+
+        /* Validate that we have space for the converted key. */
+        const size_t key_size = key_name_len / 2;
+        if (key_size > max_out_size) {
+            return ResultKvdbBufferInsufficient;
+        }
+
+        /* Convert the hex key back. */
+        u8 *out_key = static_cast<u8 *>(_out_key);
+        for (size_t i = 0; i < key_size; i++) {
+            char substr[2 * sizeof(u8) + 1];
+            file_name.GetSubstring(substr, sizeof(substr), 2 * i, sizeof(substr) - 1);
+            out_key[i] = static_cast<u8>(std::strtoul(substr, nullptr, 0x10));
+        }
+
+        *out_size = key_size;
+        return ResultSuccess;
+    }
+
+    Result FileKeyValueStore::Initialize(const char *dir) {
+        return this->InitializeWithCache(dir, nullptr, 0, 0);
+    }
+
+    Result FileKeyValueStore::InitializeWithCache(const char *dir, void *cache_buffer, size_t cache_buffer_size, size_t cache_capacity) {
+        /* Ensure that the passed path is a directory. */
+        {
+            struct stat st;
+            if (stat(dir, &st) != 0 || !(S_ISDIR(st.st_mode))) {
+                return ResultFsPathNotFound;
+            }
+        }
+
+        /* Set path. */
+        this->dir_path.Set(dir);
+
+        /* Initialize our cache. */
+        R_TRY(this->cache.Initialize(cache_buffer, cache_buffer_size, cache_capacity));
+        return ResultSuccess;
+    }
+
+    Result FileKeyValueStore::Get(size_t *out_size, void *out_value, size_t max_out_size, const void *key, size_t key_size) {
+        std::scoped_lock lk(this->lock);
+
+        /* Ensure key size is small enough. */
+        if (key_size > MaxKeySize) {
+            return ResultKvdbKeyCapacityInsufficient;
+        }
+
+        /* Try to get from cache. */
+        {
+            auto size = this->cache.TryGet(out_value, max_out_size, key, key_size);
+            if (size) {
+                *out_size = *size;
+                return ResultSuccess;
+            }
+        }
+
+        /* Open the value file. */
+        FILE *fp = fopen(this->GetPath(key, key_size), "rb");
+        if (fp == nullptr) {
+            R_TRY_CATCH(fsdevGetLastResult()) {
+                R_CATCH(ResultFsPathNotFound) {
+                    return ResultKvdbKeyNotFound;
+                }
+            } R_END_TRY_CATCH;
+        }
+        ON_SCOPE_EXIT { fclose(fp); };
+
+        /* Get the value size. */
+        fseek(fp, 0, SEEK_END);
+        const size_t value_size = ftell(fp);
+        fseek(fp, 0, SEEK_SET);
+
+        /* Ensure there's enough space for the value. */
+        if (max_out_size < value_size) {
+            return ResultKvdbBufferInsufficient;
+        }
+
+        /* Read the value. */
+        if (fread(out_value, value_size, 1, fp) != 1) {
+            return fsdevGetLastResult();
+        }
+        *out_size = value_size;
+
+        /* Cache the newly read value. */
+        this->cache.Set(key, key_size, out_value, value_size);
+        return ResultSuccess;
+    }
+
+    Result FileKeyValueStore::GetSize(size_t *out_size, const void *key, size_t key_size) {
+        std::scoped_lock lk(this->lock);
+
+        /* Ensure key size is small enough. */
+        if (key_size > MaxKeySize) {
+            return ResultKvdbKeyCapacityInsufficient;
+        }
+
+        /* Try to get from cache. */
+        {
+            auto size = this->cache.TryGetSize(key, key_size);
+            if (size) {
+                *out_size = *size;
+                return ResultSuccess;
+            }
+        }
+
+        /* Open the value file. */
+        FILE *fp = fopen(this->GetPath(key, key_size), "rb");
+        if (fp == nullptr) {
+            R_TRY_CATCH(fsdevGetLastResult()) {
+                R_CATCH(ResultFsPathNotFound) {
+                    return ResultKvdbKeyNotFound;
+                }
+            } R_END_TRY_CATCH;
+        }
+        ON_SCOPE_EXIT { fclose(fp); };
+
+        /* Get the value size. */
+        fseek(fp, 0, SEEK_END);
+        *out_size = ftell(fp);
+        return ResultSuccess;
+    }
+
+    Result FileKeyValueStore::Set(const void *key, size_t key_size, const void *value, size_t value_size) {
+        std::scoped_lock lk(this->lock);
+
+        /* Ensure key size is small enough. */
+        if (key_size > MaxKeySize) {
+            return ResultKvdbKeyCapacityInsufficient;
+        }
+
+        /* When the cache contains the key being set, Nintendo invalidates the cache. */
+        if (this->cache.Contains(key, key_size)) {
+            this->cache.Invalidate();
+        }
+
+        /* Delete the file, if it exists. Don't check result, since it's okay if it's already deleted. */
+        auto key_path = this->GetPath(key, key_size);
+        std::remove(key_path);
+
+        /* Open the value file. */
+        FILE *fp = fopen(key_path, "wb");
+        if (fp == nullptr) {
+            return fsdevGetLastResult();
+        }
+        ON_SCOPE_EXIT { fclose(fp); };
+
+        /* Write the value file. */
+        if (fwrite(value, value_size, 1, fp) != 1) {
+            return fsdevGetLastResult();
+        }
+
+        /* Flush the value file. */
+        fflush(fp);
+
+        return ResultSuccess;
+    }
+
+    Result FileKeyValueStore::Remove(const void *key, size_t key_size) {
+        std::scoped_lock lk(this->lock);
+
+        /* Ensure key size is small enough. */
+        if (key_size > MaxKeySize) {
+            return ResultKvdbKeyCapacityInsufficient;
+        }
+
+        /* When the cache contains the key being set, Nintendo invalidates the cache. */
+        if (this->cache.Contains(key, key_size)) {
+            this->cache.Invalidate();
+        }
+
+        /* Remove the file. */
+        if (std::remove(this->GetPath(key, key_size)) != 0) {
+            R_TRY_CATCH(fsdevGetLastResult()) {
+                R_CATCH(ResultFsPathNotFound) {
+                    return ResultKvdbKeyNotFound;
+                }
+            } R_END_TRY_CATCH;
+        }
+
+        return ResultSuccess;
+    }
+
+}