From 09d8f593c6971135fdb357c9df8b5c4d74bdd4f7 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Mon, 15 Jul 2019 14:38:06 -0700 Subject: [PATCH] kvdb: Implement FileKeyValueStore/Cache --- .../stratosphere/kvdb/kvdb_auto_buffer.hpp | 28 +- .../stratosphere/kvdb/kvdb_bounded_string.hpp | 10 +- .../kvdb/kvdb_file_key_value_cache.hpp | 422 ++++++++++++++++++ .../kvdb/kvdb_file_key_value_store.hpp | 127 ++++++ .../kvdb/kvdb_memory_key_value_store.hpp | 9 +- include/stratosphere/results/fs_results.hpp | 10 + include/stratosphere/results/kvdb_results.hpp | 13 +- include/stratosphere/results/utilities.h | 26 +- source/kvdb/kvdb_file_key_value_store.cpp | 351 +++++++++++++++ 9 files changed, 963 insertions(+), 33 deletions(-) create mode 100644 include/stratosphere/kvdb/kvdb_file_key_value_cache.hpp create mode 100644 include/stratosphere/kvdb/kvdb_file_key_value_store.hpp create mode 100644 source/kvdb/kvdb_file_key_value_store.cpp diff --git a/include/stratosphere/kvdb/kvdb_auto_buffer.hpp b/include/stratosphere/kvdb/kvdb_auto_buffer.hpp index 6fe115ad..269abcce 100644 --- a/include/stratosphere/kvdb/kvdb_auto_buffer.hpp +++ b/include/stratosphere/kvdb/kvdb_auto_buffer.hpp @@ -24,14 +24,19 @@ namespace sts::kvdb { class AutoBuffer { NON_COPYABLE(AutoBuffer); private: - std::unique_ptr buffer; + u8 *buffer; size_t size; public: - AutoBuffer() : size(0) { /* ... */ } + AutoBuffer() : buffer(nullptr), size(0) { /* ... */ } + + ~AutoBuffer() { + this->Reset(); + } AutoBuffer(AutoBuffer &&rhs) { - this->buffer = std::move(rhs.buffer); + this->buffer = rhs.buffer; this->size = rhs.size; + rhs.buffer = nullptr; rhs.size = 0; } @@ -46,12 +51,15 @@ namespace sts::kvdb { } void Reset() { - this->buffer.reset(); - this->size = 0; + if (this->buffer != nullptr) { + std::free(this->buffer); + this->buffer = nullptr; + this->size = 0; + } } u8 *Get() const { - return this->buffer.get(); + return this->buffer; } size_t GetSize() const { @@ -65,12 +73,10 @@ namespace sts::kvdb { } /* Allocate a buffer. */ - auto mem = new (std::nothrow) u8[size]; - if (mem == nullptr) { + this->buffer = static_cast(std::malloc(size)); + if (this->buffer == nullptr) { return ResultKvdbAllocationFailed; } - - this->buffer.reset(mem); this->size = size; return ResultSuccess; } @@ -80,7 +86,7 @@ namespace sts::kvdb { R_TRY(this->Initialize(size)); /* Copy the input data in. */ - std::memcpy(this->buffer.get(), buf, size); + std::memcpy(this->buffer, buf, size); return ResultSuccess; } diff --git a/include/stratosphere/kvdb/kvdb_bounded_string.hpp b/include/stratosphere/kvdb/kvdb_bounded_string.hpp index 16c02ef0..89272dfb 100644 --- a/include/stratosphere/kvdb/kvdb_bounded_string.hpp +++ b/include/stratosphere/kvdb/kvdb_bounded_string.hpp @@ -38,20 +38,20 @@ namespace sts::kvdb { } public: /* Constructors. */ - BoundedString() { + constexpr BoundedString() { buffer[0] = 0; } - explicit BoundedString(const char *s) { + explicit constexpr BoundedString(const char *s) { this->Set(s); } /* Static constructors. */ - static BoundedString Make(const char *s) { + static constexpr BoundedString Make(const char *s) { return BoundedString(s); } - static BoundedString MakeFormat(const char *format, ...) __attribute__((format (printf, 1, 2))) { + static constexpr BoundedString MakeFormat(const char *format, ...) __attribute__((format (printf, 1, 2))) { BoundedString string; std::va_list args; @@ -142,7 +142,7 @@ namespace sts::kvdb { return std::strncmp(this->buffer + offset, s, N - offset) == 0; } - bool EndsWith(const char *s) { + bool EndsWith(const char *s) const { const size_t suffix_length = strnlen(s, N); const size_t length = GetLength(); return suffix_length <= length && EndsWith(s, length - suffix_length); diff --git a/include/stratosphere/kvdb/kvdb_file_key_value_cache.hpp b/include/stratosphere/kvdb/kvdb_file_key_value_cache.hpp new file mode 100644 index 00000000..64a6037b --- /dev/null +++ b/include/stratosphere/kvdb/kvdb_file_key_value_cache.hpp @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "kvdb_bounded_string.hpp" +#include "kvdb_file_key_value_store.hpp" + +namespace sts::kvdb { + + namespace impl { + + template + class LruList { + private: + /* Subtypes. */ + struct LruHeader { + u32 entry_count; + }; + public: + static constexpr size_t BufferSize = sizeof(Key) * Capacity; + static constexpr size_t FileSize = sizeof(LruHeader) + BufferSize; + using Path = FileKeyValueStore::Path; + private: + Path file_path; + Key *keys; + LruHeader header; + public: + static Result CreateNewList(const char *path) { + /* Create new lru_list.dat. */ + R_TRY(fsdevCreateFile(path, FileSize, 0)); + + /* Open the file. */ + FILE *fp = fopen(path, "r+b"); + if (fp == nullptr) { + return fsdevGetLastResult(); + } + ON_SCOPE_EXIT { fclose(fp); }; + + /* Write new header with zero entries to the file. */ + LruHeader new_header = { .entry_count = 0, }; + if (fwrite(&new_header, sizeof(new_header), 1, fp) != 1) { + return fsdevGetLastResult(); + } + + return ResultSuccess; + } + private: + void RemoveIndex(size_t i) { + if (i >= this->GetCount()) { + std::abort(); + } + std::memmove(this->keys + i, this->keys + i + 1, sizeof(*this->keys) * (this->GetCount() - (i + 1))); + this->DecrementCount(); + } + + void IncrementCount() { + this->header.entry_count++; + } + + void DecrementCount() { + this->header.entry_count--; + } + public: + LruList() : keys(nullptr), header({}) { /* ... */ } + + Result Initialize(const char *path, void *buf, size_t size) { + /* Only initialize once, and ensure we have sufficient memory. */ + if (this->keys != nullptr || size < BufferSize) { + std::abort(); + } + + /* Setup member variables. */ + this->keys = static_cast(buf); + this->file_path.Set(path); + std::memset(this->keys, 0, BufferSize); + + /* Open file. */ + FILE *fp = fopen(this->file_path, "rb"); + if (fp == nullptr) { + return fsdevGetLastResult(); + } + ON_SCOPE_EXIT { fclose(fp); }; + + /* Read header. */ + if (fread(&this->header, sizeof(this->header), 1, fp) != 1) { + return fsdevGetLastResult(); + } + + /* Read entries. */ + const size_t count = this->GetCount(); + if (count > 0) { + if (fread(this->keys, std::min(BufferSize, sizeof(Key) * count), 1, fp) != 1) { + return fsdevGetLastResult(); + } + } + + return ResultSuccess; + } + + Result Save() { + /* Open file. */ + FILE *fp = fopen(this->file_path, "r+b"); + if (fp == nullptr) { + return fsdevGetLastResult(); + } + ON_SCOPE_EXIT { fclose(fp); }; + + /* Write header. */ + if (fwrite(&this->header, sizeof(this->header), 1, fp) != 1) { + return fsdevGetLastResult(); + } + + /* Write entries. */ + if (fwrite(this->keys, BufferSize, 1, fp) != 1) { + return fsdevGetLastResult(); + } + + /* Flush. */ + fflush(fp); + + return ResultSuccess; + } + + size_t GetCount() const { + return this->header.entry_count; + } + + bool IsEmpty() const { + return this->GetCount() == 0; + } + + bool IsFull() const { + return this->GetCount() >= Capacity; + } + + Key Get(size_t i) const { + if (i >= this->GetCount()) { + std::abort(); + } + return this->keys[i]; + } + + Key Peek() const { + if (this->IsEmpty()) { + std::abort(); + } + return this->Get(0); + } + + void Push(const Key &key) { + if (this->IsFull()) { + std::abort(); + } + this->keys[this->GetCount()] = key; + this->IncrementCount(); + } + + Key Pop() { + if (this->IsEmpty()) { + std::abort(); + } + this->RemoveIndex(0); + } + + bool Remove(const Key &key) { + const size_t count = this->GetCount(); + + /* Iterate over the list, removing the last entry that matches the key. */ + for (size_t i = 0; i < count; i++) { + if (this->keys[count - 1 - i] == key) { + this->RemoveIndex(count - 1 - i); + return true; + } + } + + return false; + } + + bool Contains(const Key &key) const { + const size_t count = this->GetCount(); + + /* Iterate over the list, checking to see if we have the key. */ + for (size_t i = 0; i < count; i++) { + if (this->keys[count - 1 - i] == key) { + return true; + } + } + + return false; + } + + bool Update(const Key &key) { + if (this->Remove(key)) { + this->Push(key); + return true; + } + + return false; + } + }; + + } + + template + class FileKeyValueCache { + static_assert(std::is_pod::value, "FileKeyValueCache Key must be pod!"); + static_assert(sizeof(Key) <= FileKeyValueStore::MaxKeySize, "FileKeyValueCache Key is too big!"); + public: + using LeastRecentlyUsedList = impl::LruList; + /* Note: Nintendo code in NS uses Path = BoundedString<0x180> here. */ + /* It's unclear why, since they use 0x300 everywhere else. */ + /* We'll just use 0x300, since it shouldn't make a difference, */ + /* as FileKeyValueStore paths are limited to 0x100 anyway. */ + using Path = typename LeastRecentlyUsedList::Path; + private: + FileKeyValueStore kvs; + LeastRecentlyUsedList lru_list; + private: + static constexpr Path GetLeastRecentlyUsedListPath(const char *dir) { + return Path::MakeFormat("%s/%s", dir, "lru_list.dat"); + } + + static constexpr Path GetFileKeyValueStorePath(const char *dir) { + return Path::MakeFormat("%s/%s", dir, "kvs"); + } + + static Result Exists(bool *out, const char *path, bool is_dir) { + /* Check if the path exists. */ + struct stat st; + if (stat(path, &st) != 0) { + R_TRY_CATCH(fsdevGetLastResult()) { + R_CATCH(ResultFsPathNotFound) { + /* If the path doesn't exist, nothing has gone wrong. */ + *out = false; + return ResultSuccess; + } + } R_END_TRY_CATCH; + } + + /* Check that our entry type is correct. */ + if ((is_dir && !(S_ISDIR(st.st_mode))) || (!is_dir && !(S_ISREG(st.st_mode)))) { + return ResultKvdbInvalidFilesystemState; + } + + *out = true; + return ResultSuccess; + } + + static Result DirectoryExists(bool *out, const char *path) { + return Exists(out, path, true); + } + + static Result FileExists(bool *out, const char *path) { + return Exists(out, path, false); + } + public: + static Result CreateNewCache(const char *dir) { + /* Make a new key value store filesystem, and a new lru_list.dat. */ + R_TRY(LeastRecentlyUsedList::CreateNewList(GetLeastRecentlyUsedListPath(dir))); + if (mkdir(GetFileKeyValueStorePath(dir), 0) != 0) { + return fsdevGetLastResult(); + } + + return ResultSuccess; + } + + static Result ValidateExistingCache(const char *dir) { + /* Check for existence. */ + bool has_lru = false, has_kvs = false; + R_TRY(FileExists(&has_lru, GetLeastRecentlyUsedListPath(dir))); + R_TRY(DirectoryExists(&has_kvs, GetFileKeyValueStorePath(dir))); + + /* If neither exists, CreateNewCache was never called. */ + if (!has_lru && !has_kvs) { + return ResultKvdbNotCreated; + } + + /* If one exists but not the other, we have an invalid state. */ + if (has_lru ^ has_kvs) { + return ResultKvdbInvalidFilesystemState; + } + + return ResultSuccess; + } + private: + void RemoveOldestKey() { + const Key &oldest_key = this->lru_list.Peek(); + this->lru_list.Pop(); + this->kvs.Remove(oldest_key); + } + public: + Result Initialize(const char *dir, void *buf, size_t size) { + /* Initialize list. */ + R_TRY(this->lru_list.Initialize(GetLeastRecentlyUsedListPath(dir), buf, size)); + + /* Initialize kvs. */ + /* NOTE: Despite creating the kvs folder and returning an error if it does not exist, */ + /* Nintendo does not use the kvs folder, and instead uses the passed dir. */ + /* This causes lru_list.dat to be in the same directory as the store's .val files */ + /* instead of in the same directory as a folder containing the store's .val files. */ + /* This is probably a Nintendo bug, but because system saves contain data in the wrong */ + /* layout it can't really be fixed without breaking existing devices... */ + R_TRY(this->kvs.Initialize(dir)); + + return ResultSuccess; + } + + size_t GetCount() const { + return this->lru_list.GetCount(); + } + + size_t GetCapacity() const { + return Capacity; + } + + Key GetKey(size_t i) const { + return this->lru_list.Get(i); + } + + bool Contains(const Key &key) const { + return this->lru_list.Contains(key); + } + + Result Get(size_t *out_size, void *out_value, size_t max_out_size, const Key &key) { + /* Note that we accessed the key. */ + this->lru_list.Update(key); + return this->kvs.Get(out_size, out_value, max_out_size, key); + } + + template + Result Get(Value *out_value, const Key &key) { + /* Note that we accessed the key. */ + this->lru_list.Update(key); + return this->kvs.Get(out_value, key); + } + + Result GetSize(size_t *out_size, const Key &key) { + return this->kvs.GetSize(out_size, key); + } + + Result Set(const Key &key, const void *value, size_t value_size) { + if (this->lru_list.Update(key)) { + /* If an entry for the key exists, delete the existing value file. */ + this->kvs.Remove(key); + } else { + /* If the list is full, we need to remove the oldest key. */ + if (this->lru_list.IsFull()) { + this->RemoveOldestKey(); + } + + /* Add the key to the list. */ + this->lru_list.Push(key); + } + + /* Loop, trying to save the new value to disk. */ + while (true) { + /* Try to set the key. */ + R_TRY_CATCH(this->kvs.Set(key, value, value_size)) { + R_CATCH_RANGE(ResultFsNotEnoughFreeSpace) { + /* If our entry is the only thing in the Lru list, remove it. */ + if (this->lru_list.GetCount() == 1) { + this->lru_list.Pop(); + R_TRY(this->lru_list.Save()); + return R_TRY_CATCH_RESULT; + } + + /* Otherwise, remove the oldest element from the cache and try again. */ + this->RemoveOldestKey(); + continue; + } + } R_END_TRY_CATCH; + + /* If we got here, we succeeded. */ + break; + } + + /* Save the list. */ + R_TRY(this->lru_list.Save()); + + return ResultSuccess; + } + + template + Result Set(const Key &key, const Value &value) { + return this->Set(key, &value, sizeof(Value)); + } + + Result Remove(const Key &key) { + /* Remove the key. */ + this->lru_list.Remove(key); + R_TRY(this->kvs.Remove(key)); + R_TRY(this->lru_list.Save()); + + return ResultSuccess; + } + + Result RemoveAll() { + /* TODO: Nintendo doesn't check errors here. Should we? */ + while (!this->lru_list.IsEmpty()) { + this->RemoveOldestKey(); + } + R_TRY(this->lru_list.Save()); + + return ResultSuccess; + } + }; + +} diff --git a/include/stratosphere/kvdb/kvdb_file_key_value_store.hpp b/include/stratosphere/kvdb/kvdb_file_key_value_store.hpp new file mode 100644 index 00000000..708de515 --- /dev/null +++ b/include/stratosphere/kvdb/kvdb_file_key_value_store.hpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +#include "kvdb_bounded_string.hpp" + +namespace sts::kvdb { + + class FileKeyValueStore { + NON_COPYABLE(FileKeyValueStore); + NON_MOVEABLE(FileKeyValueStore); + public: + static constexpr size_t MaxPathLength = 0x300; /* TODO: FS_MAX_PATH - 1? */ + static constexpr size_t MaxFileLength = 0xFF; + static constexpr char FileExtension[5] = ".val"; + static constexpr size_t FileExtensionLength = sizeof(FileExtension) - 1; + static constexpr size_t MaxKeySize = (MaxFileLength - FileExtensionLength) / 2; + using Path = kvdb::BoundedString; + using FileName = kvdb::BoundedString; + private: + /* Subtypes. */ + struct Entry { + u8 key[MaxKeySize]; + void *value; + size_t key_size; + size_t value_size; + }; + static_assert(std::is_pod::value, "FileKeyValueStore::Entry definition!"); + + class Cache { + private: + u8 *backing_buffer = nullptr; + size_t backing_buffer_size = 0; + size_t backing_buffer_free_offset = 0; + Entry *entries = nullptr; + size_t count = 0; + size_t capacity = 0; + private: + void *Allocate(size_t size); + + bool HasEntries() const { + return this->entries != nullptr && this->capacity != 0; + } + public: + Result Initialize(void *buffer, size_t buffer_size, size_t capacity); + void Invalidate(); + std::optional TryGet(void *out_value, size_t max_out_size, const void *key, size_t key_size); + std::optional TryGetSize(const void *key, size_t key_size); + void Set(const void *key, size_t key_size, const void *value, size_t value_size); + bool Contains(const void *key, size_t key_size); + }; + private: + HosMutex lock; + Path dir_path; + Cache cache; + private: + Path GetPath(const void *key, size_t key_size); + Result GetKey(size_t *out_size, void *out_key, size_t max_out_size, const FileName &file_name); + public: + FileKeyValueStore() { /* ... */ } + + /* Basic accessors. */ + Result Initialize(const char *dir); + Result InitializeWithCache(const char *dir, void *cache_buffer, size_t cache_buffer_size, size_t cache_capacity); + Result Get(size_t *out_size, void *out_value, size_t max_out_size, const void *key, size_t key_size); + Result GetSize(size_t *out_size, const void *key, size_t key_size); + Result Set(const void *key, size_t key_size, const void *value, size_t value_size); + Result Remove(const void *key, size_t key_size); + + /* Niceties. */ + template + Result Get(size_t *out_size, void *out_value, size_t max_out_size, const Key &key) { + static_assert(std::is_pod::value && sizeof(Key) <= MaxKeySize, "Invalid FileKeyValueStore Key!"); + return this->Get(out_size, out_value, max_out_size, &key, sizeof(Key)); + } + + template + Result Get(Value *out_value, const Key &key) { + static_assert(std::is_pod::value && !std::is_pointer::value, "Invalid FileKeyValueStore Value!"); + size_t size = 0; + R_TRY(this->Get(&size, out_value, sizeof(Value), key)); + if (size < sizeof(Value)) { + std::abort(); + } + return ResultSuccess; + } + + template + Result GetSize(size_t *out_size, const Key &key) { + return this->GetSize(out_size, &key, sizeof(Key)); + } + + template + Result Set(const Key &key, const void *value, size_t value_size) { + static_assert(std::is_pod::value && sizeof(Key) <= MaxKeySize, "Invalid FileKeyValueStore Key!"); + return this->Set(&key, sizeof(Key), value, value_size); + } + + template + Result Set(const Key &key, const Value &value) { + static_assert(std::is_pod::value && !std::is_pointer::value, "Invalid FileKeyValueStore Value!"); + return this->Set(key, &value, sizeof(Value)); + } + + template + Result Remove(const Key &key) { + return this->Remove(&key, sizeof(Key)); + } + }; + +} \ No newline at end of file diff --git a/include/stratosphere/kvdb/kvdb_memory_key_value_store.hpp b/include/stratosphere/kvdb/kvdb_memory_key_value_store.hpp index 984fe5f8..958e5000 100644 --- a/include/stratosphere/kvdb/kvdb_memory_key_value_store.hpp +++ b/include/stratosphere/kvdb/kvdb_memory_key_value_store.hpp @@ -150,7 +150,7 @@ namespace sts::kvdb { /* We need to add a new entry. Check we have room, move future keys forward. */ if (this->count >= this->capacity) { std::free(new_value); - return ResultKvdbTooManyKeys; + return ResultKvdbKeyCapacityInsufficient; } std::memmove(it + 1, it, sizeof(*it) * (this->end() - it)); this->count++; @@ -163,7 +163,7 @@ namespace sts::kvdb { Result AddUnsafe(const Key &key, void *value, size_t value_size) { if (this->count >= this->capacity) { - return ResultKvdbTooManyKeys; + return ResultKvdbKeyCapacityInsufficient; } this->entries[this->count++] = Entry(key, value, value_size); @@ -506,9 +506,12 @@ namespace sts::kvdb { /* Try to delete temporary archive, but allow deletion failure (it may not exist). */ std::remove(this->temp_path.Get()); + /* Create new temporary archive. */ + R_TRY(fsdevCreateFile(this->temp_path.Get(), buffer.GetSize(), 0)); + /* Write data to the temporary archive. */ { - FILE *f = fopen(this->temp_path, "wb"); + FILE *f = fopen(this->temp_path, "r+b"); if (f == nullptr) { return fsdevGetLastResult(); } diff --git a/include/stratosphere/results/fs_results.hpp b/include/stratosphere/results/fs_results.hpp index fc8c924d..4bb0b89a 100644 --- a/include/stratosphere/results/fs_results.hpp +++ b/include/stratosphere/results/fs_results.hpp @@ -25,6 +25,16 @@ static constexpr Result ResultFsPathAlreadyExists = MAKERESULT(Module_Fs, 2); static constexpr Result ResultFsTargetLocked = MAKERESULT(Module_Fs, 7); static constexpr Result ResultFsDirectoryNotEmpty = MAKERESULT(Module_Fs, 8); +static constexpr Result ResultFsNotEnoughFreeSpaceRangeStart = MAKERESULT(Module_Fs, 30); + static constexpr Result ResultFsNotEnoughFreeSpaceBisRangeStart = MAKERESULT(Module_Fs, 34); + static constexpr Result ResultFsNotEnoughFreeSpaceBisCalibration = MAKERESULT(Module_Fs, 35); + static constexpr Result ResultFsNotEnoughFreeSpaceBisSafe = MAKERESULT(Module_Fs, 36); + static constexpr Result ResultFsNotEnoughFreeSpaceBisUser = MAKERESULT(Module_Fs, 37); + static constexpr Result ResultFsNotEnoughFreeSpaceBisSystem = MAKERESULT(Module_Fs, 38); + static constexpr Result ResultFsNotEnoughFreeSpaceBisRangeEnd = MAKERESULT(Module_Fs, 39); + static constexpr Result ResultFsNotEnoughFreeSpaceSdCard = MAKERESULT(Module_Fs, 39); +static constexpr Result ResultFsNotEnoughFreeSpaceRangeEnd = MAKERESULT(Module_Fs, 45); + static constexpr Result ResultFsMountNameAlreadyExists = MAKERESULT(Module_Fs, 60); static constexpr Result ResultFsTargetNotFound = MAKERESULT(Module_Fs, 1002); diff --git a/include/stratosphere/results/kvdb_results.hpp b/include/stratosphere/results/kvdb_results.hpp index 97351914..9aef93a8 100644 --- a/include/stratosphere/results/kvdb_results.hpp +++ b/include/stratosphere/results/kvdb_results.hpp @@ -19,8 +19,11 @@ static constexpr u32 Module_Kvdb = 20; -static constexpr Result ResultKvdbTooManyKeys = MAKERESULT(Module_Kvdb, 1); -static constexpr Result ResultKvdbKeyNotFound = MAKERESULT(Module_Kvdb, 2); -static constexpr Result ResultKvdbAllocationFailed = MAKERESULT(Module_Kvdb, 4); -static constexpr Result ResultKvdbInvalidKeyValue = MAKERESULT(Module_Kvdb, 5); -static constexpr Result ResultKvdbBufferInsufficient = MAKERESULT(Module_Kvdb, 6); +static constexpr Result ResultKvdbKeyCapacityInsufficient = MAKERESULT(Module_Kvdb, 1); +static constexpr Result ResultKvdbKeyNotFound = MAKERESULT(Module_Kvdb, 2); +static constexpr Result ResultKvdbAllocationFailed = MAKERESULT(Module_Kvdb, 4); +static constexpr Result ResultKvdbInvalidKeyValue = MAKERESULT(Module_Kvdb, 5); +static constexpr Result ResultKvdbBufferInsufficient = MAKERESULT(Module_Kvdb, 6); + +static constexpr Result ResultKvdbInvalidFilesystemState = MAKERESULT(Module_Kvdb, 8); +static constexpr Result ResultKvdbNotCreated = MAKERESULT(Module_Kvdb, 9); \ No newline at end of file diff --git a/include/stratosphere/results/utilities.h b/include/stratosphere/results/utilities.h index 9a3d6c9a..da499496 100644 --- a/include/stratosphere/results/utilities.h +++ b/include/stratosphere/results/utilities.h @@ -37,40 +37,48 @@ extern "C" { }) /// Helpers for pattern-matching on a result expression, if the result would fail. +#define R_TRY_CATCH_RESULT _tmp_r_try_catch_rc + #define R_TRY_CATCH(res_expr) \ ({ \ - const Result _tmp_r_try_catch_rc = res_expr; \ - if (R_FAILED(_tmp_r_try_catch_rc)) { \ + const Result R_TRY_CATCH_RESULT = res_expr; \ + if (R_FAILED(R_TRY_CATCH_RESULT)) { \ if (false) #define R_CATCH(catch_result) \ - } else if (_tmp_r_try_catch_rc == catch_result) { \ + } else if (R_TRY_CATCH_RESULT == catch_result) { \ _Static_assert(R_FAILED(catch_result), "Catch result must be constexpr error Result!"); \ if (false) { } \ else -#define R_CATCH_RANGE(catch_result_start, catch_result_end) \ - } else if (catch_result_start <= _tmp_r_try_catch_rc && _tmp_r_try_catch_rc <= catch_result_end) { \ +#define R_GET_CATCH_RANGE_IMPL(_1, _2, NAME, ...) NAME + +#define R_CATCH_RANGE_IMPL_2(catch_result_start, catch_result_end) \ + } else if (catch_result_start <= R_TRY_CATCH_RESULT && R_TRY_CATCH_RESULT <= catch_result_end) { \ _Static_assert(R_FAILED(catch_result_start), "Catch start result must be constexpr error Result!"); \ _Static_assert(R_FAILED(catch_result_end), "Catch end result must be constexpr error Result!"); \ _Static_assert(R_MODULE(catch_result_start) == R_MODULE(catch_result_end), "Catch range modules must be equal!"); \ if (false) { } \ else +#define R_CATCH_RANGE_IMPL_1(catch_result) R_CATCH_RANGE_IMPL_2(catch_result##RangeStart, catch_result##RangeEnd) + +#define R_CATCH_RANGE(...) R_GET_CATCH_RANGE_IMPL(__VA_ARGS__, R_CATCH_RANGE_IMPL_2, R_CATCH_RANGE_IMPL_1)(__VA_ARGS__) + #define R_CATCH_MODULE(module) \ - } else if (R_MODULE(_tmp_r_try_catch_rc) == module) { \ + } else if (R_MODULE(R_TRY_CATCH_RESULT) == module) { \ _Static_assert(module != 0, "Catch module must be error!"); \ if (false) { } \ else #define R_CATCH_ALL() \ - } else if (R_FAILED(_tmp_r_try_catch_rc)) { \ + } else if (R_FAILED(R_TRY_CATCH_RESULT)) { \ if (false) { } \ else #define R_END_TRY_CATCH \ - else if (R_FAILED(_tmp_r_try_catch_rc)) { \ - return _tmp_r_try_catch_rc; \ + else if (R_FAILED(R_TRY_CATCH_RESULT)) { \ + return R_TRY_CATCH_RESULT; \ } \ } \ }) diff --git a/source/kvdb/kvdb_file_key_value_store.cpp b/source/kvdb/kvdb_file_key_value_store.cpp new file mode 100644 index 00000000..e1f77068 --- /dev/null +++ b/source/kvdb/kvdb_file_key_value_store.cpp @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +namespace sts::kvdb { + + /* Cache implementation. */ + void *FileKeyValueStore::Cache::Allocate(size_t size) { + if (this->backing_buffer_size - this->backing_buffer_free_offset < size) { + return nullptr; + } + ON_SCOPE_EXIT { this->backing_buffer_free_offset += size; }; + return this->backing_buffer + this->backing_buffer_free_offset; + } + + Result FileKeyValueStore::Cache::Initialize(void *buffer, size_t buffer_size, size_t capacity) { + this->backing_buffer = static_cast(buffer); + this->backing_buffer_size = buffer_size; + this->backing_buffer_free_offset = 0; + this->entries = nullptr; + this->count = 0; + this->capacity = capacity; + + /* If we have memory to work with, ensure it's at least enough for the cache entries. */ + if (this->backing_buffer != nullptr) { + this->entries = static_castentries)>(this->Allocate(sizeof(*this->entries) * this->capacity)); + if (this->entries == nullptr) { + return ResultKvdbBufferInsufficient; + } + } + + return ResultSuccess; + } + + void FileKeyValueStore::Cache::Invalidate() { + if (!this->HasEntries()) { + return; + } + + /* Reset the allocation pool. */ + this->backing_buffer_free_offset = 0; + this->count = 0; + this->entries = static_castentries)>(this->Allocate(sizeof(*this->entries) * this->capacity)); + if (this->entries == nullptr) { + std::abort(); + } + } + + std::optional FileKeyValueStore::Cache::TryGet(void *out_value, size_t max_out_size, const void *key, size_t key_size) { + if (!this->HasEntries()) { + return std::nullopt; + } + + /* Try to find the entry. */ + for (size_t i = 0; i < this->count; i++) { + const auto &entry = this->entries[i]; + if (entry.key_size == key_size && std::memcmp(entry.key, key, key_size) == 0) { + /* If we don't have enough space, fail to read from cache. */ + if (max_out_size < entry.value_size) { + return std::nullopt; + } + + std::memcpy(out_value, entry.value, entry.value_size); + return entry.value_size; + } + } + + return std::nullopt; + } + + std::optional FileKeyValueStore::Cache::TryGetSize(const void *key, size_t key_size) { + if (!this->HasEntries()) { + return std::nullopt; + } + + /* Try to find the entry. */ + for (size_t i = 0; i < this->count; i++) { + const auto &entry = this->entries[i]; + if (entry.key_size == key_size && std::memcmp(entry.key, key, key_size) == 0) { + return entry.value_size; + } + } + + return std::nullopt; + } + + void FileKeyValueStore::Cache::Set(const void *key, size_t key_size, const void *value, size_t value_size) { + if (!this->HasEntries()) { + return; + } + + /* Ensure key size is small enough. */ + if (key_size > MaxKeySize) { + std::abort(); + } + + /* If we're at capacity, invalidate the cache. */ + if (this->count == this->capacity) { + this->Invalidate(); + } + + /* Allocate memory for the value. */ + void *value_buf = this->Allocate(value_size); + if (value_buf == nullptr) { + /* We didn't have enough memory for the value. Invalidating might get us enough memory. */ + this->Invalidate(); + value_buf = this->Allocate(value_size); + if (value_buf == nullptr) { + /* If we still don't have enough memory, just fail to put the value in the cache. */ + return; + } + } + + auto &entry = this->entries[this->count++]; + std::memcpy(entry.key, key, key_size); + entry.key_size = key_size; + entry.value = value_buf; + std::memcpy(entry.value, value, value_size); + entry.value_size = value_size; + } + + bool FileKeyValueStore::Cache::Contains(const void *key, size_t key_size) { + return this->TryGetSize(key, key_size).has_value(); + } + + /* Store functionality. */ + FileKeyValueStore::Path FileKeyValueStore::GetPath(const void *_key, size_t key_size) { + /* Format is "/.val" */ + FileKeyValueStore::Path key_path(this->dir_path.Get()); + key_path.Append('/'); + + /* Append hex formatted key. */ + const u8 *key = static_cast(_key); + for (size_t i = 0; i < key_size; i++) { + key_path.AppendFormat("%02x", key[i]); + } + + /* Append extension. */ + key_path.Append(FileExtension); + + return key_path; + } + + Result FileKeyValueStore::GetKey(size_t *out_size, void *_out_key, size_t max_out_size, const FileKeyValueStore::FileName &file_name) { + /* Validate that the filename can be converted to a key. */ + /* TODO: Nintendo does not validate that the key is valid hex. Should we do this? */ + const size_t file_name_len = file_name.GetLength(); + const size_t key_name_len = file_name_len - FileExtensionLength; + if (file_name_len < FileExtensionLength + 2 || !file_name.EndsWith(FileExtension) || key_name_len % 2 != 0) { + return ResultKvdbInvalidKeyValue; + } + + /* Validate that we have space for the converted key. */ + const size_t key_size = key_name_len / 2; + if (key_size > max_out_size) { + return ResultKvdbBufferInsufficient; + } + + /* Convert the hex key back. */ + u8 *out_key = static_cast(_out_key); + for (size_t i = 0; i < key_size; i++) { + char substr[2 * sizeof(u8) + 1]; + file_name.GetSubstring(substr, sizeof(substr), 2 * i, sizeof(substr) - 1); + out_key[i] = static_cast(std::strtoul(substr, nullptr, 0x10)); + } + + *out_size = key_size; + return ResultSuccess; + } + + Result FileKeyValueStore::Initialize(const char *dir) { + return this->InitializeWithCache(dir, nullptr, 0, 0); + } + + Result FileKeyValueStore::InitializeWithCache(const char *dir, void *cache_buffer, size_t cache_buffer_size, size_t cache_capacity) { + /* Ensure that the passed path is a directory. */ + { + struct stat st; + if (stat(dir, &st) != 0 || !(S_ISDIR(st.st_mode))) { + return ResultFsPathNotFound; + } + } + + /* Set path. */ + this->dir_path.Set(dir); + + /* Initialize our cache. */ + R_TRY(this->cache.Initialize(cache_buffer, cache_buffer_size, cache_capacity)); + return ResultSuccess; + } + + Result FileKeyValueStore::Get(size_t *out_size, void *out_value, size_t max_out_size, const void *key, size_t key_size) { + std::scoped_lock lk(this->lock); + + /* Ensure key size is small enough. */ + if (key_size > MaxKeySize) { + return ResultKvdbKeyCapacityInsufficient; + } + + /* Try to get from cache. */ + { + auto size = this->cache.TryGet(out_value, max_out_size, key, key_size); + if (size) { + *out_size = *size; + return ResultSuccess; + } + } + + /* Open the value file. */ + FILE *fp = fopen(this->GetPath(key, key_size), "rb"); + if (fp == nullptr) { + R_TRY_CATCH(fsdevGetLastResult()) { + R_CATCH(ResultFsPathNotFound) { + return ResultKvdbKeyNotFound; + } + } R_END_TRY_CATCH; + } + ON_SCOPE_EXIT { fclose(fp); }; + + /* Get the value size. */ + fseek(fp, 0, SEEK_END); + const size_t value_size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + /* Ensure there's enough space for the value. */ + if (max_out_size < value_size) { + return ResultKvdbBufferInsufficient; + } + + /* Read the value. */ + if (fread(out_value, value_size, 1, fp) != 1) { + return fsdevGetLastResult(); + } + *out_size = value_size; + + /* Cache the newly read value. */ + this->cache.Set(key, key_size, out_value, value_size); + return ResultSuccess; + } + + Result FileKeyValueStore::GetSize(size_t *out_size, const void *key, size_t key_size) { + std::scoped_lock lk(this->lock); + + /* Ensure key size is small enough. */ + if (key_size > MaxKeySize) { + return ResultKvdbKeyCapacityInsufficient; + } + + /* Try to get from cache. */ + { + auto size = this->cache.TryGetSize(key, key_size); + if (size) { + *out_size = *size; + return ResultSuccess; + } + } + + /* Open the value file. */ + FILE *fp = fopen(this->GetPath(key, key_size), "rb"); + if (fp == nullptr) { + R_TRY_CATCH(fsdevGetLastResult()) { + R_CATCH(ResultFsPathNotFound) { + return ResultKvdbKeyNotFound; + } + } R_END_TRY_CATCH; + } + ON_SCOPE_EXIT { fclose(fp); }; + + /* Get the value size. */ + fseek(fp, 0, SEEK_END); + *out_size = ftell(fp); + return ResultSuccess; + } + + Result FileKeyValueStore::Set(const void *key, size_t key_size, const void *value, size_t value_size) { + std::scoped_lock lk(this->lock); + + /* Ensure key size is small enough. */ + if (key_size > MaxKeySize) { + return ResultKvdbKeyCapacityInsufficient; + } + + /* When the cache contains the key being set, Nintendo invalidates the cache. */ + if (this->cache.Contains(key, key_size)) { + this->cache.Invalidate(); + } + + /* Delete the file, if it exists. Don't check result, since it's okay if it's already deleted. */ + auto key_path = this->GetPath(key, key_size); + std::remove(key_path); + + /* Open the value file. */ + FILE *fp = fopen(key_path, "wb"); + if (fp == nullptr) { + return fsdevGetLastResult(); + } + ON_SCOPE_EXIT { fclose(fp); }; + + /* Write the value file. */ + if (fwrite(value, value_size, 1, fp) != 1) { + return fsdevGetLastResult(); + } + + /* Flush the value file. */ + fflush(fp); + + return ResultSuccess; + } + + Result FileKeyValueStore::Remove(const void *key, size_t key_size) { + std::scoped_lock lk(this->lock); + + /* Ensure key size is small enough. */ + if (key_size > MaxKeySize) { + return ResultKvdbKeyCapacityInsufficient; + } + + /* When the cache contains the key being set, Nintendo invalidates the cache. */ + if (this->cache.Contains(key, key_size)) { + this->cache.Invalidate(); + } + + /* Remove the file. */ + if (std::remove(this->GetPath(key, key_size)) != 0) { + R_TRY_CATCH(fsdevGetLastResult()) { + R_CATCH(ResultFsPathNotFound) { + return ResultKvdbKeyNotFound; + } + } R_END_TRY_CATCH; + } + + return ResultSuccess; + } + +} \ No newline at end of file