From 8fbb10a83cfc794e7860bec001a4a6e0089362be Mon Sep 17 00:00:00 2001 From: Adubbz Date: Tue, 17 Mar 2020 16:35:56 +1100 Subject: [PATCH] ncm: more work --- .../stratosphere/ncm/ncm_content_meta.hpp | 4 + .../ncm/ncm_install_progress_state.hpp | 2 +- .../stratosphere/ncm/ncm_install_task.hpp | 25 +- .../source/ncm/ncm_content_meta.cpp | 49 ++++ .../source/ncm/ncm_install_task.cpp | 271 +++++++++++++++++- .../include/vapours/results/ncm_results.hpp | 5 +- 6 files changed, 346 insertions(+), 10 deletions(-) diff --git a/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_meta.hpp b/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_meta.hpp index 9f651d2bb..3241d1a5f 100644 --- a/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_meta.hpp +++ b/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_meta.hpp @@ -336,6 +336,10 @@ namespace ams::ncm { class InstallContentMetaReader : public ContentMetaAccessor { public: constexpr InstallContentMetaReader(const void *data, size_t size) : ContentMetaAccessor(data, size) { /* ... */ } + + size_t CalculateConvertSize() const; + + void ConvertToContentMeta(void *dst, size_t size) const; }; class InstallContentMetaWriter : public ContentMetaAccessor { diff --git a/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_progress_state.hpp b/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_progress_state.hpp index 76fb93f80..682b656dd 100644 --- a/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_progress_state.hpp +++ b/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_progress_state.hpp @@ -23,7 +23,7 @@ namespace ams::ncm { DataPrepared = 1, Prepared = 2, Downloaded = 3, - Commited = 4, + Committed = 4, Fatal = 5, }; diff --git a/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_task.hpp b/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_task.hpp index 3d46eb6eb..48b6ab7fa 100644 --- a/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_task.hpp +++ b/libraries/libstratosphere/include/stratosphere/ncm/ncm_install_task.hpp @@ -32,6 +32,12 @@ PrepareContentMeta (both), WritePlaceHolderBuffer, Get/Delete InstallContentMeta NotCommitted = 2, }; + enum InstallConfig { + InstallConfig_SystemUpdate = (1 << 2), + InstallConfig_RequiresExFatDriver = (1 << 3), + InstallConfig_IgnoreTicket = (1 << 4), + }; + class InstallTaskBase { private: crypto::Sha256Generator sha256_generator; @@ -49,7 +55,16 @@ PrepareContentMeta (both), WritePlaceHolderBuffer, Get/Delete InstallContentMeta public: virtual ~InstallTaskBase() { /* TODO */ }; private: + ALWAYS_INLINE Result SetLastResultOnFailure(Result result) { + if (R_FAILED(result)) { + this->SetLastResult(result); + } + return result; + } + Result PrepareImpl(); + Result ExecuteImpl(); + Result CommitImpl(const StorageContentMetaKey *keys, s32 num_keys); protected: Result Initialize(StorageId install_storage, InstallTaskDataBase *data, u32 config); Result CountInstallContentMetaData(s32 *out_count); @@ -76,6 +91,12 @@ PrepareContentMeta (both), WritePlaceHolderBuffer, Get/Delete InstallContentMeta void CleanupProgress(); Result ListContentMetaKey(s32 *out_keys_written, StorageContentMetaKey *out_keys, s32 out_keys_count, s32 offset, ListContentMetaKeyFilter filter); Result ListApplicationContentMetaKey(s32 *out_keys_written, ApplicationContentMetaKey *out_keys, s32 out_keys_count, s32 offset); + Result Execute(); + void StartThroughputMeasurement(); + Result WritePlaceHolder(const ContentMetaKey &key, InstallContentInfo *content_info); + Result PrepareAndExecute(); + Result VerifyAllNotCommitted(const StorageContentMetaKey *keys, s32 num_keys); + Result Commit(const StorageContentMetaKey *keys, s32 num_keys); void ResetLastResult(); s64 GetThroughput(); @@ -92,8 +113,8 @@ PrepareContentMeta (both), WritePlaceHolderBuffer, Get/Delete InstallContentMeta virtual Result GetLatestVersion(std::optional *out_version, u64 id); virtual Result CheckInstallable(); virtual Result OnExecuteComplete(); - void *OnWritePlaceHolder; - void *InstallTicket; + virtual Result OnWritePlaceHolder(const ContentMetaKey &key, InstallContentInfo *content_info) = 0; + virtual Result InstallTicket(const fs::RightsId &rights_id, ContentMetaType meta_type) = 0; }; } diff --git a/libraries/libstratosphere/source/ncm/ncm_content_meta.cpp b/libraries/libstratosphere/source/ncm/ncm_content_meta.cpp index 6a57acd10..39ac6c728 100644 --- a/libraries/libstratosphere/source/ncm/ncm_content_meta.cpp +++ b/libraries/libstratosphere/source/ncm/ncm_content_meta.cpp @@ -30,6 +30,17 @@ namespace ams::ncm { dst->attributes = src.attributes; } + void ConvertInstallContentMetaHeaderToContentMetaHeader(ContentMetaHeader *dst, const InstallContentMetaHeader &src) { + /* Clear destination. */ + *dst = {}; + + /* Set converted fields. */ + dst->extended_header_size = src.extended_header_size; + dst->content_meta_count = src.content_meta_count; + dst->content_count = src.content_meta_count; + dst->attributes = src.attributes; + } + } size_t PackagedContentMetaReader::CountDeltaFragments() const { @@ -98,4 +109,42 @@ namespace ams::ncm { } } + size_t InstallContentMetaReader::CalculateConvertSize() const { + return CalculateSizeImpl(this->GetExtendedHeaderSize(), this->GetContentCount(), this->GetContentMetaCount(), this->GetExtendedDataSize(), false); + } + + void InstallContentMetaReader::ConvertToContentMeta(void *dst, size_t size) const { + /* Ensure we have enough space to convert. */ + AMS_ABORT_UNLESS(size >= this->CalculateConvertSize()); + + /* Prepare for conversion. */ + const auto *install_header = this->GetHeader(); + uintptr_t dst_addr = reinterpret_cast(dst); + + /* Convert the header. */ + ContentMetaHeader header; + ConvertInstallContentMetaHeaderToContentMetaHeader(std::addressof(header), *install_header); + + /* Copy the header. */ + std::memcpy(reinterpret_cast(dst_addr), std::addressof(header), sizeof(header)); + dst_addr += sizeof(header); + + /* Copy the extended header. */ + std::memcpy(reinterpret_cast(dst_addr), reinterpret_cast(this->GetExtendedHeaderAddress()), install_header->extended_header_size); + dst_addr += install_header->extended_header_size; + + /* Copy content infos. */ + for (size_t i = 0; i < this->GetContentCount(); i++) { + /* Copy the current info. */ + std::memcpy(reinterpret_cast(dst_addr), std::addressof(this->GetContentInfo(i)->info), sizeof(ContentInfo)); + dst_addr += sizeof(ContentInfo); + } + + /* Copy content meta infos. */ + for (size_t i = 0; i < this->GetContentMetaCount(); i++) { + std::memcpy(reinterpret_cast(dst_addr), this->GetContentMetaInfo(i), sizeof(ContentMetaInfo)); + dst_addr += sizeof(ContentMetaInfo); + } + } + } diff --git a/libraries/libstratosphere/source/ncm/ncm_install_task.cpp b/libraries/libstratosphere/source/ncm/ncm_install_task.cpp index 1ae93863c..010fd30d6 100644 --- a/libraries/libstratosphere/source/ncm/ncm_install_task.cpp +++ b/libraries/libstratosphere/source/ncm/ncm_install_task.cpp @@ -17,6 +17,23 @@ namespace ams::ncm { + namespace { + + bool Contains(const StorageContentMetaKey *keys, s32 num_keys, const ContentMetaKey &key, StorageId storage_id) { + for (s32 i = 0; i < num_keys; i++) { + const StorageContentMetaKey &storage_key = keys[i]; + + /* Check if the key matches the input key and storage id. */ + if (storage_key.key == key && storage_key.storage_id == storage_id) { + return true; + } + } + + return false; + } + + } + Result InstallTaskBase::OnPrepareComplete() { return ResultSuccess(); } @@ -57,12 +74,8 @@ namespace ams::ncm { } Result InstallTaskBase::Prepare() { - /* Call the implementation. */ - Result result = this->PrepareImpl(); - - /* Update the last result. */ - this->SetLastResult(result); - return result; + R_TRY(this->SetLastResultOnFailure(this->PrepareImpl())); + return ResultSuccess(); } Result InstallTaskBase::PrepareImpl() { @@ -412,6 +425,251 @@ namespace ams::ncm { return ResultSuccess(); } + Result InstallTaskBase::Execute() { + R_TRY(this->SetLastResultOnFailure(this->ExecuteImpl())); + return ResultSuccess(); + } + + Result InstallTaskBase::ExecuteImpl() { + this->StartThroughputMeasurement(); + + /* Count the number of content meta entries. */ + s32 count; + R_TRY(this->data->Count(std::addressof(count))); + + /* Iterate over content meta. */ + for (s32 i = 0; i < count; i++) { + /* Obtain the content meta. */ + InstallContentMeta content_meta; + R_TRY(this->data->Get(&content_meta, i)); + + /* Update the data when we are done. */ + ON_SCOPE_EXIT { this->data->Update(content_meta, i); }; + + /* Create a writer. */ + const auto writer = content_meta.GetWriter(); + + /* Iterate over content infos. */ + for (size_t j = 0; j < writer.GetContentCount(); j++) { + auto *content_info = writer.GetWritableContentInfo(j); + + /* Write prepared content infos. */ + if (content_info->install_state == InstallState::Prepared) { + R_TRY(this->WritePlaceHolder(writer.GetKey(), content_info)); + content_info->install_state = InstallState::Installed; + } + } + } + + /* Execution has finished, signal this and update the state. */ + R_TRY(this->OnExecuteComplete()); + this->SetProgressState(InstallProgressState::Downloaded); + return ResultSuccess(); + } + + void InstallTaskBase::StartThroughputMeasurement() { + std::scoped_lock lk(this->throughput_mutex); + this->throughput.installed = 0; + this->throughput.elapsed_time = TimeSpan(); + this->throughput_start_time = os::ConvertToTimeSpan(os::GetSystemTick()); + } + + Result InstallTaskBase::WritePlaceHolder(const ContentMetaKey &key, InstallContentInfo *content_info) { + if (content_info->is_sha256_calculated) { + /* Update the hash with the buffered data. */ + this->sha256_generator.InitializeWithContext(std::addressof(content_info->context)); + this->sha256_generator.Update(content_info->buffered_data, content_info->buffered_data_size); + } else { + /* Initialize the generator. */ + this->sha256_generator.Initialize(); + } + + { + ON_SCOPE_EXIT { + /* Update this content info's sha256 data. */ + this->sha256_generator.GetContext(std::addressof(content_info->context)); + content_info->buffered_data_size = this->sha256_generator.GetBufferedDataSize(); + this->sha256_generator.GetBufferedData(content_info->buffered_data, this->sha256_generator.GetBufferedDataSize()); + content_info->is_sha256_calculated = true; + }; + + /* Perform the placeholder write. */ + R_TRY(this->OnWritePlaceHolder(key, content_info)); + } + + /* Compare generated hash to expected hash if verification required. */ + if (content_info->verify_hash) { + u8 hash[crypto::Sha256Generator::HashSize]; + this->sha256_generator.GetHash(hash, crypto::Sha256Generator::HashSize); + R_UNLESS(std::memcmp(hash, content_info->digest.data, crypto::Sha256Generator::HashSize) == 0, ncm::ResultInvalidContentHash()); + } + + if (!(this->config & InstallConfig_IgnoreTicket)) { + ncm::RightsId rights_id; + { + /* Open the content storage and obtain the rights id. */ + ncm::ContentStorage storage; + R_TRY(OpenContentStorage(std::addressof(storage), content_info->storage_id)); + R_TRY(storage.GetRightsId(std::addressof(rights_id), content_info->placeholder_id)); + } + + /* Install a ticket if necessary. */ + if (this->IsNecessaryInstallTicket(rights_id.id)) { + R_TRY_CATCH(this->InstallTicket(rights_id.id, content_info->meta_type)) { + R_CATCH(ncm::ResultIgnorableInstallTicketFailure) { /* We can ignore the installation failure. */ } + } R_END_TRY_CATCH; + } + } + + return ResultSuccess(); + } + + Result InstallTaskBase::PrepareAndExecute() { + R_TRY(this->SetLastResultOnFailure(this->PrepareImpl())); + R_TRY(this->SetLastResultOnFailure(this->ExecuteImpl())); + return ResultSuccess(); + } + + Result InstallTaskBase::VerifyAllNotCommitted(const StorageContentMetaKey *keys, s32 num_keys) { + /* No keys to check. */ + R_SUCCEED_IF(keys == nullptr); + + /* Count the number of content meta entries. */ + s32 count; + R_TRY(this->data->Count(std::addressof(count))); + + s32 num_not_committed = 0; + + /* Iterate over content meta. */ + for (s32 i = 0; i < count; i++) { + /* Obtain the content meta. */ + InstallContentMeta content_meta; + R_TRY(this->data->Get(&content_meta, i)); + + /* Create a reader. */ + const auto reader = content_meta.GetReader(); + + if (Contains(keys, num_keys, reader.GetKey(), reader.GetStorageId())) { + /* Ensure content meta isn't committed. */ + R_UNLESS(!reader.GetHeader()->committed, ncm::ResultListPartiallyNotCommitted()); + num_not_committed++; + } + } + + /* Ensure number of uncommitted keys equals the number of input keys. */ + R_UNLESS(num_not_committed == num_keys, ncm::ResultListPartiallyNotCommitted()); + return ResultSuccess(); + } + + Result InstallTaskBase::CommitImpl(const StorageContentMetaKey *keys, s32 num_keys) { + /* Ensure progress state is Downloaded. */ + R_UNLESS(this->GetProgress().state == InstallProgressState::Downloaded, ncm::ResultInvalidInstallTaskState()); + + /* Ensure keys aren't committed. */ + R_TRY(this->VerifyAllNotCommitted(keys, num_keys)); + + /* Count the number of content meta entries. */ + s32 count; + R_TRY(this->data->Count(std::addressof(count))); + + /* List of storages to commit. */ + StorageList commit_list; + + /* Iterate over content meta. */ + for (s32 i = 0; i < count; i++) { + /* Obtain the content meta. */ + InstallContentMeta content_meta; + R_TRY(this->data->Get(&content_meta, i)); + + /* Create a reader. */ + const auto reader = content_meta.GetReader(); + const auto cur_key = reader.GetKey(); + const auto storage_id = reader.GetStorageId(); + const size_t convert_size = reader.CalculateConvertSize(); + + /* Skip content meta not contained in input keys. */ + if (keys != nullptr && !Contains(keys, num_keys, cur_key, storage_id)) { + continue; + } + + /* Skip already committed. This check is primarily for if keys is nullptr. */ + if (reader.GetHeader()->committed) { + continue; + } + + /* Helper for performing an update. */ + const auto DoUpdate = [&]() ALWAYS_INLINE_LAMBDA { return this->data->Update(content_meta, i); }; + + /* Commit the current meta. */ + { + /* Ensure that if something goes wrong during commit, we still try to update. */ + auto update_guard = SCOPE_GUARD { DoUpdate(); }; + + /* Open a writer. */ + const auto writer = content_meta.GetWriter(); + + /* Convert to content meta and store to a buffer. */ + std::unique_ptr content_meta_buffer(new (std::nothrow) char[convert_size]); + R_UNLESS(content_meta_buffer != nullptr, ncm::ResultAllocationFailed()); + reader.ConvertToContentMeta(content_meta_buffer.get(), convert_size); + + /* Open the content storage for this meta. */ + ContentStorage content_storage; + R_TRY(OpenContentStorage(&content_storage, storage_id)); + + /* Open the content meta database for this meta. */ + ContentMetaDatabase meta_db; + R_TRY(OpenContentMetaDatabase(std::addressof(meta_db), storage_id)); + + /* Iterate over content infos. */ + for (size_t j = 0; j < reader.GetContentCount(); j++) { + const auto *content_info = reader.GetContentInfo(j); + + /* Register non-existing content infos. */ + if (content_info->install_state != InstallState::AlreadyExists) { + R_TRY(content_storage.Register(content_info->placeholder_id, content_info->info.content_id)); + } + } + + /* Store the content meta. */ + R_TRY(meta_db.Set(reader.GetKey(), content_meta_buffer.get(), convert_size)); + + /* Mark as committed. */ + writer.GetWritableHeader()->committed = true; + + /* Mark storage id to be committed later. */ + commit_list.Push(reader.GetStorageId()); + + /* We successfully commited this meta, so we want to check for errors when updating. */ + update_guard.Cancel(); + } + + /* Try to update, checking for failure. */ + R_TRY(DoUpdate()); + } + + /* Commit all applicable content meta databases. */ + for (s32 i = 0; i < commit_list.Count(); i++) { + ContentMetaDatabase meta_db; + R_TRY(OpenContentMetaDatabase(std::addressof(meta_db), commit_list[i])); + R_TRY(meta_db.Commit()); + } + + /* Change progress state to committed if keys are nullptr. */ + if (keys == nullptr) { + this->SetProgressState(InstallProgressState::Committed); + } + + return ResultSuccess(); + } + + Result InstallTaskBase::Commit(const StorageContentMetaKey *keys, s32 num_keys) { + auto fatal_guard = SCOPE_GUARD { SetProgressState(InstallProgressState::Fatal); }; + R_TRY(this->SetLastResultOnFailure(this->CommitImpl(keys, num_keys))); + fatal_guard.Cancel(); + return ResultSuccess(); + } + /* ... */ void InstallTaskBase::IncrementProgress(s64 size) { @@ -422,6 +680,7 @@ namespace ams::ncm { void InstallTaskBase::UpdateThroughputMeasurement(s64 throughput) { std::scoped_lock lk(this->throughput_mutex); + /* Update throughput only if start time has been set. */ if (this->throughput_start_time.GetNanoSeconds() != 0) { this->throughput.installed += throughput; this->throughput.elapsed_time = os::ConvertToTimeSpan(os::GetSystemTick()) - this->throughput_start_time; diff --git a/libraries/libvapours/include/vapours/results/ncm_results.hpp b/libraries/libvapours/include/vapours/results/ncm_results.hpp index d597c0d94..4bbb693c1 100644 --- a/libraries/libvapours/include/vapours/results/ncm_results.hpp +++ b/libraries/libvapours/include/vapours/results/ncm_results.hpp @@ -32,16 +32,19 @@ namespace ams::ncm { R_DEFINE_ERROR_RESULT(InvalidContentStorage, 100); R_DEFINE_ERROR_RESULT(InvalidContentMetaDatabase, 110); - R_DEFINE_ERROR_RESULT(InvalidPackageFormat, 130); + R_DEFINE_ERROR_RESULT(InvalidContentHash, 140); + R_DEFINE_ERROR_RESULT(InvalidInstallTaskState, 160); R_DEFINE_ERROR_RESULT(InvalidPlaceHolderFile, 170); R_DEFINE_ERROR_RESULT(BufferInsufficient, 180); R_DEFINE_ERROR_RESULT(WriteToReadOnlyContentStorage, 190); R_DEFINE_ERROR_RESULT(NotEnoughInstallSpace, 200); R_DEFINE_ERROR_RESULT(InvalidContentMetaKey, 240); + R_DEFINE_ERROR_RESULT(IgnorableInstallTicketFailure, 280); R_DEFINE_ERROR_RESULT(ContentStorageBaseNotFound, 310); + R_DEFINE_ERROR_RESULT(ListPartiallyNotCommitted, 330); R_DEFINE_ERROR_RANGE(ContentStorageNotActive, 250, 258); R_DEFINE_ERROR_RESULT(GameCardContentStorageNotActive, 251);