From 4a22efc7619ab775dfdff89cbb9d02e450717a54 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Sat, 11 Jun 2022 16:54:23 -0700 Subject: [PATCH] hac2l: add --onlyupdated, --updatedgen=n support for romfs print/extract --- source/hactool_fs_utils.cpp | 241 +++++++++++++++++++++++++++++++ source/hactool_fs_utils.hpp | 4 + source/hactool_options.cpp | 2 + source/hactool_options.hpp | 2 + source/hactool_processor.nca.cpp | 14 +- 5 files changed, 261 insertions(+), 2 deletions(-) diff --git a/source/hactool_fs_utils.cpp b/source/hactool_fs_utils.cpp index a66b706..ef2af0f 100644 --- a/source/hactool_fs_utils.cpp +++ b/source/hactool_fs_utils.cpp @@ -140,6 +140,106 @@ namespace ams::hactool { R_RETURN(iter_result); } + Result PrintUpdatedRomFsDirectory(fssystem::RomFsFileSystem *fs, std::shared_ptr &indirect, std::shared_ptr &aes_ctr_ex, s32 min_gen, const char *prefix, const char *path) { + /* Get the fs path. */ + ams::fs::Path fs_path; + R_UNLESS(path != nullptr, fs::ResultNullptrArgument()); + R_TRY(fs_path.SetShallowBuffer(path)); + + /* Allocate a work buffer. */ + constexpr size_t EntryBufferSize = 2_MB; + void *buffer = std::malloc(EntryBufferSize); + if (buffer == nullptr) { + fprintf(stderr, "[Warning]: Failed to allocate work buffer to print updated romfs directory (%s%s)!\n", prefix, path); + R_SUCCEED(); + } + ON_SCOPE_EXIT { std::free(buffer); }; + + /* Set up entry buffers. */ + auto *indirect_entries = reinterpret_cast(reinterpret_cast(buffer) + 0); + const auto max_indirect_entries = (EntryBufferSize / 2) / sizeof(*indirect_entries); + + auto *aes_ctr_ex_entries = reinterpret_cast(reinterpret_cast(buffer) + (EntryBufferSize / 2)); + const auto max_aes_ctr_ex_entries = (EntryBufferSize / 2) / sizeof(*aes_ctr_ex_entries); + + /* Iterate, printing the contents of the directory. */ + const auto iter_result = fssystem::IterateDirectoryRecursively(fs, + fs_path, + [&] (const fs::Path &, const fs::DirectoryEntry &) -> Result { + R_SUCCEED(); + }, + [&] (const fs::Path &, const fs::DirectoryEntry &) -> Result { + R_SUCCEED(); + }, + [&] (const fs::Path &path, const fs::DirectoryEntry &ent) -> Result { + /* Get the file base offset. */ + s64 file_offset = 0; + R_TRY(fs->GetFileBaseOffset(std::addressof(file_offset), path)); + + /* We'll want to get the maximum generation that the file was updated in. */ + s32 max_gen = 0; + bool was_updated = false; + + /* Get the indirect entries. */ + s32 indirect_count = 0; + R_TRY(indirect->GetEntryList(indirect_entries, std::addressof(indirect_count), max_indirect_entries, file_offset, ent.file_size)); + + /* Check all indirect entries. */ + s64 offset_in_file = 0; + for (auto i = 0; i < indirect_count; ++i) { + /* Get the current entry. */ + const auto &indirect_entry = indirect_entries[i]; + + /* Determine the current entry size, which we'll advance by after this iteration. */ + size_t cur_size = ent.file_size - offset_in_file; + if (i + 1 < indirect_count) { + cur_size = std::min(indirect_entries[i + 1].GetVirtualOffset() - std::max(file_offset, indirect_entry.GetVirtualOffset()), cur_size); + } + ON_SCOPE_EXIT { offset_in_file += cur_size; }; + + /* We should have non-zero in this entry. */ + AMS_ABORT_UNLESS(cur_size > 0); + + /* If the entry is from the base storage, skip it. */ + if (indirect_entry.storage_index == 0) { + continue; + } + + /* The file has been updated at least once. */ + was_updated = true; + + /* Now, let's find the generation it was updated in. */ + { + /* Get the aes ctr ex entries. */ + s32 aes_ctr_ex_count = 0; + R_TRY(aes_ctr_ex->GetEntryList(aes_ctr_ex_entries, std::addressof(aes_ctr_ex_count), max_aes_ctr_ex_entries, indirect_entry.GetPhysicalOffset(), cur_size)); + + /* Check all aes ctr ex entries. */ + for (auto j = 0; j < aes_ctr_ex_count; ++j) { + /* Get the current entry. */ + const auto &aes_ctr_ex_entry = aes_ctr_ex_entries[j]; + + /* Check the entry's generation. */ + max_gen = std::max(max_gen, aes_ctr_ex_entry.generation); + } + } + } + + /* If we should, print. */ + if (was_updated && max_gen >= min_gen) { + printf("[%02d] %s%s\n", max_gen, prefix, path.GetString()); + } + + R_SUCCEED(); + } + ); + if (R_FAILED(iter_result)) { + fprintf(stderr, "[Warning]: Failed to print updated romfs directory (%s): 2%03d-%04d\n", path, iter_result.GetModule(), iter_result.GetDescription()); + } + + R_RETURN(iter_result); + } + Result ExtractDirectory(std::shared_ptr &dst_fs, std::shared_ptr &src_fs, const char *prefix, const char *dst_path, const char *src_path) { /* Allocate a work buffer. */ void *buffer = std::malloc(WorkBufferSize); @@ -300,6 +400,147 @@ namespace ams::hactool { R_RETURN(res); } + Result ExtractUpdatedRomFsDirectory(std::shared_ptr &dst_fs, fssystem::RomFsFileSystem *src_fs, std::shared_ptr &indirect, std::shared_ptr &aes_ctr_ex, s32 min_gen, const char *prefix, const char *dst_path, const char *src_path) { + /* Allocate a work buffer. */ + void *buffer = std::malloc(WorkBufferSize); + if (buffer == nullptr) { + fprintf(stderr, "[Warning]: Failed to allocate work buffer to extract %s%s to %s!\n", prefix, src_path, dst_path); + R_SUCCEED(); + } + ON_SCOPE_EXIT { std::free(buffer); }; + + /* Allocate a work buffer. */ + constexpr size_t EntryBufferSize = 2_MB; + void *entry_buffer = std::malloc(EntryBufferSize); + if (entry_buffer == nullptr) { + fprintf(stderr, "[Warning]: Failed to allocate work buffer to extract updated romfs directory %s%s to %s!\n", prefix, src_path, dst_path); + R_SUCCEED(); + } + ON_SCOPE_EXIT { std::free(entry_buffer); }; + + /* Set up entry buffers. */ + auto *indirect_entries = reinterpret_cast(reinterpret_cast(entry_buffer) + 0); + const auto max_indirect_entries = (EntryBufferSize / 2) / sizeof(*indirect_entries); + + auto *aes_ctr_ex_entries = reinterpret_cast(reinterpret_cast(entry_buffer) + (EntryBufferSize / 2)); + const auto max_aes_ctr_ex_entries = (EntryBufferSize / 2) / sizeof(*aes_ctr_ex_entries); + + auto extract_impl = [&] () -> Result { + /* Set up the destination work path to point at the target directory. */ + fs::Path dst_fs_path; + R_TRY(dst_fs_path.SetShallowBuffer(dst_path)); + + /* Try to create the destination directory. */ + dst_fs->CreateDirectory(dst_fs_path); + + /* Verify that we can open the directory on the base filesystem. */ + { + std::unique_ptr sub_dir; + R_TRY(dst_fs->OpenDirectory(std::addressof(sub_dir), dst_fs_path, fs::OpenDirectoryMode_Directory)); + } + + /* Create/Initialize subdirectory filesystem. */ + fssystem::SubDirectoryFileSystem subdir_fs{dst_fs}; + R_TRY(subdir_fs.Initialize(dst_fs_path)); + + /* Set up the source path to point at the target directory. */ + fs::Path src_fs_path; + R_TRY(src_fs_path.SetShallowBuffer(src_path)); + + /* Iterate, copying files. */ + R_RETURN(fssystem::IterateDirectoryRecursively(src_fs, src_fs_path, + [&](const fs::Path &path, const fs::DirectoryEntry &) -> Result { /* On Enter Directory */ + /* Create the directory. */ + R_TRY_CATCH(subdir_fs.CreateDirectory(path)) { + R_CATCH(fs::ResultPathAlreadyExists) { /* ... */ } + } R_END_TRY_CATCH; + + R_SUCCEED(); + }, + [&](const fs::Path &path, const fs::DirectoryEntry &) -> Result { /* On Exit Directory */ + /* If the directory has no files, we need to delete it. */ + R_TRY_CATCH(subdir_fs.DeleteDirectory(path)) { + R_CATCH(fs::ResultDirectoryNotEmpty) { /* ... */ } + } R_END_TRY_CATCH; + + R_SUCCEED(); + }, + [&](const fs::Path &path, const fs::DirectoryEntry &ent) -> Result { /* On File */ + /* Delete a file, if one already exists. */ + subdir_fs.DeleteFile(path); + + /* We'll want to get the maximum generation that the file was updated in. */ + s32 max_gen = 0; + bool was_updated = false; + { + /* Get the file base offset. */ + s64 file_offset = 0; + R_TRY(src_fs->GetFileBaseOffset(std::addressof(file_offset), path)); + + /* Get the indirect entries. */ + s32 indirect_count = 0; + R_TRY(indirect->GetEntryList(indirect_entries, std::addressof(indirect_count), max_indirect_entries, file_offset, ent.file_size)); + + /* Check all indirect entries. */ + s64 offset_in_file = 0; + for (auto i = 0; i < indirect_count; ++i) { + /* Get the current entry. */ + const auto &indirect_entry = indirect_entries[i]; + + /* Determine the current entry size, which we'll advance by after this iteration. */ + size_t cur_size = ent.file_size - offset_in_file; + if (i + 1 < indirect_count) { + cur_size = std::min(indirect_entries[i + 1].GetVirtualOffset() - std::max(file_offset, indirect_entry.GetVirtualOffset()), cur_size); + } + ON_SCOPE_EXIT { offset_in_file += cur_size; }; + + /* We should have non-zero in this entry. */ + AMS_ABORT_UNLESS(cur_size > 0); + + /* If the entry is from the base storage, skip it. */ + if (indirect_entry.storage_index == 0) { + continue; + } + + /* The file has been updated at least once. */ + was_updated = true; + + /* Now, let's find the generation it was updated in. */ + { + /* Get the aes ctr ex entries. */ + s32 aes_ctr_ex_count = 0; + R_TRY(aes_ctr_ex->GetEntryList(aes_ctr_ex_entries, std::addressof(aes_ctr_ex_count), max_aes_ctr_ex_entries, indirect_entry.GetPhysicalOffset(), cur_size)); + + /* Check all aes ctr ex entries. */ + for (auto j = 0; j < aes_ctr_ex_count; ++j) { + /* Get the current entry. */ + const auto &aes_ctr_ex_entry = aes_ctr_ex_entries[j]; + + /* Check the entry's generation. */ + max_gen = std::max(max_gen, aes_ctr_ex_entry.generation); + } + } + } + } + + /* If we should, copy the file. */ + if (was_updated && max_gen >= min_gen) { + printf("Saving [%02d] %s%s...\n", max_gen, prefix, path.GetString()); + R_TRY(fssystem::CopyFile(std::addressof(subdir_fs), src_fs, path, path, buffer, WorkBufferSize)); + } + + R_SUCCEED(); + } + )); + }; + + const auto res = extract_impl(); + if (R_FAILED(res)) { + fprintf(stderr, "[Warning]: Failed to extract %s%s to %s: 2%03d-%04d\n", prefix, src_path, dst_path, res.GetModule(), res.GetDescription()); + } + R_RETURN(res); + } + Result SaveToFile(std::shared_ptr &fs, const char *path, fs::IStorage *storage, s64 offset, size_t size) { /* Allocate a work buffer. */ void *buffer = std::malloc(WorkBufferSize); diff --git a/source/hactool_fs_utils.hpp b/source/hactool_fs_utils.hpp index 0b5af76..f116172 100644 --- a/source/hactool_fs_utils.hpp +++ b/source/hactool_fs_utils.hpp @@ -33,9 +33,13 @@ namespace ams::hactool { Result PrintDirectory(std::shared_ptr &fs, const char *prefix, const char *path); + Result PrintUpdatedRomFsDirectory(fssystem::RomFsFileSystem *fs, std::shared_ptr &indirect, std::shared_ptr &aes_ctr_ex, s32 min_gen, const char *prefix, const char *path); + Result ExtractDirectory(std::shared_ptr &dst_fs, std::shared_ptr &src_fs, const char *prefix, const char *dst_path, const char *src_path); Result ExtractDirectoryWithProgress(std::shared_ptr &dst_fs, std::shared_ptr &src_fs, const char *prefix, const char *dst_path, const char *src_path); + Result ExtractUpdatedRomFsDirectory(std::shared_ptr &dst_fs, fssystem::RomFsFileSystem *src_fs, std::shared_ptr &indirect, std::shared_ptr &aes_ctr_ex, s32 min_gen, const char *prefix, const char *dst_path, const char *src_path); + Result SaveToFile(std::shared_ptr &fs, const char *path, fs::IStorage *storage, s64 offset, size_t size); Result SaveToFile(std::shared_ptr &fs, const char *path, fs::IStorage *storage); diff --git a/source/hactool_options.cpp b/source/hactool_options.cpp index cb5cca3..82c8d2d 100644 --- a/source/hactool_options.cpp +++ b/source/hactool_options.cpp @@ -136,6 +136,7 @@ namespace ams::hactool { MakeOptionHandler("raw", 'r', [] (Options &options) { options.raw = true; }), MakeOptionHandler("enablehash", 'h', [] (Options &options) { options.enable_hash = true; }), MakeOptionHandler("disablekeywarns", [] (Options &options) { options.disable_key_warns = true; }), + MakeOptionHandler("onlyupdated", [] (Options &options) { options.only_updated = true; }), MakeOptionHandler("keyset", 'k', [] (Options &options, const char *arg) { return CreateFilePath(std::addressof(options.key_file_path), arg); }), MakeOptionHandler("titlekeys", [] (Options &options, const char *arg) { return CreateFilePath(std::addressof(options.titlekey_path), arg); }), MakeOptionHandler("consolekeys", [] (Options &options, const char *arg) { return CreateFilePath(std::addressof(options.consolekey_path), arg); }), @@ -167,6 +168,7 @@ namespace ams::hactool { MakeOptionHandler("appindex", [] (Options &options, const char *arg) { return ParseIntegerArgument(std::addressof(options.preferred_app_index), arg); }), MakeOptionHandler("programindex", [] (Options &options, const char *arg) { return ParseIntegerArgument(std::addressof(options.preferred_program_index), arg); }), MakeOptionHandler("appversion", [] (Options &options, const char *arg) { return ParseIntegerArgument(std::addressof(options.preferred_version), arg); }), + MakeOptionHandler("updatedgen", [] (Options &options, const char *arg) { return ParseIntegerArgument(std::addressof(options.updated_generation), arg); }), }; } diff --git a/source/hactool_options.hpp b/source/hactool_options.hpp index f257bc4..7c83f1b 100644 --- a/source/hactool_options.hpp +++ b/source/hactool_options.hpp @@ -42,6 +42,8 @@ namespace ams::hactool { bool dev = false; bool enable_hash = false; bool disable_key_warns = false; + bool only_updated = false; + int updated_generation = 0; int preferred_app_index = -1; int preferred_program_index = -1; int preferred_version = -1; diff --git a/source/hactool_processor.nca.cpp b/source/hactool_processor.nca.cpp index d67e274..7dc5cf0 100644 --- a/source/hactool_processor.nca.cpp +++ b/source/hactool_processor.nca.cpp @@ -433,12 +433,22 @@ namespace ams::hactool { /* If we have a path, extract to it. */ if (dir_path != nullptr) { - ExtractDirectory(m_local_fs, ctx.file_systems[i], prefix, dir_path, "/"); + if (ctx.romfs_index == i && m_options.only_updated && !ctx.header_readers[i].ExistsCompressionLayer() && ctx.storage_contexts[i].aes_ctr_ex_storage != nullptr && ctx.storage_contexts[i].indirect_storage != nullptr) { + if (!m_options.list_romfs) { + ExtractUpdatedRomFsDirectory(m_local_fs, static_cast(ctx.file_systems[i].get()), ctx.storage_contexts[i].indirect_storage, ctx.storage_contexts[i].aes_ctr_ex_storage, m_options.updated_generation, prefix, dir_path, "/"); + } + } else { + ExtractDirectory(m_local_fs, ctx.file_systems[i], prefix, dir_path, "/"); + } } } if (ctx.romfs_index == i && ctx.is_mounted[i] && m_options.list_romfs) { - PrintDirectory(ctx.file_systems[i], "rom:", "/"); + if (m_options.only_updated && !ctx.header_readers[i].ExistsCompressionLayer() && ctx.storage_contexts[i].aes_ctr_ex_storage != nullptr && ctx.storage_contexts[i].indirect_storage != nullptr) { + PrintUpdatedRomFsDirectory(static_cast(ctx.file_systems[i].get()), ctx.storage_contexts[i].indirect_storage, ctx.storage_contexts[i].aes_ctr_ex_storage, m_options.updated_generation, "rom:", "/"); + } else { + PrintDirectory(ctx.file_systems[i], "rom:", "/"); + } } }