diff --git a/rpcs3/Crypto/unpkg.cpp b/rpcs3/Crypto/unpkg.cpp index feaa0acbcb..b4888a2829 100644 --- a/rpcs3/Crypto/unpkg.cpp +++ b/rpcs3/Crypto/unpkg.cpp @@ -13,6 +13,8 @@ #include "util/sysinfo.hpp" #include "Loader/PSF.h" +#include + LOG_CHANNEL(pkg_log, "PKG"); package_reader::package_reader(const std::string& path) @@ -689,34 +691,174 @@ package_error package_reader::check_target_app_version() const return package_error::app_version; } -fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false); - -usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic_t& sync, thread_key thread_data_key, atomic_t& entry_indexer, std::vector& entries) +bool package_reader::fill_data(std::map& all_install_entries) { + if (!m_is_valid) + { + return false; + } + + m_install_entries.clear(); + m_install_path.clear(); + m_bootable_file_path.clear(); + m_entry_indexer = 0; + m_written_bytes = 0; + + // Get full path and create the directory + std::string dir = rpcs3::utils::get_hdd0_dir(); + + // Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType + switch (m_metadata.content_type) + { + case PKG_CONTENT_TYPE_THEME: + dir += "theme/"; + break; + case PKG_CONTENT_TYPE_WIDGET: + dir += "widget/"; + break; + case PKG_CONTENT_TYPE_LICENSE: + dir += "home/" + Emu.GetUsr() + "/exdata/"; + break; + case PKG_CONTENT_TYPE_VSH_MODULE: + dir += "vsh/modules/"; + break; + case PKG_CONTENT_TYPE_PSN_AVATAR: + dir += "home/" + Emu.GetUsr() + "/psn_avatar/"; + break; + case PKG_CONTENT_TYPE_VMC: + dir += "tmp/vmc/"; + break; + // TODO: Find out if other content types are installed elsewhere + default: + dir += "game/"; + break; + } + + // TODO: Verify whether other content types require appending title ID + if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE) + dir += m_install_dir + '/'; + + // If false, an existing directory is being overwritten: cannot cancel the operation + m_was_null = !fs::is_dir(dir); + + if (!fs::create_path(dir)) + { + pkg_log.error("Could not create the installation directory %s", dir); + return false; + } + + m_install_path = dir; + + if (!decrypt_data()) + { + return false; + } + usz num_failures = 0; - for (usz entry_index = entry_indexer++; num_failures == 0 && entry_index < entries.size(); entry_index = entry_indexer++) + std::vector entries(m_header.file_count); + + std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry)); + + // Create directories first + for (const auto& entry : entries) { - const auto& entry = ::at32(entries, entry_index); - - if ((entry.type & 0xff) == PKG_FILE_ENTRY_FOLDER || (entry.type & 0xff) == 0x12u) - { - continue; - } - if (entry.name_size > PKG_MAX_FILENAME_SIZE) { num_failures++; pkg_log.error("PKG name size is too big (0x%x)", entry.name_size); + break; + } + + const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u; + decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data()); + + const std::string name{reinterpret_cast(m_bufs.back().get()), entry.name_size}; + std::string path = dir + vfs::escape(name); + + const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS); + + (log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad); + + switch (const u8 entry_type = entry.type & 0xff) + { + case PKG_FILE_ENTRY_FOLDER: + case 0x12: + { + if (fs::is_dir(path)) + { + pkg_log.warning("Reused existing directory %s", path); + } + else if (fs::create_path(path)) + { + pkg_log.notice("Created directory %s", path); + } + else + { + num_failures++; + pkg_log.error("Failed to create directory %s", path); + break; + } + + break; + } + default: + { + const std::string true_path = std::filesystem::weakly_canonical(std::filesystem::u8path(path)).string(); + auto map_ptr = &*all_install_entries.try_emplace(true_path).first; + + m_install_entries.push_back({ map_ptr, name, entry.file_offset, entry.file_size, entry.type, entry.pad }); + + if (map_ptr->second && !(entry.type & PKG_FILE_ENTRY_OVERWRITE)) + { + // Cannot override + continue; + } + + // Link + map_ptr->second = &m_install_entries.back(); + continue; + } + } + } + + if (num_failures != 0) + { + pkg_log.error("Package installation failed: %s", dir); + return false; + } + + return true; +} + +fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false); + +usz package_reader::extract_worker(thread_key thread_data_key, std::map& all_install_entries) +{ + usz num_failures = 0; + + while (true) + { + const usz maybe_index = m_entry_indexer++; + + if (maybe_index >= m_install_entries.size()) + { + break; + } + + const install_entry& entry = ::at32(m_install_entries, maybe_index); + + if (!entry.is_dominating()) + { + // Overwritten by another entry + m_written_bytes += entry.file_size; continue; } const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u; - std::span data_span = decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); - - const std::string name{data_span.data(), entry.name_size}; - const std::string path = dir + vfs::escape(name); + const std::string& path = entry.weak_reference->first; + const std::string& name = entry.name; const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS); (log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad); @@ -761,7 +903,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic { const u64 block_size = std::min(BUF_SIZE, entry.file_size - pos); - data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); + const std::span data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); if (data_span.size() != block_size) { @@ -777,18 +919,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic break; } - if (sync.fetch_add((block_size + 0.0) / m_header.data_size) < 0.) - { - // Cancel the installation - num_failures++; - - if (was_null) - { - pkg_log.error("Package installation cancelled: %s", dir); - out.close(); - return num_failures; - } - } + m_written_bytes += block_size; } if (is_buffered) @@ -844,156 +975,57 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic return num_failures; } -bool package_reader::extract_data(atomic_t& sync) +bool package_reader::extract_data(std::deque& readers, std::deque& bootable_paths) { - if (!m_is_valid) + std::map all_install_entries; + + for (auto& reader : readers) { - return false; - } - - // Get full path and create the directory - std::string dir = rpcs3::utils::get_hdd0_dir(); - - // Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType - switch (m_metadata.content_type) - { - case PKG_CONTENT_TYPE_THEME: - dir += "theme/"; - break; - case PKG_CONTENT_TYPE_WIDGET: - dir += "widget/"; - break; - case PKG_CONTENT_TYPE_LICENSE: - dir += "home/" + Emu.GetUsr() + "/exdata/"; - break; - case PKG_CONTENT_TYPE_VSH_MODULE: - dir += "vsh/modules/"; - break; - case PKG_CONTENT_TYPE_PSN_AVATAR: - dir += "home/" + Emu.GetUsr() + "/psn_avatar/"; - break; - case PKG_CONTENT_TYPE_VMC: - dir += "tmp/vmc/"; - break; - // TODO: Find out if other content types are installed elsewhere - default: - dir += "game/"; - break; - } - - // TODO: Verify whether other content types require appending title ID - if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE) - dir += m_install_dir + '/'; - - // If false, an existing directory is being overwritten: cannot cancel the operation - const bool was_null = !fs::is_dir(dir); - - if (!fs::create_path(dir)) - { - pkg_log.error("Could not create the installation directory %s", dir); - return false; - } - - if (!decrypt_data()) - { - return false; - } - - atomic_t num_failures = 0; - - std::vector entries(m_header.file_count); - - std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry)); - - // Create directories first - for (const auto& entry : entries) - { - if (entry.name_size > PKG_MAX_FILENAME_SIZE) + if (!reader.fill_data(all_install_entries)) { - num_failures++; - pkg_log.error("PKG name size is too big (0x%x)", entry.name_size); + return false; + } + } + + usz num_failures = 0; + + for (auto& reader : readers) + { + reader.m_bufs.resize(std::min(utils::get_thread_count(), reader.m_install_entries.size())); + + atomic_t thread_indexer = 0; + + named_thread_group workers("PKG Installer "sv, std::max(reader.m_bufs.size(), 1) - 1, [&]() + { + num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries); + }); + + num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries); + workers.join(); + + reader.m_bufs.clear(); + reader.m_bufs.shrink_to_fit(); + + if (num_failures) + { + if (reader.m_was_null) + { + fs::remove_all(reader.m_install_path, true); + } + + pkg_log.success("Package failed to install ('%s')", reader.m_install_path); break; } - - switch (const u8 entry_type = entry.type & 0xff) + else if (reader.get_progress(1) != 1) { - case PKG_FILE_ENTRY_FOLDER: - case 0x12: - { - const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u; - decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data()); - - const std::string name{reinterpret_cast(m_bufs.back().get()), entry.name_size}; - const std::string path = dir + vfs::escape(name); - - const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS); - - (log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad); - - if (fs::is_dir(path)) - { - pkg_log.warning("Reused existing directory %s", path); - } - else if (fs::create_path(path)) - { - pkg_log.notice("Created directory %s", path); - } - else - { - num_failures++; - pkg_log.error("Failed to create directory %s", path); - break; - } - - break; - } - default: - { - continue; - } + pkg_log.warning("Missing %d bytes from PKG total files size.", reader.m_header.data_size - reader.m_written_bytes); + reader.m_written_bytes = reader.m_header.data_size; // Mark as completed anyway } + + // May be empty + bootable_paths.emplace_back(std::move(reader.m_bootable_file_path)); } - if (num_failures != 0) - { - if (was_null) - { - fs::remove_all(dir, true); - } - - pkg_log.error("Package installation failed: %s", dir); - return false; - } - - atomic_t entry_indexer = 0; - atomic_t thread_indexer = 0; - - m_bufs.resize(std::min(1 /*utils::get_thread_count()*/, entries.size())); - - named_thread_group workers("PKG Installer "sv, std::max(m_bufs.size(), 1) - 1, [&]() - { - num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries); - }); - - num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries); - - workers.join(); - - if (num_failures == 0) - { - pkg_log.success("Package successfully installed to %s", dir); - } - else - { - if (was_null) - { - fs::remove_all(dir, true); - } - - pkg_log.error("Package installation failed: %s", dir); - } - - m_bufs.clear(); return num_failures == 0; } diff --git a/rpcs3/Crypto/unpkg.h b/rpcs3/Crypto/unpkg.h index 45ff6e9d6c..d9d1045e22 100644 --- a/rpcs3/Crypto/unpkg.h +++ b/rpcs3/Crypto/unpkg.h @@ -7,6 +7,7 @@ #include #include #include +#include // Constants enum : u32 @@ -305,24 +306,57 @@ enum class package_error class package_reader { + struct thread_key + { + const usz unique_num = umax; + }; + + struct install_entry + { + typename std::map::value_type* weak_reference{}; + std::string name; + + u64 file_offset{}; + u64 file_size{}; + u32 type{}; + u32 pad{}; + + // Check if the entry is the same one registered in entries to install + bool is_dominating() const + { + return weak_reference->second == this; + } + }; + public: package_reader(const std::string& path); ~package_reader(); bool is_valid() const { return m_is_valid; } package_error check_target_app_version() const; - bool extract_data(atomic_t& sync); + static bool extract_data(std::deque& readers, std::deque& bootable_paths); psf::registry get_psf() const { return m_psf; } - std::string try_get_bootable_file_path_if_created_new() const + int get_progress(int maximum = 100) const { - return m_bootable_file_path; + const usz wr = m_written_bytes; + + return wr >= m_header.data_size ? maximum : ::narrow(wr * maximum / m_header.data_size); } - struct thread_key + void abort_extract() { - const usz unique_num = umax; - }; + m_entry_indexer.fetch_op([this](usz& v) + { + if (v < m_install_entries.size()) + { + v = m_install_entries.size(); + return true; + } + + return false; + }); + } private: bool read_header(); @@ -331,11 +365,18 @@ private: bool decrypt_data(); void archive_seek(s64 new_offset, const fs::seek_mode damode = fs::seek_set); u64 archive_read(void* data_ptr, u64 num_bytes); + bool fill_data(std::map& all_install_entries); std::span archive_read_block(u64 offset, void* data_ptr, u64 num_bytes); std::span decrypt(u64 offset, u64 size, const uchar* key, thread_key thread_data_key = {0}); - usz extract_worker(const std::string& dir, bool was_null, atomic_t& sync, thread_key thread_data_key, atomic_t& entry_indexer, std::vector& entries); + usz extract_worker(thread_key thread_data_key, std::map& all_install_entries); - const usz BUF_SIZE = 8192 * 1024; // 8 MB + std::deque m_install_entries; + std::string m_install_path; + atomic_t m_entry_indexer = 0; + atomic_t m_written_bytes = 0; + bool m_was_null = false; + + static constexpr usz BUF_SIZE = 8192 * 1024; // 8 MB bool m_is_valid = false; diff --git a/rpcs3/Emu/system_utils.cpp b/rpcs3/Emu/system_utils.cpp index 55ffea0336..a702d8464e 100644 --- a/rpcs3/Emu/system_utils.cpp +++ b/rpcs3/Emu/system_utils.cpp @@ -83,27 +83,28 @@ namespace rpcs3::utils { sys_log.success("Installing package: %s", path); - atomic_t progress(0.); int int_progress = 0; + std::deque reader; + reader.emplace_back(path); + // Run PKG unpacking asynchronously named_thread worker("PKG Installer", [&] { - package_reader reader(path); - return reader.extract_data(progress); + std::deque bootables; + + return package_reader::extract_data(reader, bootables); }); // Wait for the completion while (std::this_thread::sleep_for(5ms), worker <= thread_state::aborting) { // TODO: update unified progress dialog - double pval = progress; - pval < 0 ? pval += 1. : pval; - pval *= 100.; + const int pval = reader[0].get_progress(100); - if (static_cast(pval) > int_progress) + if (pval > int_progress) { - int_progress = static_cast(pval); + int_progress = pval; sys_log.success("... %u%%", int_progress); } } diff --git a/rpcs3/rpcs3qt/main_window.cpp b/rpcs3/rpcs3qt/main_window.cpp index 56601e8563..54723a5656 100644 --- a/rpcs3/rpcs3qt/main_window.cpp +++ b/rpcs3/rpcs3qt/main_window.cpp @@ -843,19 +843,10 @@ void main_window::HandlePackageInstallation(QStringList file_paths) pdlg.setAutoClose(false); pdlg.show(); - // Synchronization variable - atomic_t progress(0.); - package_error error = package_error::no_error; - bool cancelled = false; - std::map bootable_paths_installed; // -> title id - - for (usz i = 0, count = packages.size(); i < count; i++) + auto get_app_info = [](compat::package_info& package) { - progress = 0.; - - const compat::package_info& package = ::at32(packages, i); QString app_info = package.title; // This should always be non-empty if (!package.title_id.isEmpty() || !package.version.isEmpty()) @@ -878,76 +869,100 @@ void main_window::HandlePackageInstallation(QStringList file_paths) } } - pdlg.SetValue(0); - pdlg.setLabelText(tr("Installing package (%0/%1), please wait...\n\n%2").arg(i + 1).arg(count).arg(app_info)); - pdlg.show(); + return app_info; + }; - const QFileInfo file_info(package.path); - const std::string path = sstr(package.path); - const std::string file_name = sstr(file_info.fileName()); + bool cancelled = false; + std::map bootable_paths_installed; // -> title id - std::string bootable_path; + std::deque readers; - // Run PKG unpacking asynchronously - named_thread worker("PKG Installer", [path, &progress, &error, &bootable_path] + for (usz i = 0; error == package_error::no_error && i < packages.size(); i++) + { + readers.emplace_back(sstr(packages[i].path)); + error = readers.back().check_target_app_version(); + } + + std::deque bootable_paths; + + // Run PKG unpacking asynchronously + named_thread worker("PKG Installer", [&readers, &error, &bootable_paths] + { + if (error == package_error::no_error) { - package_reader reader(path); - error = reader.check_target_app_version(); - - if (error == package_error::no_error) + if (package_reader::extract_data(readers, bootable_paths)) { - if (reader.extract_data(progress)) - { - bootable_path = reader.try_get_bootable_file_path_if_created_new(); - return true; - } + return true; + } + } + + return false; + }); + + pdlg.show(); + + // Wait for the completion + for (usz i = 0, set_text = umax; i < readers.size();) + { + std::this_thread::sleep_for(5ms); + + if (pdlg.wasCanceled()) + { + cancelled = true; + + for (auto& reader : readers) + { + reader.abort_extract(); } - return false; - }); - - // Wait for the completion - while (worker <= thread_state::aborting) - { - std::this_thread::sleep_for(5ms); - - if (pdlg.wasCanceled()) - { - cancelled = true; - progress -= 1.; - break; - } - - // Update progress window - double pval = progress; - if (pval < 0.) pval += 1.; - - pdlg.SetValue(static_cast(pval * pdlg.maximum())); - QCoreApplication::processEvents(); + break; } - if (worker()) + // Update progress window + const int progress = readers[i].get_progress(pdlg.maximum()); + pdlg.SetValue(progress); + + if (set_text != i) { - pdlg.SetValue(pdlg.maximum()); - std::this_thread::sleep_for(100ms); - } - else - { - pdlg.hide(); - pdlg.SignalFailure(); + pdlg.setLabelText(tr("Installing package (%0/%1), please wait...\n\n%2").arg(i + 1).arg(readers.size()).arg(get_app_info(packages[i]))); + set_text = i; } - if (worker()) + QCoreApplication::processEvents(); + + if (progress == pdlg.maximum()) + { + i++; + } + } + + if (worker()) + { + pdlg.SetValue(pdlg.maximum()); + std::this_thread::sleep_for(100ms); + + if (true) { m_game_list_frame->Refresh(true); - gui_log.success("Successfully installed %s (title_id=%s, title=%s, version=%s).", file_name, sstr(package.title_id), sstr(package.title), sstr(package.version)); - - if (!bootable_path.empty()) + + for (const auto& package : packages) { - bootable_paths_installed[bootable_path] = package.title_id; + gui_log.success("Successfully installed %s (title_id=%s, title=%s, version=%s).", sstr(package.path), sstr(package.title_id), sstr(package.title), sstr(package.version)); } - if (i == (count - 1)) + gui_log.success("Package(s) successfully installed!"); + + for (usz index = 0; index < bootable_paths.size(); index++) + { + if (bootable_paths[index].empty()) + { + continue; + } + + bootable_paths_installed[bootable_paths[index]] = packages[index].title_id; + } + + if (true) { pdlg.hide(); @@ -1024,30 +1039,39 @@ void main_window::HandlePackageInstallation(QStringList file_paths) } } } - else + } + else + { + pdlg.hide(); + pdlg.SignalFailure(); + + if (!cancelled) { - if (!cancelled) + const compat::package_info* package = nullptr; + + for (usz i = 0; i < readers.size(); i++) { - if (error == package_error::app_version) + // Figure out what package failed the installation + if (readers[i].get_progress(1) != 1) { - gui_log.error("Cannot install %s.", file_name); - QMessageBox::warning(this, tr("Warning!"), tr("The following package cannot be installed on top of the current data:\n%1!").arg(package.path)); - } - else - { - gui_log.error("Failed to install %s.", file_name); - QMessageBox::critical(this, tr("Failure!"), tr("Failed to install software from package:\n%1!" - "\nThis is very likely caused by external interference from a faulty anti-virus software." - "\nPlease add RPCS3 to your anti-virus\' whitelist or use better anti-virus software.").arg(package.path)); + package = &packages[i]; } } - return; - } - // return if the thread was still running after cancel - if (cancelled) - { - return; + ensure(package); + + if (error == package_error::app_version) + { + gui_log.error("Cannot install %s.", sstr(package->path)); + QMessageBox::warning(this, tr("Warning!"), tr("The following package cannot be installed on top of the current data:\n%1!").arg(package->path)); + } + else + { + gui_log.error("Failed to install %s.", sstr(package->path)); + QMessageBox::critical(this, tr("Failure!"), tr("Failed to install software from package:\n%1!" + "\nThis is very likely caused by external interference from a faulty anti-virus software." + "\nPlease add RPCS3 to your anti-virus\' whitelist or use better anti-virus software.").arg(package->path)); + } } } }