1
0
mirror of https://github.com/RPCS3/rpcs3.git synced 2024-11-23 03:02:53 +01:00

PKG: Pack multiple PKGs into fast+efficient install (#13147)

This commit is contained in:
Elad Ashkenazi 2023-01-03 14:31:39 +02:00 committed by GitHub
parent 7423abb136
commit 7c2d6f8a23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 364 additions and 266 deletions

View File

@ -13,6 +13,8 @@
#include "util/sysinfo.hpp" #include "util/sysinfo.hpp"
#include "Loader/PSF.h" #include "Loader/PSF.h"
#include <filesystem>
LOG_CHANNEL(pkg_log, "PKG"); LOG_CHANNEL(pkg_log, "PKG");
package_reader::package_reader(const std::string& path) package_reader::package_reader(const std::string& path)
@ -689,34 +691,174 @@ package_error package_reader::check_target_app_version() const
return package_error::app_version; return package_error::app_version;
} }
fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false); bool package_reader::fill_data(std::map<std::string, install_entry*>& all_install_entries)
usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic_t<double>& sync, thread_key thread_data_key, atomic_t<usz>& entry_indexer, std::vector<PKGEntry>& entries)
{ {
usz num_failures = 0; if (!m_is_valid)
for (usz entry_index = entry_indexer++; num_failures == 0 && entry_index < entries.size(); entry_index = entry_indexer++)
{ {
const auto& entry = ::at32(entries, entry_index); return false;
if ((entry.type & 0xff) == PKG_FILE_ENTRY_FOLDER || (entry.type & 0xff) == 0x12u)
{
continue;
} }
m_install_entries.clear();
m_install_path.clear();
m_bootable_file_path.clear();
m_entry_indexer = 0;
m_written_bytes = 0;
// Get full path and create the directory
std::string dir = rpcs3::utils::get_hdd0_dir();
// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
switch (m_metadata.content_type)
{
case PKG_CONTENT_TYPE_THEME:
dir += "theme/";
break;
case PKG_CONTENT_TYPE_WIDGET:
dir += "widget/";
break;
case PKG_CONTENT_TYPE_LICENSE:
dir += "home/" + Emu.GetUsr() + "/exdata/";
break;
case PKG_CONTENT_TYPE_VSH_MODULE:
dir += "vsh/modules/";
break;
case PKG_CONTENT_TYPE_PSN_AVATAR:
dir += "home/" + Emu.GetUsr() + "/psn_avatar/";
break;
case PKG_CONTENT_TYPE_VMC:
dir += "tmp/vmc/";
break;
// TODO: Find out if other content types are installed elsewhere
default:
dir += "game/";
break;
}
// TODO: Verify whether other content types require appending title ID
if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE)
dir += m_install_dir + '/';
// If false, an existing directory is being overwritten: cannot cancel the operation
m_was_null = !fs::is_dir(dir);
if (!fs::create_path(dir))
{
pkg_log.error("Could not create the installation directory %s", dir);
return false;
}
m_install_path = dir;
if (!decrypt_data())
{
return false;
}
usz num_failures = 0;
std::vector<PKGEntry> entries(m_header.file_count);
std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));
// Create directories first
for (const auto& entry : entries)
{
if (entry.name_size > PKG_MAX_FILENAME_SIZE) if (entry.name_size > PKG_MAX_FILENAME_SIZE)
{ {
num_failures++; num_failures++;
pkg_log.error("PKG name size is too big (0x%x)", entry.name_size); pkg_log.error("PKG name size is too big (0x%x)", entry.name_size);
break;
}
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());
const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
std::string path = dir + vfs::escape(name);
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
switch (const u8 entry_type = entry.type & 0xff)
{
case PKG_FILE_ENTRY_FOLDER:
case 0x12:
{
if (fs::is_dir(path))
{
pkg_log.warning("Reused existing directory %s", path);
}
else if (fs::create_path(path))
{
pkg_log.notice("Created directory %s", path);
}
else
{
num_failures++;
pkg_log.error("Failed to create directory %s", path);
break;
}
break;
}
default:
{
const std::string true_path = std::filesystem::weakly_canonical(std::filesystem::u8path(path)).string();
auto map_ptr = &*all_install_entries.try_emplace(true_path).first;
m_install_entries.push_back({ map_ptr, name, entry.file_offset, entry.file_size, entry.type, entry.pad });
if (map_ptr->second && !(entry.type & PKG_FILE_ENTRY_OVERWRITE))
{
// Cannot override
continue;
}
// Link
map_ptr->second = &m_install_entries.back();
continue;
}
}
}
if (num_failures != 0)
{
pkg_log.error("Package installation failed: %s", dir);
return false;
}
return true;
}
fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false);
usz package_reader::extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries)
{
usz num_failures = 0;
while (true)
{
const usz maybe_index = m_entry_indexer++;
if (maybe_index >= m_install_entries.size())
{
break;
}
const install_entry& entry = ::at32(m_install_entries, maybe_index);
if (!entry.is_dominating())
{
// Overwritten by another entry
m_written_bytes += entry.file_size;
continue; continue;
} }
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u; const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
std::span<const char> data_span = decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); const std::string& path = entry.weak_reference->first;
const std::string& name = entry.name;
const std::string name{data_span.data(), entry.name_size};
const std::string path = dir + vfs::escape(name);
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS); const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad); (log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
@ -761,7 +903,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
{ {
const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos); const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos);
data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
if (data_span.size() != block_size) if (data_span.size() != block_size)
{ {
@ -777,18 +919,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
break; break;
} }
if (sync.fetch_add((block_size + 0.0) / m_header.data_size) < 0.) m_written_bytes += block_size;
{
// Cancel the installation
num_failures++;
if (was_null)
{
pkg_log.error("Package installation cancelled: %s", dir);
out.close();
return num_failures;
}
}
} }
if (is_buffered) if (is_buffered)
@ -844,156 +975,57 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
return num_failures; return num_failures;
} }
bool package_reader::extract_data(atomic_t<double>& sync) bool package_reader::extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths)
{ {
if (!m_is_valid) std::map<std::string, install_entry*> all_install_entries;
for (auto& reader : readers)
{
if (!reader.fill_data(all_install_entries))
{ {
return false; return false;
} }
// Get full path and create the directory
std::string dir = rpcs3::utils::get_hdd0_dir();
// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
switch (m_metadata.content_type)
{
case PKG_CONTENT_TYPE_THEME:
dir += "theme/";
break;
case PKG_CONTENT_TYPE_WIDGET:
dir += "widget/";
break;
case PKG_CONTENT_TYPE_LICENSE:
dir += "home/" + Emu.GetUsr() + "/exdata/";
break;
case PKG_CONTENT_TYPE_VSH_MODULE:
dir += "vsh/modules/";
break;
case PKG_CONTENT_TYPE_PSN_AVATAR:
dir += "home/" + Emu.GetUsr() + "/psn_avatar/";
break;
case PKG_CONTENT_TYPE_VMC:
dir += "tmp/vmc/";
break;
// TODO: Find out if other content types are installed elsewhere
default:
dir += "game/";
break;
} }
// TODO: Verify whether other content types require appending title ID usz num_failures = 0;
if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE)
dir += m_install_dir + '/';
// If false, an existing directory is being overwritten: cannot cancel the operation for (auto& reader : readers)
const bool was_null = !fs::is_dir(dir);
if (!fs::create_path(dir))
{ {
pkg_log.error("Could not create the installation directory %s", dir); reader.m_bufs.resize(std::min<usz>(utils::get_thread_count(), reader.m_install_entries.size()));
return false;
}
if (!decrypt_data())
{
return false;
}
atomic_t<usz> num_failures = 0;
std::vector<PKGEntry> entries(m_header.file_count);
std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));
// Create directories first
for (const auto& entry : entries)
{
if (entry.name_size > PKG_MAX_FILENAME_SIZE)
{
num_failures++;
pkg_log.error("PKG name size is too big (0x%x)", entry.name_size);
break;
}
switch (const u8 entry_type = entry.type & 0xff)
{
case PKG_FILE_ENTRY_FOLDER:
case 0x12:
{
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());
const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
const std::string path = dir + vfs::escape(name);
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
if (fs::is_dir(path))
{
pkg_log.warning("Reused existing directory %s", path);
}
else if (fs::create_path(path))
{
pkg_log.notice("Created directory %s", path);
}
else
{
num_failures++;
pkg_log.error("Failed to create directory %s", path);
break;
}
break;
}
default:
{
continue;
}
}
}
if (num_failures != 0)
{
if (was_null)
{
fs::remove_all(dir, true);
}
pkg_log.error("Package installation failed: %s", dir);
return false;
}
atomic_t<usz> entry_indexer = 0;
atomic_t<usz> thread_indexer = 0; atomic_t<usz> thread_indexer = 0;
m_bufs.resize(std::min<usz>(1 /*utils::get_thread_count()*/, entries.size())); named_thread_group workers("PKG Installer "sv, std::max<usz>(reader.m_bufs.size(), 1) - 1, [&]()
named_thread_group workers("PKG Installer "sv, std::max<usz>(m_bufs.size(), 1) - 1, [&]()
{ {
num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries); num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
}); });
num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries); num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
workers.join(); workers.join();
if (num_failures == 0) reader.m_bufs.clear();
reader.m_bufs.shrink_to_fit();
if (num_failures)
{ {
pkg_log.success("Package successfully installed to %s", dir); if (reader.m_was_null)
}
else
{ {
if (was_null) fs::remove_all(reader.m_install_path, true);
{
fs::remove_all(dir, true);
} }
pkg_log.error("Package installation failed: %s", dir); pkg_log.success("Package failed to install ('%s')", reader.m_install_path);
break;
}
else if (reader.get_progress(1) != 1)
{
pkg_log.warning("Missing %d bytes from PKG total files size.", reader.m_header.data_size - reader.m_written_bytes);
reader.m_written_bytes = reader.m_header.data_size; // Mark as completed anyway
}
// May be empty
bootable_paths.emplace_back(std::move(reader.m_bootable_file_path));
} }
m_bufs.clear();
return num_failures == 0; return num_failures == 0;
} }

View File

@ -7,6 +7,7 @@
#include <sstream> #include <sstream>
#include <iomanip> #include <iomanip>
#include <span> #include <span>
#include <deque>
// Constants // Constants
enum : u32 enum : u32
@ -305,24 +306,57 @@ enum class package_error
class package_reader class package_reader
{ {
struct thread_key
{
const usz unique_num = umax;
};
struct install_entry
{
typename std::map<std::string, install_entry*>::value_type* weak_reference{};
std::string name;
u64 file_offset{};
u64 file_size{};
u32 type{};
u32 pad{};
// Check if the entry is the same one registered in entries to install
bool is_dominating() const
{
return weak_reference->second == this;
}
};
public: public:
package_reader(const std::string& path); package_reader(const std::string& path);
~package_reader(); ~package_reader();
bool is_valid() const { return m_is_valid; } bool is_valid() const { return m_is_valid; }
package_error check_target_app_version() const; package_error check_target_app_version() const;
bool extract_data(atomic_t<double>& sync); static bool extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths);
psf::registry get_psf() const { return m_psf; } psf::registry get_psf() const { return m_psf; }
std::string try_get_bootable_file_path_if_created_new() const int get_progress(int maximum = 100) const
{ {
return m_bootable_file_path; const usz wr = m_written_bytes;
return wr >= m_header.data_size ? maximum : ::narrow<int>(wr * maximum / m_header.data_size);
} }
struct thread_key void abort_extract()
{ {
const usz unique_num = umax; m_entry_indexer.fetch_op([this](usz& v)
}; {
if (v < m_install_entries.size())
{
v = m_install_entries.size();
return true;
}
return false;
});
}
private: private:
bool read_header(); bool read_header();
@ -331,11 +365,18 @@ private:
bool decrypt_data(); bool decrypt_data();
void archive_seek(s64 new_offset, const fs::seek_mode damode = fs::seek_set); void archive_seek(s64 new_offset, const fs::seek_mode damode = fs::seek_set);
u64 archive_read(void* data_ptr, u64 num_bytes); u64 archive_read(void* data_ptr, u64 num_bytes);
bool fill_data(std::map<std::string, install_entry*>& all_install_entries);
std::span<const char> archive_read_block(u64 offset, void* data_ptr, u64 num_bytes); std::span<const char> archive_read_block(u64 offset, void* data_ptr, u64 num_bytes);
std::span<const char> decrypt(u64 offset, u64 size, const uchar* key, thread_key thread_data_key = {0}); std::span<const char> decrypt(u64 offset, u64 size, const uchar* key, thread_key thread_data_key = {0});
usz extract_worker(const std::string& dir, bool was_null, atomic_t<double>& sync, thread_key thread_data_key, atomic_t<usz>& entry_indexer, std::vector<PKGEntry>& entries); usz extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries);
const usz BUF_SIZE = 8192 * 1024; // 8 MB std::deque<install_entry> m_install_entries;
std::string m_install_path;
atomic_t<usz> m_entry_indexer = 0;
atomic_t<usz> m_written_bytes = 0;
bool m_was_null = false;
static constexpr usz BUF_SIZE = 8192 * 1024; // 8 MB
bool m_is_valid = false; bool m_is_valid = false;

View File

@ -83,27 +83,28 @@ namespace rpcs3::utils
{ {
sys_log.success("Installing package: %s", path); sys_log.success("Installing package: %s", path);
atomic_t<double> progress(0.);
int int_progress = 0; int int_progress = 0;
std::deque<package_reader> reader;
reader.emplace_back(path);
// Run PKG unpacking asynchronously // Run PKG unpacking asynchronously
named_thread worker("PKG Installer", [&] named_thread worker("PKG Installer", [&]
{ {
package_reader reader(path); std::deque<std::string> bootables;
return reader.extract_data(progress);
return package_reader::extract_data(reader, bootables);
}); });
// Wait for the completion // Wait for the completion
while (std::this_thread::sleep_for(5ms), worker <= thread_state::aborting) while (std::this_thread::sleep_for(5ms), worker <= thread_state::aborting)
{ {
// TODO: update unified progress dialog // TODO: update unified progress dialog
double pval = progress; const int pval = reader[0].get_progress(100);
pval < 0 ? pval += 1. : pval;
pval *= 100.;
if (static_cast<int>(pval) > int_progress) if (pval > int_progress)
{ {
int_progress = static_cast<int>(pval); int_progress = pval;
sys_log.success("... %u%%", int_progress); sys_log.success("... %u%%", int_progress);
} }
} }

View File

@ -843,19 +843,10 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
pdlg.setAutoClose(false); pdlg.setAutoClose(false);
pdlg.show(); pdlg.show();
// Synchronization variable
atomic_t<double> progress(0.);
package_error error = package_error::no_error; package_error error = package_error::no_error;
bool cancelled = false; auto get_app_info = [](compat::package_info& package)
std::map<std::string, QString> bootable_paths_installed; // -> title id
for (usz i = 0, count = packages.size(); i < count; i++)
{ {
progress = 0.;
const compat::package_info& package = ::at32(packages, i);
QString app_info = package.title; // This should always be non-empty QString app_info = package.title; // This should always be non-empty
if (!package.title_id.isEmpty() || !package.version.isEmpty()) if (!package.title_id.isEmpty() || !package.version.isEmpty())
@ -878,27 +869,29 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
} }
} }
pdlg.SetValue(0); return app_info;
pdlg.setLabelText(tr("Installing package (%0/%1), please wait...\n\n%2").arg(i + 1).arg(count).arg(app_info)); };
pdlg.show();
const QFileInfo file_info(package.path); bool cancelled = false;
const std::string path = sstr(package.path); std::map<std::string, QString> bootable_paths_installed; // -> title id
const std::string file_name = sstr(file_info.fileName());
std::string bootable_path; std::deque<package_reader> readers;
for (usz i = 0; error == package_error::no_error && i < packages.size(); i++)
{
readers.emplace_back(sstr(packages[i].path));
error = readers.back().check_target_app_version();
}
std::deque<std::string> bootable_paths;
// Run PKG unpacking asynchronously // Run PKG unpacking asynchronously
named_thread worker("PKG Installer", [path, &progress, &error, &bootable_path] named_thread worker("PKG Installer", [&readers, &error, &bootable_paths]
{ {
package_reader reader(path);
error = reader.check_target_app_version();
if (error == package_error::no_error) if (error == package_error::no_error)
{ {
if (reader.extract_data(progress)) if (package_reader::extract_data(readers, bootable_paths))
{ {
bootable_path = reader.try_get_bootable_file_path_if_created_new();
return true; return true;
} }
} }
@ -906,48 +899,70 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
return false; return false;
}); });
pdlg.show();
// Wait for the completion // Wait for the completion
while (worker <= thread_state::aborting) for (usz i = 0, set_text = umax; i < readers.size();)
{ {
std::this_thread::sleep_for(5ms); std::this_thread::sleep_for(5ms);
if (pdlg.wasCanceled()) if (pdlg.wasCanceled())
{ {
cancelled = true; cancelled = true;
progress -= 1.;
for (auto& reader : readers)
{
reader.abort_extract();
}
break; break;
} }
// Update progress window // Update progress window
double pval = progress; const int progress = readers[i].get_progress(pdlg.maximum());
if (pval < 0.) pval += 1.; pdlg.SetValue(progress);
if (set_text != i)
{
pdlg.setLabelText(tr("Installing package (%0/%1), please wait...\n\n%2").arg(i + 1).arg(readers.size()).arg(get_app_info(packages[i])));
set_text = i;
}
pdlg.SetValue(static_cast<int>(pval * pdlg.maximum()));
QCoreApplication::processEvents(); QCoreApplication::processEvents();
if (progress == pdlg.maximum())
{
i++;
}
} }
if (worker()) if (worker())
{ {
pdlg.SetValue(pdlg.maximum()); pdlg.SetValue(pdlg.maximum());
std::this_thread::sleep_for(100ms); std::this_thread::sleep_for(100ms);
}
else
{
pdlg.hide();
pdlg.SignalFailure();
}
if (worker()) if (true)
{ {
m_game_list_frame->Refresh(true); m_game_list_frame->Refresh(true);
gui_log.success("Successfully installed %s (title_id=%s, title=%s, version=%s).", file_name, sstr(package.title_id), sstr(package.title), sstr(package.version));
if (!bootable_path.empty()) for (const auto& package : packages)
{ {
bootable_paths_installed[bootable_path] = package.title_id; gui_log.success("Successfully installed %s (title_id=%s, title=%s, version=%s).", sstr(package.path), sstr(package.title_id), sstr(package.title), sstr(package.version));
} }
if (i == (count - 1)) gui_log.success("Package(s) successfully installed!");
for (usz index = 0; index < bootable_paths.size(); index++)
{
if (bootable_paths[index].empty())
{
continue;
}
bootable_paths_installed[bootable_paths[index]] = packages[index].title_id;
}
if (true)
{ {
pdlg.hide(); pdlg.hide();
@ -1024,31 +1039,40 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
} }
} }
} }
}
else else
{ {
pdlg.hide();
pdlg.SignalFailure();
if (!cancelled) if (!cancelled)
{ {
const compat::package_info* package = nullptr;
for (usz i = 0; i < readers.size(); i++)
{
// Figure out what package failed the installation
if (readers[i].get_progress(1) != 1)
{
package = &packages[i];
}
}
ensure(package);
if (error == package_error::app_version) if (error == package_error::app_version)
{ {
gui_log.error("Cannot install %s.", file_name); gui_log.error("Cannot install %s.", sstr(package->path));
QMessageBox::warning(this, tr("Warning!"), tr("The following package cannot be installed on top of the current data:\n%1!").arg(package.path)); QMessageBox::warning(this, tr("Warning!"), tr("The following package cannot be installed on top of the current data:\n%1!").arg(package->path));
} }
else else
{ {
gui_log.error("Failed to install %s.", file_name); gui_log.error("Failed to install %s.", sstr(package->path));
QMessageBox::critical(this, tr("Failure!"), tr("Failed to install software from package:\n%1!" QMessageBox::critical(this, tr("Failure!"), tr("Failed to install software from package:\n%1!"
"\nThis is very likely caused by external interference from a faulty anti-virus software." "\nThis is very likely caused by external interference from a faulty anti-virus software."
"\nPlease add RPCS3 to your anti-virus\' whitelist or use better anti-virus software.").arg(package.path)); "\nPlease add RPCS3 to your anti-virus\' whitelist or use better anti-virus software.").arg(package->path));
} }
} }
return;
}
// return if the thread was still running after cancel
if (cancelled)
{
return;
}
} }
} }