RPCS3/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2024-11-22 10:42:36 +01:00
Code Issues Releases Wiki Activity

PKG: Pack multiple PKGs into fast+efficient install (#13147)

Browse Source
This commit is contained in:
Elad Ashkenazi 2023-01-03 14:31:39 +02:00 committed by GitHub
parent 7423abb136
commit 7c2d6f8a23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 364 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

370
rpcs3/Crypto/unpkg.cpp
View File

@ -13,6 +13,8 @@
#include "util/sysinfo.hpp"
#include "Loader/PSF.h"
#include <filesystem>
LOG_CHANNEL(pkg_log, "PKG");
package_reader::package_reader(const std::string& path)
@ -689,34 +691,174 @@ package_error package_reader::check_target_app_version() const
return package_error::app_version;
}
fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false);
usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic_t<double>& sync, thread_key thread_data_key, atomic_t<usz>& entry_indexer, std::vector<PKGEntry>& entries)
bool package_reader::fill_data(std::map<std::string, install_entry*>& all_install_entries)
{
if (!m_is_valid)
{
return false;
}
m_install_entries.clear();
m_install_path.clear();
m_bootable_file_path.clear();
m_entry_indexer = 0;
m_written_bytes = 0;
// Get full path and create the directory
std::string dir = rpcs3::utils::get_hdd0_dir();
// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
switch (m_metadata.content_type)
{
case PKG_CONTENT_TYPE_THEME:
dir += "theme/";
break;
case PKG_CONTENT_TYPE_WIDGET:
dir += "widget/";
break;
case PKG_CONTENT_TYPE_LICENSE:
dir += "home/" + Emu.GetUsr() + "/exdata/";
break;
case PKG_CONTENT_TYPE_VSH_MODULE:
dir += "vsh/modules/";
break;
case PKG_CONTENT_TYPE_PSN_AVATAR:
dir += "home/" + Emu.GetUsr() + "/psn_avatar/";
break;
case PKG_CONTENT_TYPE_VMC:
dir += "tmp/vmc/";
break;
// TODO: Find out if other content types are installed elsewhere
default:
dir += "game/";
break;
}
// TODO: Verify whether other content types require appending title ID
if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE)
dir += m_install_dir + '/';
// If false, an existing directory is being overwritten: cannot cancel the operation
m_was_null = !fs::is_dir(dir);
if (!fs::create_path(dir))
{
pkg_log.error("Could not create the installation directory %s", dir);
return false;
}
m_install_path = dir;
if (!decrypt_data())
{
return false;
}
usz num_failures = 0;
for (usz entry_index = entry_indexer++; num_failures == 0 && entry_index < entries.size(); entry_index = entry_indexer++)
std::vector<PKGEntry> entries(m_header.file_count);
std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));
// Create directories first
for (const auto& entry : entries)
{
const auto& entry = ::at32(entries, entry_index);
if ((entry.type & 0xff) == PKG_FILE_ENTRY_FOLDER || (entry.type & 0xff) == 0x12u)
{
continue;
}
if (entry.name_size > PKG_MAX_FILENAME_SIZE)
{
num_failures++;
pkg_log.error("PKG name size is too big (0x%x)", entry.name_size);
break;
}
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());
const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
std::string path = dir + vfs::escape(name);
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
switch (const u8 entry_type = entry.type & 0xff)
{
case PKG_FILE_ENTRY_FOLDER:
case 0x12:
{
if (fs::is_dir(path))
{
pkg_log.warning("Reused existing directory %s", path);
}
else if (fs::create_path(path))
{
pkg_log.notice("Created directory %s", path);
}
else
{
num_failures++;
pkg_log.error("Failed to create directory %s", path);
break;
}
break;
}
default:
{
const std::string true_path = std::filesystem::weakly_canonical(std::filesystem::u8path(path)).string();
auto map_ptr = &*all_install_entries.try_emplace(true_path).first;
m_install_entries.push_back({ map_ptr, name, entry.file_offset, entry.file_size, entry.type, entry.pad });
if (map_ptr->second && !(entry.type & PKG_FILE_ENTRY_OVERWRITE))
{
// Cannot override
continue;
}
// Link
map_ptr->second = &m_install_entries.back();
continue;
}
}
}
if (num_failures != 0)
{
pkg_log.error("Package installation failed: %s", dir);
return false;
}
return true;
}
fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, int mode, u8 *custom_klic, bool verbose = false);
usz package_reader::extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries)
{
usz num_failures = 0;
while (true)
{
const usz maybe_index = m_entry_indexer++;
if (maybe_index >= m_install_entries.size())
{
break;
}
const install_entry& entry = ::at32(m_install_entries, maybe_index);
if (!entry.is_dominating())
{
// Overwritten by another entry
m_written_bytes += entry.file_size;
continue;
}
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
std::span<const char> data_span = decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
const std::string name{data_span.data(), entry.name_size};
const std::string path = dir + vfs::escape(name);
const std::string& path = entry.weak_reference->first;
const std::string& name = entry.name;
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
@ -761,7 +903,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
{
const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos);
data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
if (data_span.size() != block_size)
{
@ -777,18 +919,7 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
break;
}
if (sync.fetch_add((block_size + 0.0) / m_header.data_size) < 0.)
{
// Cancel the installation
num_failures++;
if (was_null)
{
pkg_log.error("Package installation cancelled: %s", dir);
out.close();
return num_failures;
}
}
m_written_bytes += block_size;
}
if (is_buffered)
@ -844,156 +975,57 @@ usz package_reader::extract_worker(const std::string& dir, bool was_null, atomic
return num_failures;
}
bool package_reader::extract_data(atomic_t<double>& sync)
bool package_reader::extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths)
{
if (!m_is_valid)
std::map<std::string, install_entry*> all_install_entries;
for (auto& reader : readers)
{
return false;
}
// Get full path and create the directory
std::string dir = rpcs3::utils::get_hdd0_dir();
// Based on https://www.psdevwiki.com/ps3/PKG_files#ContentType
switch (m_metadata.content_type)
{
case PKG_CONTENT_TYPE_THEME:
dir += "theme/";
break;
case PKG_CONTENT_TYPE_WIDGET:
dir += "widget/";
break;
case PKG_CONTENT_TYPE_LICENSE:
dir += "home/" + Emu.GetUsr() + "/exdata/";
break;
case PKG_CONTENT_TYPE_VSH_MODULE:
dir += "vsh/modules/";
break;
case PKG_CONTENT_TYPE_PSN_AVATAR:
dir += "home/" + Emu.GetUsr() + "/psn_avatar/";
break;
case PKG_CONTENT_TYPE_VMC:
dir += "tmp/vmc/";
break;
// TODO: Find out if other content types are installed elsewhere
default:
dir += "game/";
break;
}
// TODO: Verify whether other content types require appending title ID
if (m_metadata.content_type != PKG_CONTENT_TYPE_LICENSE)
dir += m_install_dir + '/';
// If false, an existing directory is being overwritten: cannot cancel the operation
const bool was_null = !fs::is_dir(dir);
if (!fs::create_path(dir))
{
pkg_log.error("Could not create the installation directory %s", dir);
return false;
}
if (!decrypt_data())
{
return false;
}
atomic_t<usz> num_failures = 0;
std::vector<PKGEntry> entries(m_header.file_count);
std::memcpy(entries.data(), m_bufs.back().get(), entries.size() * sizeof(PKGEntry));
// Create directories first
for (const auto& entry : entries)
{
if (entry.name_size > PKG_MAX_FILENAME_SIZE)
if (!reader.fill_data(all_install_entries))
{
num_failures++;
pkg_log.error("PKG name size is too big (0x%x)", entry.name_size);
return false;
}
}
usz num_failures = 0;
for (auto& reader : readers)
{
reader.m_bufs.resize(std::min<usz>(utils::get_thread_count(), reader.m_install_entries.size()));
atomic_t<usz> thread_indexer = 0;
named_thread_group workers("PKG Installer "sv, std::max<usz>(reader.m_bufs.size(), 1) - 1, [&]()
{
num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
});
num_failures += reader.extract_worker(thread_key{thread_indexer++}, all_install_entries);
workers.join();
reader.m_bufs.clear();
reader.m_bufs.shrink_to_fit();
if (num_failures)
{
if (reader.m_was_null)
{
fs::remove_all(reader.m_install_path, true);
}
pkg_log.success("Package failed to install ('%s')", reader.m_install_path);
break;
}
switch (const u8 entry_type = entry.type & 0xff)
else if (reader.get_progress(1) != 1)
{
case PKG_FILE_ENTRY_FOLDER:
case 0x12:
{
const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0u;
decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data());
const std::string name{reinterpret_cast<char*>(m_bufs.back().get()), entry.name_size};
const std::string path = dir + vfs::escape(name);
const bool log_error = entry.pad || (entry.type & ~PKG_FILE_ENTRY_KNOWN_BITS);
(log_error ? pkg_log.error : pkg_log.notice)("Entry 0x%08x: %s (pad=0x%x)", entry.type, name, entry.pad);
if (fs::is_dir(path))
{
pkg_log.warning("Reused existing directory %s", path);
}
else if (fs::create_path(path))
{
pkg_log.notice("Created directory %s", path);
}
else
{
num_failures++;
pkg_log.error("Failed to create directory %s", path);
break;
}
break;
}
default:
{
continue;
}
pkg_log.warning("Missing %d bytes from PKG total files size.", reader.m_header.data_size - reader.m_written_bytes);
reader.m_written_bytes = reader.m_header.data_size; // Mark as completed anyway
}
// May be empty
bootable_paths.emplace_back(std::move(reader.m_bootable_file_path));
}
if (num_failures != 0)
{
if (was_null)
{
fs::remove_all(dir, true);
}
pkg_log.error("Package installation failed: %s", dir);
return false;
}
atomic_t<usz> entry_indexer = 0;
atomic_t<usz> thread_indexer = 0;
m_bufs.resize(std::min<usz>(1 /*utils::get_thread_count()*/, entries.size()));
named_thread_group workers("PKG Installer "sv, std::max<usz>(m_bufs.size(), 1) - 1, [&]()
{
num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries);
});
num_failures += extract_worker(dir, was_null, sync, thread_key{thread_indexer++}, entry_indexer, entries);
workers.join();
if (num_failures == 0)
{
pkg_log.success("Package successfully installed to %s", dir);
}
else
{
if (was_null)
{
fs::remove_all(dir, true);
}
pkg_log.error("Package installation failed: %s", dir);
}
m_bufs.clear();
return num_failures == 0;
}

57
rpcs3/Crypto/unpkg.h
View File

@ -7,6 +7,7 @@
#include <sstream>
#include <iomanip>
#include <span>
#include <deque>
// Constants
enum : u32
@ -305,24 +306,57 @@ enum class package_error
class package_reader
{
struct thread_key
{
const usz unique_num = umax;
};
struct install_entry
{
typename std::map<std::string, install_entry*>::value_type* weak_reference{};
std::string name;
u64 file_offset{};
u64 file_size{};
u32 type{};
u32 pad{};
// Check if the entry is the same one registered in entries to install
bool is_dominating() const
{
return weak_reference->second == this;
}
};
public:
package_reader(const std::string& path);
~package_reader();
bool is_valid() const { return m_is_valid; }
package_error check_target_app_version() const;
bool extract_data(atomic_t<double>& sync);
static bool extract_data(std::deque<package_reader>& readers, std::deque<std::string>& bootable_paths);
psf::registry get_psf() const { return m_psf; }
std::string try_get_bootable_file_path_if_created_new() const
int get_progress(int maximum = 100) const
{
return m_bootable_file_path;
const usz wr = m_written_bytes;
return wr >= m_header.data_size ? maximum : ::narrow<int>(wr * maximum / m_header.data_size);
}
struct thread_key
void abort_extract()
{
const usz unique_num = umax;
};
m_entry_indexer.fetch_op([this](usz& v)
{
if (v < m_install_entries.size())
{
v = m_install_entries.size();
return true;
}
return false;
});
}
private:
bool read_header();
@ -331,11 +365,18 @@ private:
bool decrypt_data();
void archive_seek(s64 new_offset, const fs::seek_mode damode = fs::seek_set);
u64 archive_read(void* data_ptr, u64 num_bytes);
bool fill_data(std::map<std::string, install_entry*>& all_install_entries);
std::span<const char> archive_read_block(u64 offset, void* data_ptr, u64 num_bytes);
std::span<const char> decrypt(u64 offset, u64 size, const uchar* key, thread_key thread_data_key = {0});
usz extract_worker(const std::string& dir, bool was_null, atomic_t<double>& sync, thread_key thread_data_key, atomic_t<usz>& entry_indexer, std::vector<PKGEntry>& entries);
usz extract_worker(thread_key thread_data_key, std::map<std::string, install_entry*>& all_install_entries);
const usz BUF_SIZE = 8192 * 1024; // 8 MB
std::deque<install_entry> m_install_entries;
std::string m_install_path;
atomic_t<usz> m_entry_indexer = 0;
atomic_t<usz> m_written_bytes = 0;
bool m_was_null = false;
static constexpr usz BUF_SIZE = 8192 * 1024; // 8 MB
bool m_is_valid = false;

17
rpcs3/Emu/system_utils.cpp
View File

@ -83,27 +83,28 @@ namespace rpcs3::utils
{
sys_log.success("Installing package: %s", path);
atomic_t<double> progress(0.);
int int_progress = 0;
std::deque<package_reader> reader;
reader.emplace_back(path);
// Run PKG unpacking asynchronously
named_thread worker("PKG Installer", [&]
{
package_reader reader(path);
return reader.extract_data(progress);
std::deque<std::string> bootables;
return package_reader::extract_data(reader, bootables);
});
// Wait for the completion
while (std::this_thread::sleep_for(5ms), worker <= thread_state::aborting)
{
// TODO: update unified progress dialog
double pval = progress;
pval < 0 ? pval += 1. : pval;
pval *= 100.;
const int pval = reader[0].get_progress(100);
if (static_cast<int>(pval) > int_progress)
if (pval > int_progress)
{
int_progress = static_cast<int>(pval);
int_progress = pval;
sys_log.success("... %u%%", int_progress);
}
}

186
rpcs3/rpcs3qt/main_window.cpp
View File

@ -843,19 +843,10 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
pdlg.setAutoClose(false);
pdlg.show();
// Synchronization variable
atomic_t<double> progress(0.);
package_error error = package_error::no_error;
bool cancelled = false;
std::map<std::string, QString> bootable_paths_installed; // -> title id
for (usz i = 0, count = packages.size(); i < count; i++)
auto get_app_info = [](compat::package_info& package)
{
progress = 0.;
const compat::package_info& package = ::at32(packages, i);
QString app_info = package.title; // This should always be non-empty
if (!package.title_id.isEmpty() || !package.version.isEmpty())
@ -878,76 +869,100 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
}
}
pdlg.SetValue(0);
pdlg.setLabelText(tr("Installing package (%0/%1), please wait...\n\n%2").arg(i + 1).arg(count).arg(app_info));
pdlg.show();
return app_info;
};
const QFileInfo file_info(package.path);
const std::string path = sstr(package.path);
const std::string file_name = sstr(file_info.fileName());
bool cancelled = false;
std::map<std::string, QString> bootable_paths_installed; // -> title id
std::string bootable_path;
std::deque<package_reader> readers;
// Run PKG unpacking asynchronously
named_thread worker("PKG Installer", [path, &progress, &error, &bootable_path]
for (usz i = 0; error == package_error::no_error && i < packages.size(); i++)
{
readers.emplace_back(sstr(packages[i].path));
error = readers.back().check_target_app_version();
}
std::deque<std::string> bootable_paths;
// Run PKG unpacking asynchronously
named_thread worker("PKG Installer", [&readers, &error, &bootable_paths]
{
if (error == package_error::no_error)
{
package_reader reader(path);
error = reader.check_target_app_version();
if (error == package_error::no_error)
if (package_reader::extract_data(readers, bootable_paths))
{
if (reader.extract_data(progress))
{
bootable_path = reader.try_get_bootable_file_path_if_created_new();
return true;
}
return true;
}
}
return false;
});
pdlg.show();
// Wait for the completion
for (usz i = 0, set_text = umax; i < readers.size();)
{
std::this_thread::sleep_for(5ms);
if (pdlg.wasCanceled())
{
cancelled = true;
for (auto& reader : readers)
{
reader.abort_extract();
}
return false;
});
// Wait for the completion
while (worker <= thread_state::aborting)
{
std::this_thread::sleep_for(5ms);
if (pdlg.wasCanceled())
{
cancelled = true;
progress -= 1.;
break;
}
// Update progress window
double pval = progress;
if (pval < 0.) pval += 1.;
pdlg.SetValue(static_cast<int>(pval * pdlg.maximum()));
QCoreApplication::processEvents();
break;
}
if (worker())
// Update progress window
const int progress = readers[i].get_progress(pdlg.maximum());
pdlg.SetValue(progress);
if (set_text != i)
{
pdlg.SetValue(pdlg.maximum());
std::this_thread::sleep_for(100ms);
}
else
{
pdlg.hide();
pdlg.SignalFailure();
pdlg.setLabelText(tr("Installing package (%0/%1), please wait...\n\n%2").arg(i + 1).arg(readers.size()).arg(get_app_info(packages[i])));
set_text = i;
}
if (worker())
QCoreApplication::processEvents();
if (progress == pdlg.maximum())
{
i++;
}
}
if (worker())
{
pdlg.SetValue(pdlg.maximum());
std::this_thread::sleep_for(100ms);
if (true)
{
m_game_list_frame->Refresh(true);
gui_log.success("Successfully installed %s (title_id=%s, title=%s, version=%s).", file_name, sstr(package.title_id), sstr(package.title), sstr(package.version));
if (!bootable_path.empty())
for (const auto& package : packages)
{
bootable_paths_installed[bootable_path] = package.title_id;
gui_log.success("Successfully installed %s (title_id=%s, title=%s, version=%s).", sstr(package.path), sstr(package.title_id), sstr(package.title), sstr(package.version));
}
if (i == (count - 1))
gui_log.success("Package(s) successfully installed!");
for (usz index = 0; index < bootable_paths.size(); index++)
{
if (bootable_paths[index].empty())
{
continue;
}
bootable_paths_installed[bootable_paths[index]] = packages[index].title_id;
}
if (true)
{
pdlg.hide();
@ -1024,30 +1039,39 @@ void main_window::HandlePackageInstallation(QStringList file_paths)
}
}
}
else
}
else
{
pdlg.hide();
pdlg.SignalFailure();
if (!cancelled)
{
if (!cancelled)
const compat::package_info* package = nullptr;
for (usz i = 0; i < readers.size(); i++)
{
if (error == package_error::app_version)
// Figure out what package failed the installation
if (readers[i].get_progress(1) != 1)
{
gui_log.error("Cannot install %s.", file_name);
QMessageBox::warning(this, tr("Warning!"), tr("The following package cannot be installed on top of the current data:\n%1!").arg(package.path));
}
else
{
gui_log.error("Failed to install %s.", file_name);
QMessageBox::critical(this, tr("Failure!"), tr("Failed to install software from package:\n%1!"
"\nThis is very likely caused by external interference from a faulty anti-virus software."
"\nPlease add RPCS3 to your anti-virus\' whitelist or use better anti-virus software.").arg(package.path));
package = &packages[i];
}
}
return;
}
// return if the thread was still running after cancel
if (cancelled)
{
return;
ensure(package);
if (error == package_error::app_version)
{
gui_log.error("Cannot install %s.", sstr(package->path));
QMessageBox::warning(this, tr("Warning!"), tr("The following package cannot be installed on top of the current data:\n%1!").arg(package->path));
}
else
{
gui_log.error("Failed to install %s.", sstr(package->path));
QMessageBox::critical(this, tr("Failure!"), tr("Failed to install software from package:\n%1!"
"\nThis is very likely caused by external interference from a faulty anti-virus software."
"\nPlease add RPCS3 to your anti-virus\' whitelist or use better anti-virus software.").arg(package->path));
}
}
}
}