From 8aee672466fa7ae99bda6d5a3fbe18b4e34814f9 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Thu, 21 Jun 2018 23:41:10 +0200 Subject: [PATCH] rwlib: refactor FileIndex - make FileIndex const correct (std::map::operator[] modifies the std::map) - use a single map - keys are paths relative in the game data directory + filenames (so paths like "data/main.scm" can be found twice in the map) - normalization of the file paths is done inside FileIndex, so there shouldn't be any ::tolower's anymore. - added a normalizer helper function "normalizeFilePath" - added Documentation --- rwengine/src/engine/GameData.cpp | 10 +- rwengine/src/engine/GameWorld.cpp | 8 +- rwlib/source/platform/FileHandle.hpp | 4 +- rwlib/source/platform/FileIndex.cpp | 136 ++++++++++++++------------- rwlib/source/platform/FileIndex.hpp | 117 ++++++++++++----------- tests/test_FileIndex.cpp | 49 +++++++--- tests/test_Text.cpp | 2 +- 7 files changed, 173 insertions(+), 153 deletions(-) diff --git a/rwengine/src/engine/GameData.cpp b/rwengine/src/engine/GameData.cpp index 25c3fbdf..76586417 100644 --- a/rwengine/src/engine/GameData.cpp +++ b/rwengine/src/engine/GameData.cpp @@ -41,7 +41,6 @@ GameData::GameData(Logger* log, const rwfs::path& path) GameData::~GameData() = default; void GameData::load() { - index.indexGameDirectory(datpath); index.indexTree(datpath); loadIMG("models/gta3.img"); @@ -191,8 +190,7 @@ void GameData::loadCOL(const size_t zone, const std::string& name) { } void GameData::loadIMG(const std::string& name) { - auto syspath = index.findFilePath(name).string(); - index.indexArchive(syspath); + index.indexArchive(name); } void GameData::loadIPL(const std::string& path) { @@ -290,7 +288,7 @@ void GameData::loadHandling(const std::string& path) { } SCMFile* GameData::loadSCM(const std::string& path) { - auto scm_h = index.openFilePath(path); + auto scm_h = index.openFileRaw(path); SCMFile* scm = new SCMFile; scm->loadFile(scm_h->data, scm_h->length); scm_h.reset(); @@ -298,7 +296,7 @@ SCMFile* GameData::loadSCM(const std::string& path) { } void GameData::loadGXT(const std::string& name) { - auto file = index.openFilePath(name); + auto file = index.openFileRaw(name); LoaderGXT loader; @@ -419,7 +417,7 @@ ClumpPtr GameData::loadClump(const std::string& name, const std::string& texture } void GameData::loadModelFile(const std::string& name) { - auto file = index.openFilePath(name); + auto file = index.openFileRaw(name); if (!file) { logger->log("Data", Logger::Error, "Failed to load model file " + name); return; diff --git a/rwengine/src/engine/GameWorld.cpp b/rwengine/src/engine/GameWorld.cpp index 602ffce1..f61cf571 100644 --- a/rwengine/src/engine/GameWorld.cpp +++ b/rwengine/src/engine/GameWorld.cpp @@ -691,11 +691,7 @@ void GameWorld::PhysicsTickCallback(btDynamicsWorld* physWorld, } void GameWorld::loadCutscene(const std::string& name) { - std::string lowerName(name); - std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), - ::tolower); - - auto datfile = data->index.openFile(lowerName + ".dat"); + auto datfile = data->index.openFile(name + ".dat"); CutsceneData* cutscene = new CutsceneData; @@ -704,7 +700,7 @@ void GameWorld::loadCutscene(const std::string& name) { loaderdat.load(cutscene->tracks, datfile); } - data->loadIFP(lowerName + ".ifp"); + data->loadIFP(name + ".ifp"); cutsceneAudioLoaded = data->loadAudioStream(name + ".mp3"); diff --git a/rwlib/source/platform/FileHandle.hpp b/rwlib/source/platform/FileHandle.hpp index 4ab1801a..aae88aa6 100644 --- a/rwlib/source/platform/FileHandle.hpp +++ b/rwlib/source/platform/FileHandle.hpp @@ -1,13 +1,11 @@ #ifndef _LIBRW_FILEHANDLE_HPP_ #define _LIBRW_FILEHANDLE_HPP_ -#include - /** * @brief Contains a pointer to a file's contents. */ struct FileContentsInfo { - char* data; + char *data; size_t length; FileContentsInfo(char* mem, size_t len) : data(mem), length(len) { diff --git a/rwlib/source/platform/FileIndex.cpp b/rwlib/source/platform/FileIndex.cpp index c8c575f7..5da3a8a2 100644 --- a/rwlib/source/platform/FileIndex.cpp +++ b/rwlib/source/platform/FileIndex.cpp @@ -1,38 +1,69 @@ #include "platform/FileIndex.hpp" #include +#include #include -#include -#include - -#include +#include +#include +#include "FileHandle.hpp" #include "loaders/LoaderIMG.hpp" -#include "platform/FileHandle.hpp" -void FileIndex::indexGameDirectory(const rwfs::path& base_path) { - gamedatapath_ = base_path; +#include "rw/debug.hpp" - for (const rwfs::path& path : - rwfs::recursive_directory_iterator(base_path)) { +std::string FileIndex::normalizeFilePath(const std::string &filePath) { + std::ostringstream oss; + std::transform(filePath.cbegin(), filePath.cend(), std::ostreambuf_iterator(oss), [](char c) { + if (c == '\\') { + return '/'; + } + return static_cast(std::tolower(c)); + }); + return oss.str(); +} + +void FileIndex::indexTree(const rwfs::path &path) { + // Remove the trailing "/" or "/." from base_path. Boost 1.66 and c++17 have different lexically_relative behavior. + rwfs::path basePath = (path / ".").lexically_normal(); + basePath = basePath.parent_path(); + + for (const rwfs::path &path : + rwfs::recursive_directory_iterator(basePath)) { if (!rwfs::is_regular_file(path)) { continue; } + auto relPath = path.lexically_relative(basePath); + std::string relPathName = normalizeFilePath(relPath.string()); + indexedData_[relPathName] = {IndexedDataType::FILE, path.string(), ""}; - std::string name = path.string(); - std::transform(name.begin(), name.end(), name.begin(), ::tolower); - filesystemfiles_[name] = path; + auto filename = normalizeFilePath(path.filename().string()); + indexedData_[filename] = {IndexedDataType::FILE, path.string(), ""}; } } -FileHandle FileIndex::openFilePath(const std::string& file_path) { - auto datapath = findFilePath(file_path); - std::ifstream dfile(datapath.string(), - std::ios_base::binary | std::ios_base::ate); +const FileIndex::IndexedData *FileIndex::getIndexedDataAt(const std::string &filePath) const { + auto normPath = normalizeFilePath(filePath); + return &indexedData_.at(normPath); +} + +rwfs::path FileIndex::findFilePath(const std::string &filePath) const { + return getIndexedDataAt(filePath)->path; +} + +FileHandle FileIndex::openFileRaw(const std::string &filePath) const { + const auto *indexData = getIndexedDataAt(filePath); + std::ifstream dfile(indexData->path, std::ios::binary); if (!dfile.is_open()) { - throw std::runtime_error("Unable to open file: " + file_path); + throw std::runtime_error("Unable to open file: " + filePath); } +#if RW_DEBUG + if (indexData->type != IndexedDataType::FILE) { + RW_MESSAGE("Reading raw data from archive \"" << filePath << "\""); + } +#endif + + dfile.seekg(0, std::ios::end); auto length = dfile.tellg(); dfile.seekg(0); auto data = new char[length]; @@ -41,84 +72,57 @@ FileHandle FileIndex::openFilePath(const std::string& file_path) { return std::make_shared(data, length); } -void FileIndex::indexTree(const rwfs::path& root) { - for (const rwfs::path& path : rwfs::recursive_directory_iterator(root)) { - if (!rwfs::is_regular_file(path)) { - continue; - } - - std::string directory = path.parent_path().string(); - std::string realName = path.filename().string(); - std::string lowerName = realName; - std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), - ::tolower); - files[lowerName] = {lowerName, realName, directory, ""}; - } -} - -void FileIndex::indexArchive(const std::string& archive) { - // Split directory from archive name - auto archive_path = rwfs::path(archive); - auto directory = archive_path.parent_path(); - auto archive_basename = archive_path.filename(); - auto archive_full_path = directory / archive_basename; +void FileIndex::indexArchive(const std::string &archive) { + rwfs::path path = findFilePath(archive); LoaderIMG img; - if (!img.load(archive_full_path.string())) { - throw std::runtime_error("Failed to load IMG archive: " + - archive_full_path.string()); + if (!img.load(path.string())) { + throw std::runtime_error("Failed to load IMG archive: " + path.string()); } - std::string lowerName; for (size_t i = 0; i < img.getAssetCount(); ++i) { - auto& asset = img.getAssetInfoByIndex(i); + auto &asset = img.getAssetInfoByIndex(i); if (asset.size == 0) continue; - lowerName = asset.name; - std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), - ::tolower); + std::string assetName = normalizeFilePath(asset.name); - files[lowerName] = {lowerName, asset.name, directory.string(), - archive_basename.string()}; + indexedData_[assetName] = {IndexedDataType::ARCHIVE, path.string(), asset.name}; } } -FileHandle FileIndex::openFile(const std::string& filename) { - auto iterator = files.find(filename); - if (iterator == files.end()) { +FileHandle FileIndex::openFile(const std::string &filePath) { + auto cleanFilePath = normalizeFilePath(filePath); + auto indexedDataPos = indexedData_.find(cleanFilePath); + if (indexedDataPos == indexedData_.end()) { return nullptr; } - IndexData& f = iterator->second; - bool isArchive = !f.archive.empty(); + const auto &indexedData = indexedDataPos->second; - auto fsName = f.directory + "/" + f.originalName; - - char* data = nullptr; + char *data = nullptr; size_t length = 0; - if (isArchive) { - fsName = f.directory + "/" + f.archive; - + if (indexedData.type == IndexedDataType::ARCHIVE) { LoaderIMG img; - if (!img.load(fsName)) { - throw std::runtime_error("Failed to load IMG archive: " + fsName); + if (!img.load(indexedData.path)) { + throw std::runtime_error("Failed to load IMG archive: " + indexedData.path); } LoaderIMGFile file; - if (img.findAssetInfo(f.originalName, file)) { + auto filename = rwfs::path(indexedData.assetData).filename().string(); + if (img.findAssetInfo(filename, file)) { length = file.size * 2048; - data = img.loadToMemory(f.originalName); + data = img.loadToMemory(filename); } } else { - std::ifstream dfile(fsName.c_str(), std::ios_base::binary); + std::ifstream dfile(indexedData.path, std::ios::binary); if (!dfile.is_open()) { - throw std::runtime_error("Unable to open file: " + fsName); + throw std::runtime_error("Unable to open file: " + indexedData.path); } - dfile.seekg(0, std::ios_base::end); + dfile.seekg(0, std::ios::end); length = dfile.tellg(); dfile.seekg(0); data = new char[length]; diff --git a/rwlib/source/platform/FileIndex.hpp b/rwlib/source/platform/FileIndex.hpp index e9798aab..9770dc0d 100644 --- a/rwlib/source/platform/FileIndex.hpp +++ b/rwlib/source/platform/FileIndex.hpp @@ -1,91 +1,96 @@ #ifndef _LIBRW_FILEINDEX_HPP_ #define _LIBRW_FILEINDEX_HPP_ -#include -#include -#include -#include +#include "rw/filesystem.hpp" +#include "rw/forward.hpp" + #include -#include -#include - class FileIndex { -private: - /** - * Mapping type (lower case name) => (on disk name) - */ - using FileSystemMap = std::unordered_map; - - rwfs::path gamedatapath_; - FileSystemMap filesystemfiles_; - public: /** - * @brief indexDirectory finds the true case for each file in the tree - * @param base_path + * @brief normalizeString Normalize a file path + * @param filePath the path to normalize + * @return normalized file path + */ + static std::string normalizeFilePath(const std::string &filePath); + + /** + * @brief indexDirectory index all files at path + * @param path the path to index * * This is used to build the mapping of lower-case file paths to the * true case on the file system for platforms where this is an issue. - * */ - void indexGameDirectory(const rwfs::path& base_path); + void indexTree(const rwfs::path &path); /** * @brief findFilePath finds disk path for a game data file - * @param path + * @param filePath the path to find * @return The file path as it exists on disk + * @throws if this FileIndex has not indexed the path */ - rwfs::path findFilePath(std::string path) { - auto backslash = std::string::npos; - while ((backslash = path.find("\\")) != std::string::npos) { - path.replace(backslash, 1, "/"); - } - auto realpath = gamedatapath_ / path; - std::string name = realpath.string(); - std::transform(name.begin(), name.end(), name.begin(), ::tolower); - - return filesystemfiles_[name]; - } + rwfs::path findFilePath(const std::string &filePath) const; /** - * @brief openFilePath opens a file on the disk - * @param file_path - * @return A handle for the file on the disk + * @brief openFileRaw Opens a raw file on the disk + * @param filePath the path to open + * @return FileHandle to the file + * @throws if this FileIndex has not indexed the path */ - FileHandle openFilePath(const std::string& file_path); - - struct IndexData { - /// Lowercase identifying filename - std::string filename; - /// Original filename - std::string originalName; - /// Containing directory - std::string directory; - /// The archive filename (if applicable) - std::string archive; - }; - - /** - * Adds the files contained within the given directory tree to the - * file index. - */ - void indexTree(const rwfs::path& root); + FileHandle openFileRaw(const std::string &filePath) const; /** * Adds the files contained within the given Archive file to the * file index. + * @param filePath path to the archive + * @throws if this FileIndex has not indexed the archive itself */ - void indexArchive(const std::string& archive); + void indexArchive(const std::string &filePath); /** * Returns a FileHandle for the file if it can be found in the * file index, otherwise an empty FileHandle is returned. + * @param filePath name of the file to open + * @return FileHandle to the file, nullptr if this FileINdexed has not indexed the path */ - FileHandle openFile(const std::string& filename); + FileHandle openFile(const std::string &filePath); private: - std::map files; + /** + * @brief Type of the indexed data. + */ + enum IndexedDataType { + /// Is a file on disk + FILE, + /// Is a member of an archive + ARCHIVE, + }; + + /** + * @brief All information of indexed data. + */ + struct IndexedData { + /// Type of indexed data. + IndexedDataType type; + /// Path of indexed data. + std::string path; + /// Extra data of assets (FIXME: use c++17 std::variant or std::option) + std::string assetData; + }; + + /** + * @brief indexedData_ A mapping from filepath (relative to game data path) to an IndexedData item. + */ + std::unordered_map indexedData_; + + /** + * @brief getIndexedDataAt Get IndexedData for filePath + * @param filePath the file path to get the IndexedData for + * @return IndexedData pointer if this FileIndex has indexed the filePath + * @throws If this FileIndex has not indexed filePath + */ + const IndexedData *getIndexedDataAt(const std::string &filePath) const; }; #endif diff --git a/tests/test_FileIndex.cpp b/tests/test_FileIndex.cpp index 2c108b54..7546335d 100644 --- a/tests/test_FileIndex.cpp +++ b/tests/test_FileIndex.cpp @@ -2,23 +2,35 @@ #include #include "test_Globals.hpp" -#include -namespace fs = rwfs; - BOOST_AUTO_TEST_SUITE(FileIndexTests) -#if RW_TEST_WITH_DATA -BOOST_AUTO_TEST_CASE(test_directory_paths) { - FileIndex index; +BOOST_AUTO_TEST_CASE(test_normalizeName) { + std::string ref = "a/b/c"; + { + std::string dirty = "a\\b\\c"; + BOOST_CHECK_EQUAL(ref, FileIndex::normalizeFilePath(dirty)); + } + { + std::string dirty = "A/B/C"; + BOOST_CHECK_EQUAL(ref, FileIndex::normalizeFilePath(dirty)); + } + { + std::string dirty = "A\\B\\C"; + BOOST_CHECK_EQUAL(ref, FileIndex::normalizeFilePath(dirty)); + } +} - index.indexGameDirectory(Global::getGamePath()); +#if RW_TEST_WITH_DATA +BOOST_AUTO_TEST_CASE(test_indexTree) { + FileIndex index; + index.indexTree(Global::getGamePath()); { std::string upperpath{"DATA/CULLZONE.DAT"}; auto truepath = index.findFilePath(upperpath); BOOST_ASSERT(!truepath.empty()); BOOST_CHECK(upperpath != truepath); - fs::path expected{Global::getGamePath()}; + rwfs::path expected{Global::getGamePath()}; expected /= "data/CULLZONE.DAT"; BOOST_CHECK_EQUAL(truepath.string(), expected.string()); } @@ -27,28 +39,35 @@ BOOST_AUTO_TEST_CASE(test_directory_paths) { auto truepath = index.findFilePath(upperpath); BOOST_ASSERT(!truepath.empty()); BOOST_CHECK(upperpath != truepath); - fs::path expected{Global::getGamePath()}; + rwfs::path expected{Global::getGamePath()}; expected /= "data/maps/comnbtm/comNbtm.ipl"; BOOST_CHECK_EQUAL(truepath.string(), expected.string()); } } -BOOST_AUTO_TEST_CASE(test_file) { +BOOST_AUTO_TEST_CASE(test_openFile) { FileIndex index; - index.indexTree(Global::getGamePath() + "/data"); auto handle = index.openFile("cullzone.dat"); BOOST_CHECK(handle != nullptr); } -BOOST_AUTO_TEST_CASE(test_file_archive) { +BOOST_AUTO_TEST_CASE(test_indexArchive) { FileIndex index; + index.indexTree(Global::getGamePath()); - index.indexArchive(Global::getGamePath() + "/models/gta3.img"); + { + auto handle = index.openFile("landstal.dff"); + BOOST_CHECK(handle == nullptr); + } - auto handle = index.openFile("landstal.dff"); - BOOST_CHECK(handle != nullptr); + index.indexArchive("models/gta3.img"); + + { + auto handle = index.openFile("landstal.dff"); + BOOST_CHECK(handle != nullptr); + } } #endif diff --git a/tests/test_Text.cpp b/tests/test_Text.cpp index 3ce4237f..c69654ec 100644 --- a/tests/test_Text.cpp +++ b/tests/test_Text.cpp @@ -11,7 +11,7 @@ BOOST_AUTO_TEST_SUITE(TextTests) #if RW_TEST_WITH_DATA BOOST_AUTO_TEST_CASE(load_test) { { - auto d = Global::get().e->data->index.openFilePath("text/english.gxt"); + auto d = Global::get().e->data->index.openFileRaw("text/english.gxt"); GameTexts texts;