Mirrors/openrw
1
0
Fork 0
mirror of https://github.com/rwengine/openrw.git synced 2024-11-22 02:12:45 +01:00
Code Issues Projects Releases Activity

rwlib: refactor FileIndex

Browse Source 
- make FileIndex const correct (std::map::operator[] modifies the std::map)
- use a single map
- keys are paths relative in the game data directory + filenames
  (so paths like "data/main.scm" can be found twice in the map)
- normalization of the file paths is done inside FileIndex,
  so there shouldn't be any ::tolower's anymore.
- added a normalizer helper function "normalizeFilePath"
- added Documentation
This commit is contained in:
Anonymous Maarten 2018-06-21 23:41:10 +02:00
parent 062115f6bd
commit 8aee672466
7 changed files with 173 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
rwengine/src/engine/GameData.cpp
View File

@ -41,7 +41,6 @@ GameData::GameData(Logger* log, const rwfs::path& path)
GameData::~GameData() = default;
void GameData::load() {
index.indexGameDirectory(datpath);
index.indexTree(datpath);
loadIMG("models/gta3.img");
@ -191,8 +190,7 @@ void GameData::loadCOL(const size_t zone, const std::string& name) {
}
void GameData::loadIMG(const std::string& name) {
auto syspath = index.findFilePath(name).string();
index.indexArchive(syspath);
index.indexArchive(name);
}
void GameData::loadIPL(const std::string& path) {
@ -290,7 +288,7 @@ void GameData::loadHandling(const std::string& path) {
}
SCMFile* GameData::loadSCM(const std::string& path) {
auto scm_h = index.openFilePath(path);
auto scm_h = index.openFileRaw(path);
SCMFile* scm = new SCMFile;
scm->loadFile(scm_h->data, scm_h->length);
scm_h.reset();
@ -298,7 +296,7 @@ SCMFile* GameData::loadSCM(const std::string& path) {
}
void GameData::loadGXT(const std::string& name) {
auto file = index.openFilePath(name);
auto file = index.openFileRaw(name);
LoaderGXT loader;
@ -419,7 +417,7 @@ ClumpPtr GameData::loadClump(const std::string& name, const std::string& texture
}
void GameData::loadModelFile(const std::string& name) {
auto file = index.openFilePath(name);
auto file = index.openFileRaw(name);
if (!file) {
logger->log("Data", Logger::Error, "Failed to load model file " + name);
return;

8
rwengine/src/engine/GameWorld.cpp
View File

@ -691,11 +691,7 @@ void GameWorld::PhysicsTickCallback(btDynamicsWorld* physWorld,
}
void GameWorld::loadCutscene(const std::string& name) {
std::string lowerName(name);
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(),
::tolower);
auto datfile = data->index.openFile(lowerName + ".dat");
auto datfile = data->index.openFile(name + ".dat");
CutsceneData* cutscene = new CutsceneData;
@ -704,7 +700,7 @@ void GameWorld::loadCutscene(const std::string& name) {
loaderdat.load(cutscene->tracks, datfile);
}
data->loadIFP(lowerName + ".ifp");
data->loadIFP(name + ".ifp");
cutsceneAudioLoaded = data->loadAudioStream(name + ".mp3");

4
rwlib/source/platform/FileHandle.hpp
View File

@ -1,13 +1,11 @@
#ifndef _LIBRW_FILEHANDLE_HPP_
#define _LIBRW_FILEHANDLE_HPP_
#include <memory>
/**
* @brief Contains a pointer to a file's contents.
*/
struct FileContentsInfo {
char* data;
char *data;
size_t length;
FileContentsInfo(char* mem, size_t len) : data(mem), length(len) {

136
rwlib/source/platform/FileIndex.cpp
View File

@ -1,38 +1,69 @@
#include "platform/FileIndex.hpp"
#include <algorithm>
#include <cctype>
#include <fstream>
#include <memory>
#include <stdexcept>
#include <boost/range/iterator_range.hpp>
#include <iterator>
#include <sstream>
#include "FileHandle.hpp"
#include "loaders/LoaderIMG.hpp"
#include "platform/FileHandle.hpp"
void FileIndex::indexGameDirectory(const rwfs::path& base_path) {
gamedatapath_ = base_path;
#include "rw/debug.hpp"
for (const rwfs::path& path :
rwfs::recursive_directory_iterator(base_path)) {
std::string FileIndex::normalizeFilePath(const std::string &filePath) {
std::ostringstream oss;
std::transform(filePath.cbegin(), filePath.cend(), std::ostreambuf_iterator<char>(oss), [](char c) {
if (c == '\\') {
return '/';
}
return static_cast<char>(std::tolower(c));
});
return oss.str();
}
void FileIndex::indexTree(const rwfs::path &path) {
// Remove the trailing "/" or "/." from base_path. Boost 1.66 and c++17 have different lexically_relative behavior.
rwfs::path basePath = (path / ".").lexically_normal();
basePath = basePath.parent_path();
for (const rwfs::path &path :
rwfs::recursive_directory_iterator(basePath)) {
if (!rwfs::is_regular_file(path)) {
continue;
}
auto relPath = path.lexically_relative(basePath);
std::string relPathName = normalizeFilePath(relPath.string());
indexedData_[relPathName] = {IndexedDataType::FILE, path.string(), ""};
std::string name = path.string();
std::transform(name.begin(), name.end(), name.begin(), ::tolower);
filesystemfiles_[name] = path;
auto filename = normalizeFilePath(path.filename().string());
indexedData_[filename] = {IndexedDataType::FILE, path.string(), ""};
}
}
FileHandle FileIndex::openFilePath(const std::string& file_path) {
auto datapath = findFilePath(file_path);
std::ifstream dfile(datapath.string(),
std::ios_base::binary | std::ios_base::ate);
const FileIndex::IndexedData *FileIndex::getIndexedDataAt(const std::string &filePath) const {
auto normPath = normalizeFilePath(filePath);
return &indexedData_.at(normPath);
}
rwfs::path FileIndex::findFilePath(const std::string &filePath) const {
return getIndexedDataAt(filePath)->path;
}
FileHandle FileIndex::openFileRaw(const std::string &filePath) const {
const auto *indexData = getIndexedDataAt(filePath);
std::ifstream dfile(indexData->path, std::ios::binary);
if (!dfile.is_open()) {
throw std::runtime_error("Unable to open file: " + file_path);
throw std::runtime_error("Unable to open file: " + filePath);
}
#if RW_DEBUG
if (indexData->type != IndexedDataType::FILE) {
RW_MESSAGE("Reading raw data from archive \"" << filePath << "\"");
}
#endif
dfile.seekg(0, std::ios::end);
auto length = dfile.tellg();
dfile.seekg(0);
auto data = new char[length];
@ -41,84 +72,57 @@ FileHandle FileIndex::openFilePath(const std::string& file_path) {
return std::make_shared<FileContentsInfo>(data, length);
}
void FileIndex::indexTree(const rwfs::path& root) {
for (const rwfs::path& path : rwfs::recursive_directory_iterator(root)) {
if (!rwfs::is_regular_file(path)) {
continue;
}
std::string directory = path.parent_path().string();
std::string realName = path.filename().string();
std::string lowerName = realName;
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(),
::tolower);
files[lowerName] = {lowerName, realName, directory, ""};
}
}
void FileIndex::indexArchive(const std::string& archive) {
// Split directory from archive name
auto archive_path = rwfs::path(archive);
auto directory = archive_path.parent_path();
auto archive_basename = archive_path.filename();
auto archive_full_path = directory / archive_basename;
void FileIndex::indexArchive(const std::string &archive) {
rwfs::path path = findFilePath(archive);
LoaderIMG img;
if (!img.load(archive_full_path.string())) {
throw std::runtime_error("Failed to load IMG archive: " +
archive_full_path.string());
if (!img.load(path.string())) {
throw std::runtime_error("Failed to load IMG archive: " + path.string());
}
std::string lowerName;
for (size_t i = 0; i < img.getAssetCount(); ++i) {
auto& asset = img.getAssetInfoByIndex(i);
auto &asset = img.getAssetInfoByIndex(i);
if (asset.size == 0) continue;
lowerName = asset.name;
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(),
::tolower);
std::string assetName = normalizeFilePath(asset.name);
files[lowerName] = {lowerName, asset.name, directory.string(),
archive_basename.string()};
indexedData_[assetName] = {IndexedDataType::ARCHIVE, path.string(), asset.name};
}
}
FileHandle FileIndex::openFile(const std::string& filename) {
auto iterator = files.find(filename);
if (iterator == files.end()) {
FileHandle FileIndex::openFile(const std::string &filePath) {
auto cleanFilePath = normalizeFilePath(filePath);
auto indexedDataPos = indexedData_.find(cleanFilePath);
if (indexedDataPos == indexedData_.end()) {
return nullptr;
}
IndexData& f = iterator->second;
bool isArchive = !f.archive.empty();
const auto &indexedData = indexedDataPos->second;
auto fsName = f.directory + "/" + f.originalName;
char* data = nullptr;
char *data = nullptr;
size_t length = 0;
if (isArchive) {
fsName = f.directory + "/" + f.archive;
if (indexedData.type == IndexedDataType::ARCHIVE) {
LoaderIMG img;
if (!img.load(fsName)) {
throw std::runtime_error("Failed to load IMG archive: " + fsName);
if (!img.load(indexedData.path)) {
throw std::runtime_error("Failed to load IMG archive: " + indexedData.path);
}
LoaderIMGFile file;
if (img.findAssetInfo(f.originalName, file)) {
auto filename = rwfs::path(indexedData.assetData).filename().string();
if (img.findAssetInfo(filename, file)) {
length = file.size * 2048;
data = img.loadToMemory(f.originalName);
data = img.loadToMemory(filename);
}
} else {
std::ifstream dfile(fsName.c_str(), std::ios_base::binary);
std::ifstream dfile(indexedData.path, std::ios::binary);
if (!dfile.is_open()) {
throw std::runtime_error("Unable to open file: " + fsName);
throw std::runtime_error("Unable to open file: " + indexedData.path);
}
dfile.seekg(0, std::ios_base::end);
dfile.seekg(0, std::ios::end);
length = dfile.tellg();
dfile.seekg(0);
data = new char[length];

117
rwlib/source/platform/FileIndex.hpp
View File

@ -1,91 +1,96 @@
#ifndef _LIBRW_FILEINDEX_HPP_
#define _LIBRW_FILEINDEX_HPP_
#include <algorithm>
#include <cctype>
#include <map>
#include <string>
#include "rw/filesystem.hpp"
#include "rw/forward.hpp"
#include <unordered_map>
#include <rw/filesystem.hpp>
#include <rw/forward.hpp>
class FileIndex {
private:
/**
* Mapping type (lower case name) => (on disk name)
*/
using FileSystemMap = std::unordered_map<rwfs::path, rwfs::path>;
rwfs::path gamedatapath_;
FileSystemMap filesystemfiles_;
public:
/**
* @brief indexDirectory finds the true case for each file in the tree
* @param base_path
* @brief normalizeString Normalize a file path
* @param filePath the path to normalize
* @return normalized file path
*/
static std::string normalizeFilePath(const std::string &filePath);
/**
* @brief indexDirectory index all files at path
* @param path the path to index
*
* This is used to build the mapping of lower-case file paths to the
* true case on the file system for platforms where this is an issue.
*
*/
void indexGameDirectory(const rwfs::path& base_path);
void indexTree(const rwfs::path &path);
/**
* @brief findFilePath finds disk path for a game data file
* @param path
* @param filePath the path to find
* @return The file path as it exists on disk
* @throws if this FileIndex has not indexed the path
*/
rwfs::path findFilePath(std::string path) {
auto backslash = std::string::npos;
while ((backslash = path.find("\\")) != std::string::npos) {
path.replace(backslash, 1, "/");
}
auto realpath = gamedatapath_ / path;
std::string name = realpath.string();
std::transform(name.begin(), name.end(), name.begin(), ::tolower);
return filesystemfiles_[name];
}
rwfs::path findFilePath(const std::string &filePath) const;
/**
* @brief openFilePath opens a file on the disk
* @param file_path
* @return A handle for the file on the disk
* @brief openFileRaw Opens a raw file on the disk
* @param filePath the path to open
* @return FileHandle to the file
* @throws if this FileIndex has not indexed the path
*/
FileHandle openFilePath(const std::string& file_path);
struct IndexData {
/// Lowercase identifying filename
std::string filename;
/// Original filename
std::string originalName;
/// Containing directory
std::string directory;
/// The archive filename (if applicable)
std::string archive;
};
/**
* Adds the files contained within the given directory tree to the
* file index.
*/
void indexTree(const rwfs::path& root);
FileHandle openFileRaw(const std::string &filePath) const;
/**
* Adds the files contained within the given Archive file to the
* file index.
* @param filePath path to the archive
* @throws if this FileIndex has not indexed the archive itself
*/
void indexArchive(const std::string& archive);
void indexArchive(const std::string &filePath);
/**
* Returns a FileHandle for the file if it can be found in the
* file index, otherwise an empty FileHandle is returned.
* @param filePath name of the file to open
* @return FileHandle to the file, nullptr if this FileINdexed has not indexed the path
*/
FileHandle openFile(const std::string& filename);
FileHandle openFile(const std::string &filePath);
private:
std::map<std::string, IndexData> files;
/**
* @brief Type of the indexed data.
*/
enum IndexedDataType {
/// Is a file on disk
FILE,
/// Is a member of an archive
ARCHIVE,
};
/**
* @brief All information of indexed data.
*/
struct IndexedData {
/// Type of indexed data.
IndexedDataType type;
/// Path of indexed data.
std::string path;
/// Extra data of assets (FIXME: use c++17 std::variant or std::option)
std::string assetData;
};
/**
* @brief indexedData_ A mapping from filepath (relative to game data path) to an IndexedData item.
*/
std::unordered_map<std::string, IndexedData> indexedData_;
/**
* @brief getIndexedDataAt Get IndexedData for filePath
* @param filePath the file path to get the IndexedData for
* @return IndexedData pointer if this FileIndex has indexed the filePath
* @throws If this FileIndex has not indexed filePath
*/
const IndexedData *getIndexedDataAt(const std::string &filePath) const;
};
#endif

49
tests/test_FileIndex.cpp
View File

@ -2,23 +2,35 @@
#include <platform/FileIndex.hpp>
#include "test_Globals.hpp"
#include <rw/filesystem.hpp>
namespace fs = rwfs;
BOOST_AUTO_TEST_SUITE(FileIndexTests)
#if RW_TEST_WITH_DATA
BOOST_AUTO_TEST_CASE(test_directory_paths) {
FileIndex index;
BOOST_AUTO_TEST_CASE(test_normalizeName) {
std::string ref = "a/b/c";
{
std::string dirty = "a\\b\\c";
BOOST_CHECK_EQUAL(ref, FileIndex::normalizeFilePath(dirty));
}
{
std::string dirty = "A/B/C";
BOOST_CHECK_EQUAL(ref, FileIndex::normalizeFilePath(dirty));
}
{
std::string dirty = "A\\B\\C";
BOOST_CHECK_EQUAL(ref, FileIndex::normalizeFilePath(dirty));
}
}
index.indexGameDirectory(Global::getGamePath());
#if RW_TEST_WITH_DATA
BOOST_AUTO_TEST_CASE(test_indexTree) {
FileIndex index;
index.indexTree(Global::getGamePath());
{
std::string upperpath{"DATA/CULLZONE.DAT"};
auto truepath = index.findFilePath(upperpath);
BOOST_ASSERT(!truepath.empty());
BOOST_CHECK(upperpath != truepath);
fs::path expected{Global::getGamePath()};
rwfs::path expected{Global::getGamePath()};
expected /= "data/CULLZONE.DAT";
BOOST_CHECK_EQUAL(truepath.string(), expected.string());
}
@ -27,28 +39,35 @@ BOOST_AUTO_TEST_CASE(test_directory_paths) {
auto truepath = index.findFilePath(upperpath);
BOOST_ASSERT(!truepath.empty());
BOOST_CHECK(upperpath != truepath);
fs::path expected{Global::getGamePath()};
rwfs::path expected{Global::getGamePath()};
expected /= "data/maps/comnbtm/comNbtm.ipl";
BOOST_CHECK_EQUAL(truepath.string(), expected.string());
}
}
BOOST_AUTO_TEST_CASE(test_file) {
BOOST_AUTO_TEST_CASE(test_openFile) {
FileIndex index;
index.indexTree(Global::getGamePath() + "/data");
auto handle = index.openFile("cullzone.dat");
BOOST_CHECK(handle != nullptr);
}
BOOST_AUTO_TEST_CASE(test_file_archive) {
BOOST_AUTO_TEST_CASE(test_indexArchive) {
FileIndex index;
index.indexTree(Global::getGamePath());
index.indexArchive(Global::getGamePath() + "/models/gta3.img");
{
auto handle = index.openFile("landstal.dff");
BOOST_CHECK(handle == nullptr);
}
auto handle = index.openFile("landstal.dff");
BOOST_CHECK(handle != nullptr);
index.indexArchive("models/gta3.img");
{
auto handle = index.openFile("landstal.dff");
BOOST_CHECK(handle != nullptr);
}
}
#endif

2
tests/test_Text.cpp
View File

@ -11,7 +11,7 @@ BOOST_AUTO_TEST_SUITE(TextTests)
#if RW_TEST_WITH_DATA
BOOST_AUTO_TEST_CASE(load_test) {
{
auto d = Global::get().e->data->index.openFilePath("text/english.gxt");
auto d = Global::get().e->data->index.openFileRaw("text/english.gxt");
GameTexts texts;