Remove old file indexing system and IO handling

- Use FileIndex inside GameData to handle normalisation
- Remove old raw pointer API for loading data

rwengine/include/core/FileIndex.hpp

@@ -26,6 +26,12 @@ public:
 	 */
 	void indexDirectory(const std::string& directory);
 	
+	/**
+	 * Adds the files contained within the given directory tree to the
+	 * file index.
+	 */
+	void indexTree(const std::string& root);
+	
 	/**
 	 * Adds the files contained within the given Archive file to the
 	 * file index.

rwengine/include/engine/GameData.hpp

@@ -16,6 +16,7 @@
 
 #include <audio/MADStream.hpp>
 #include <render/TextureData.hpp>
+#include <core/FileIndex.hpp>
 
 #include <memory>
 
@@ -41,17 +42,7 @@ private:
 	std::string splash;
 	
 public:
-	
+
-	/**
-	 * @struct FileInfo
-	 *  Stores information about a file the engine might want to load
-	 */
-	struct FileInfo
-	{
-		bool archived; /// Is the file inside an IMG or on the filesystem?
-		std::string path; /// Path to the file containing the file.
-	};
-	
 	/**
 	 * ctor
 	 * @param path Path to the root of the game data.
@@ -152,21 +143,7 @@ public:
 
 	void loadSplash(const std::string& name);
 
-	/**
+	FileHandle openFile(const std::string& name);
-	 * Returns a pointer to the named file if it is available,
-	 * the memory must be freed by the caller.
-	 * @param name the filename in the archive
-	 * @return pointer to the data, NULL if it is not available
-	 */
-	char* openFile(const std::string& name);
-	FileHandle openFile2(const std::string& name);
-
-	/**
-	 * @brief loadFile Marks a file as open, and opens it.
-	 * @param name
-	 * @return
-	 */
-	char* loadFile(const std::string& name);
 
 	/**
 	 * @brief getAtlas Returns atlas i, creating it if the situation calls for it.
@@ -181,6 +158,8 @@ public:
 		return textures[{name, alpha}];
 	}
 	
+	FileIndex index;
+	
 	/**
 	 * Files that have been loaded previously
 	 */
@@ -191,12 +170,7 @@ public:
 	 */
 	std::map<std::string, std::string> iplLocations;
 	std::map<std::string, std::string> ideLocations;
-	
+
-	/**
-	 * Maps file names to data about the file.
-	 */
-	std::map<std::string, FileInfo> _knownFiles;
-	
 	/**
 	 * Map of loaded archives
 	 */

rwengine/include/loaders/LoaderDFF.hpp

@@ -68,7 +68,7 @@ private:
 	GameData* _gameData;
 	std::string _file;
 	ModelCallback _callback;
-	FileHandle _data;
+	FileHandle data;
 public:
 
 	LoadModelJob(WorkContext* context, GameData* gd, const std::string& file, ModelCallback cb);

rwengine/include/loaders/TextureLoader.hpp

@@ -7,6 +7,9 @@
 
 #include <loaders/rwbinarystream.h>
 
+#include <WorkContext.hpp>
+#include <core/FileHandle.hpp>
+#include <functional>
 #include <string>
 #include <map>
 
@@ -15,20 +18,16 @@ class GameData;
 class TextureLoader
 {
 public:
-	bool loadFromFile(std::string filename, GameData* gameData);
+	bool loadFromMemory(FileHandle file, GameData* gameData);
-	bool loadFromMemory(char *data, GameData* gameData);
 };
 
-#include <WorkContext.hpp>
-#include <functional>
-
 // TODO: refactor this interface to be more like ModelLoader so they can be rolled into one.
 class LoadTextureArchiveJob : public WorkJob
 {
 private:
 	GameData* _gameData;
 	std::string _file;
-	char* _data;
+	FileHandle data;
 public:
 
 	LoadTextureArchiveJob(WorkContext* context, GameData* gd, const std::string& file);

rwengine/src/core/FileIndex.cpp

@@ -29,6 +29,27 @@ void FileIndex::indexDirectory(const std::string& directory)
 			};
 		}
 	}
+	closedir(dp);
+}
+
+void FileIndex::indexTree(const std::string& root)
+{
+	indexDirectory(root);
+	
+	DIR* dp = opendir(root.c_str());
+	dirent* ep;
+	if ( dp == NULL ) {
+		throw std::runtime_error("Unable to open directory: " + root);
+	}
+	while( (ep = readdir(dp)) )
+	{
+		if( ep->d_type == DT_DIR && ep->d_name[0] != '.' )
+		{
+			std::string path = root + "/" + ep->d_name;
+			indexTree(path);
+		}
+	}
+	closedir(dp);
 }
 
 void FileIndex::indexArchive(const std::string& archive)
@@ -120,11 +141,16 @@ FileHandle FileIndex::openFile(const std::string& filename)
 		}
 
 		dfile.seekg(0, std::ios_base::end);
-		size_t length = dfile.tellg();
+		length = dfile.tellg();
 		dfile.seekg(0);
-		char *data = new char[length];
+		data = new char[length];
 		dfile.read(data, length);
 	}
 	
+	if( data == nullptr )
+	{
+		return nullptr;
+	}
+	
 	return FileHandle( new FileContentsInfo{ data, length } );
 }

rwengine/src/engine/GameData.cpp

@@ -41,7 +41,7 @@ std::string findPathRealCase(const std::string& base, const std::string& path)
 	// Open the current "base" path (i.e. the real path)
 	DIR* dp = opendir(base.c_str());
 	dirent* ep;
-		
+	
 	if( dp != NULL) {
 		while( (ep = readdir(dp)) ) {
 			realName = ep->d_name;
@@ -96,25 +96,12 @@ GameData::~GameData()
 
 void GameData::load()
 {
+	index.indexTree(datpath);
+	
 	parseDAT(datpath+"/data/default.dat");
 	parseDAT(datpath+"/data/gta3.dat");
 	
-	_knownFiles.insert({"wheels.DFF", {false, datpath+"/models/Generic/wheels.DFF"}});
+	loadDFF("wheels.dff");
-	_knownFiles.insert({"loplyguy.dff", {false, datpath+"/models/Generic/loplyguy.dff"}});
-	_knownFiles.insert({"weapons.dff", {false, datpath+"/models/Generic/weapons.dff"}});
-	_knownFiles.insert({"arrow.dff", {false, datpath+"/models/Generic/arrow.DFF"}});
-	_knownFiles.insert({"particle.txd", {false, datpath+"/models/particle.txd"}});
-	_knownFiles.insert({"hud.txd", {false, datpath+"/models/hud.txd"}});
-	_knownFiles.insert({"english.gxt", {false, datpath+"/TEXT/english.gxt"}});
-	_knownFiles.insert({"ped.ifp", {false, datpath+"/anim/ped.ifp"}});
-	_knownFiles.insert({"fonts.txd", {false, datpath+"/models/fonts.txd"}});
-
-	_knownFiles.insert({"news.txd", {false, datpath+"/txd/NEWS.TXD"}});
-	_knownFiles.insert({"splash1.txd", {false, datpath+"/txd/SPLASH1.TXD"}});
-	_knownFiles.insert({"splash2.txd", {false, datpath+"/txd/SPLASH2.TXD"}});
-	_knownFiles.insert({"splash3.txd", {false, datpath+"/txd/SPLASH3.TXD"}});
-
-	loadDFF("wheels.DFF");
 	loadDFF("weapons.dff");
 	loadDFF("arrow.dff");
 	loadTXD("particle.txd");
@@ -179,9 +166,7 @@ void GameData::parseDAT(const std::string& path)
 							texpath[t] = '/';
 						}
 					}
-					texpath = findPathRealCase(datpath, texpath);
 					std::string texname = texpath.substr(texpath.find_last_of("/")+1);
-					_knownFiles.insert({ texname, { false, texpath }});
 					loadTXD(texname);
 				}
 			}
@@ -219,32 +204,7 @@ void GameData::loadCOL(const size_t zone, const std::string& name)
 
 void GameData::loadIMG(const std::string& name)
 {
-	LoaderIMG imgLoader; 
+	index.indexArchive(datpath + name);
-	std::string archivePath = datpath + name;
-
-	if (imgLoader.load(archivePath)) {
-		for (size_t i = 0; i < imgLoader.getAssetCount(); i++) {
-			auto &asset = imgLoader.getAssetInfoByIndex(i);
-
-			std::string filename = asset.name;
-			
-			if(asset.size == 0)
-			{
-				engine->logger.warning("Data", "Ignoring asset " + filename + " with size 0");
-			}
-			else
-            {
-				// Enter the asset twice.. 
-				_knownFiles.insert({ filename, { true, archivePath }});
-				for(size_t t = 0; t < filename.size(); ++t)
-				{
-					filename[t] = tolower(filename[t]);
-				}
-				_knownFiles.insert({ filename, { true, archivePath }});
-			}
-		}
-		archives.insert({archivePath, imgLoader});
-	}
 }
 
 void GameData::loadIPL(const std::string& name)
@@ -364,7 +324,7 @@ SCMFile *GameData::loadSCM(const std::string &path)
 
 void GameData::loadGXT(const std::string &name)
 {
-	auto d = openFile2(name);
+	auto d = openFile(name);
 
 	LoaderGXT loader;
 
@@ -470,7 +430,7 @@ void GameData::loadDFF(const std::string& name, bool async)
 
 void GameData::loadIFP(const std::string &name)
 {
-	auto f = openFile2(name);
+	auto f = openFile(name);
 
 	if(f)
 	{
@@ -563,109 +523,14 @@ void GameData::loadSplash(const std::string &name)
 	engine->state.currentSplash = lower;
 }
 
-char* GameData::openFile(const std::string& name)
+FileHandle GameData::openFile(const std::string &name)
 {
-	auto i = _knownFiles.find(name);
+	auto file = index.openFile(name);
-	if(i != _knownFiles.end())
+	if( file == nullptr )
 	{
-		if(i->second.archived)
+		engine->logger.error("Data", "Unable to open file: " + name);
-		{
-			// Find the archive
-			auto ai = archives.find(i->second.path);
-			if(ai != archives.end())
-			{
-				return ai->second.loadToMemory(name);
-			}
-			else 
-			{
-				engine->logger.error("Data", "Archive not found " + i->second.path);
-			}
-		}
-		else
-		{
-			std::ifstream dfile(i->second.path);
-			if ( ! dfile.is_open()) {
-				engine->logger.error("Data", "Error opening file " + i->second.path);
-				return nullptr;
-			}
-
-			dfile.seekg(0, std::ios_base::end);
-			size_t length = dfile.tellg();
-			dfile.seekg(0);
-			char *data = new char[length];
-			dfile.read(data, length);
-			
-			return data;
-		}
 	}
-	else 
+	return file;
-	{
-		engine->logger.error("Data", "Unable to locate file: " + name);
-	}
-	
-	return nullptr;
-}
-
-FileHandle GameData::openFile2(const std::string &name)
-{
-	auto i = _knownFiles.find(name);
-	if(i != _knownFiles.end())
-	{
-		char* data = nullptr;
-		size_t length = 0;
-
-		if(i->second.archived)
-		{
-			// Find the archive
-			auto ai = archives.find(i->second.path);
-			if(ai != archives.end())
-			{
-				LoaderIMGFile asset;
-				if( ai->second.findAssetInfo(name, asset) )
-				{
-					data = ai->second.loadToMemory(name);
-					length = asset.size * 2048;
-				}
-			}
-			else
-			{
-				engine->logger.error("Data", "Archive not found " + i->second.path);
-			}
-		}
-		else
-		{
-			std::ifstream dfile(i->second.path);
-			if ( ! dfile.is_open()) {
-				engine->logger.error("Data", "Error opening file " + i->second.path);
-				return nullptr;
-			}
-
-			dfile.seekg(0, std::ios_base::end);
-			length = dfile.tellg();
-			dfile.seekg(0);
-			data = new char[length];
-			dfile.read(data, length);
-		}
-
-		return FileHandle( new FileContentsInfo{ data, length } );
-	}
-	else
-	{
-		engine->logger.error("Data", "Unable to locate file: " + name);
-	}
-	return nullptr;
-}
-
-char* GameData::loadFile(const std::string &name)
-{
-	auto it = loadedFiles.find(name);
-	 if( it != loadedFiles.end() ) {
-		engine->logger.warning("Data", "File " + name + " already loaded");
-	}
-
-	loadedFiles[name] = true;
-
-	return openFile(name);
 }
 
 TextureAtlas* GameData::getAtlas(size_t i)

rwengine/src/engine/GameWorld.cpp

@@ -215,6 +215,9 @@ InstanceObject *GameWorld::createInstance(const uint16_t id, const glm::vec3& po
 		std::string modelname = oi->modelName;
 		std::string texturename = oi->textureName;
 
+		std::transform(std::begin(modelname), std::end(modelname), std::begin(modelname), tolower);
+		std::transform(std::begin(texturename), std::end(texturename), std::begin(texturename), tolower);
+
 		// Ensure the relevant data is loaded.
 		if(! oi->modelName.empty()) {
 			if( modelname != "null" ) {
@@ -660,7 +663,7 @@ void GameWorld::loadCutscene(const std::string &name)
 	std::string lowerName(name);
 	std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
 
-	auto datfile = gameData.openFile2(lowerName + ".dat");
+	auto datfile = gameData.openFile(lowerName + ".dat");
 
 	CutsceneData* cutscene = new CutsceneData;
 

rwengine/src/loaders/LoaderDFF.cpp

@@ -456,26 +456,26 @@ Model* LoaderDFF::loadFromMemory(FileHandle file, GameData *gameData)
 }
 
 LoadModelJob::LoadModelJob(WorkContext *context, GameData* gd, const std::string &file, ModelCallback cb)
-	: WorkJob(context), _gameData(gd), _file(file), _callback(cb), _data(nullptr)
+	: WorkJob(context), _gameData(gd), _file(file), _callback(cb)
 {
 
 }
 
 void LoadModelJob::work()
 {
-	_data = _gameData->openFile2(_file);
+	data = _gameData->openFile(_file);
 }
 
 void LoadModelJob::complete()
 {
 	Model* m = nullptr;
 	// TODO error status
-	if( _data ) {
+	if( data ) {
 
 		// TODO allow some of the loading to process in a seperate thread.
 		LoaderDFF loader;
 
-		m = loader.loadFromMemory(_data, _gameData);
+		m = loader.loadFromMemory(data, _gameData);
 	}
 
 	_callback(m);

rwengine/src/loaders/TextureLoader.cpp

@@ -7,23 +7,6 @@
 #include <iostream>
 #include <algorithm>
 
-bool TextureLoader::loadFromFile(std::string filename, GameData* gameData)
-{
-	std::ifstream dfile(filename);
-	if ( ! dfile.is_open()) {
-		std::cerr << "Error opening file " << filename << std::endl;
-		return false;
-	}
-
-	dfile.seekg(0, std::ios_base::end);
-	size_t length = dfile.tellg();
-	dfile.seekg(0);
-	char *data = new char[length];
-	dfile.read(data, length);
-
-	return loadFromMemory(data, gameData);
-}
-
 GLuint gErrorTextureData[] = { 0xFFFF00FF, 0xFF000000, 0xFF000000, 0xFFFF00FF };
 GLuint gDebugTextureData[] = {0xFF0000FF, 0xFF00FF00};
 GLuint gTextureRed[] = {0xFF0000FF};
@@ -201,8 +184,9 @@ TextureData::Handle createTexture(RW::BSTextureNative& texNative, RW::BinaryStre
 	return TextureData::create( textureName, { texNative.width, texNative.height }, transparent );
 }
 
-bool TextureLoader::loadFromMemory(char *data, GameData *gameData)
+bool TextureLoader::loadFromMemory(FileHandle file, GameData *gameData)
 {
+	auto data = file->data;
 	RW::BinaryStreamSection root(data);
 	/*auto texDict =*/ root.readStructure<RW::BSTextureDictionary>();
 
@@ -218,7 +202,7 @@ bool TextureLoader::loadFromMemory(char *data, GameData *gameData)
 		std::string alpha = std::string(texNative.alphaName);
 		std::transform(name.begin(), name.end(), name.begin(), ::tolower );
 		std::transform(alpha.begin(), alpha.end(), alpha.begin(), ::tolower );
-
+		
 		auto texture = createTexture(texNative, rootSection);
 
 		gameData->textures[{name, alpha}] = texture;
@@ -233,23 +217,21 @@ bool TextureLoader::loadFromMemory(char *data, GameData *gameData)
 
 
 LoadTextureArchiveJob::LoadTextureArchiveJob(WorkContext *context, GameData *gd, const std::string &file)
-	: WorkJob(context), _gameData(gd), _file(file), _data(nullptr)
+	: WorkJob(context), _gameData(gd), _file(file)
 {
 
 }
 
 void LoadTextureArchiveJob::work()
 {
-	_data = _gameData->openFile(_file);
+	data = _gameData->openFile(_file);
 }
 
 void LoadTextureArchiveJob::complete()
 {
 	// TODO error status
-	if(_data) {
+	if(data) {
 		TextureLoader loader;
-		loader.loadFromMemory(_data, _gameData);
+		loader.loadFromMemory(data, _gameData);
 	}
-
-	delete[] _data;
 }

rwengine/src/render/GameRenderer.cpp

@@ -647,7 +647,7 @@ void GameRenderer::renderInstance(InstanceObject *instance)
 	}
 
 	glm::mat4 matrixModel;
-	if( instance->body ) {
+	if( instance->body && instance->body->body ) {
 		instance->body->body->getWorldTransform().getOpenGLMatrix(glm::value_ptr(matrixModel));
 	}
 	else {
@@ -917,6 +917,10 @@ void GameRenderer::renderGeometry(Model* model, size_t g, const glm::mat4& model
 					auto& tC = mat.textures[0].name;
 					auto& tA = mat.textures[0].alphaName;
 					tex = engine->gameData.findTexture(tC, tA);
+					if( ! tex )
+					{
+						//engine->logger.warning("Renderer", "Missing texture: " + tC + " " + tA);
+					}
 					mat.textures[0].texture = tex;
 				}
 				if( tex )

rwengine/src/render/GameShaders.cpp

@@ -257,6 +257,6 @@ void main()
 {
 	vec4 c = texture2D(texture, TexCoords);
 	// Set colour to 0, 0, 0, 1 for textured mode.
-	outColour = colour + c.rgba;
+	outColour = vec4(colour.rgb + c.rgb, colour.a);
 })";
 }

rwgame/RWGame.cpp

@@ -70,7 +70,7 @@ RWGame::RWGame(const std::string& gamepath, int argc, char* argv[])
 	
 	// Initalize all the archives.
 	engine->gameData.loadIMG("/models/gta3");
-	engine->gameData.loadIMG("/models/txd");
+	//engine->gameData.loadIMG("/models/txd");
 	engine->gameData.loadIMG("/anim/cuts");
 	
 	// Initialize renderer