From dfe6ec3eaa960eb016f81d97d64c40746ddccbd5 Mon Sep 17 00:00:00 2001
From: Daniel Evans <danharibo@gmail.com>
Date: Fri, 6 Mar 2015 01:40:29 +0000
Subject: [PATCH] Remove old file indexing system and IO handling

- Use FileIndex inside GameData to handle normalisation
- Remove old raw pointer API for loading data
---
 rwengine/include/core/FileIndex.hpp        |   6 +
 rwengine/include/engine/GameData.hpp       |  38 +----
 rwengine/include/loaders/LoaderDFF.hpp     |   2 +-
 rwengine/include/loaders/TextureLoader.hpp |  11 +-
 rwengine/src/core/FileIndex.cpp            |  30 +++-
 rwengine/src/engine/GameData.cpp           | 159 ++-------------------
 rwengine/src/engine/GameWorld.cpp          |   5 +-
 rwengine/src/loaders/LoaderDFF.cpp         |   8 +-
 rwengine/src/loaders/TextureLoader.cpp     |  32 +----
 rwengine/src/render/GameRenderer.cpp       |   6 +-
 rwengine/src/render/GameShaders.cpp        |   2 +-
 rwgame/RWGame.cpp                          |   2 +-
 12 files changed, 80 insertions(+), 221 deletions(-)

diff --git a/rwengine/include/core/FileIndex.hpp b/rwengine/include/core/FileIndex.hpp
index 67d7b8e6..51695379 100644
--- a/rwengine/include/core/FileIndex.hpp
+++ b/rwengine/include/core/FileIndex.hpp
@@ -26,6 +26,12 @@ public:
 	 */
 	void indexDirectory(const std::string& directory);
 	
+	/**
+	 * Adds the files contained within the given directory tree to the
+	 * file index.
+	 */
+	void indexTree(const std::string& root);
+	
 	/**
 	 * Adds the files contained within the given Archive file to the
 	 * file index.
diff --git a/rwengine/include/engine/GameData.hpp b/rwengine/include/engine/GameData.hpp
index fa9a4bf2..c246a3c7 100644
--- a/rwengine/include/engine/GameData.hpp
+++ b/rwengine/include/engine/GameData.hpp
@@ -16,6 +16,7 @@
 
 #include <audio/MADStream.hpp>
 #include <render/TextureData.hpp>
+#include <core/FileIndex.hpp>
 
 #include <memory>
 
@@ -41,17 +42,7 @@ private:
 	std::string splash;
 	
 public:
-	
-	/**
-	 * @struct FileInfo
-	 *  Stores information about a file the engine might want to load
-	 */
-	struct FileInfo
-	{
-		bool archived; /// Is the file inside an IMG or on the filesystem?
-		std::string path; /// Path to the file containing the file.
-	};
-	
+
 	/**
 	 * ctor
 	 * @param path Path to the root of the game data.
@@ -152,21 +143,7 @@ public:
 
 	void loadSplash(const std::string& name);
 
-	/**
-	 * Returns a pointer to the named file if it is available,
-	 * the memory must be freed by the caller.
-	 * @param name the filename in the archive
-	 * @return pointer to the data, NULL if it is not available
-	 */
-	char* openFile(const std::string& name);
-	FileHandle openFile2(const std::string& name);
-
-	/**
-	 * @brief loadFile Marks a file as open, and opens it.
-	 * @param name
-	 * @return
-	 */
-	char* loadFile(const std::string& name);
+	FileHandle openFile(const std::string& name);
 
 	/**
 	 * @brief getAtlas Returns atlas i, creating it if the situation calls for it.
@@ -181,6 +158,8 @@ public:
 		return textures[{name, alpha}];
 	}
 	
+	FileIndex index;
+	
 	/**
 	 * Files that have been loaded previously
 	 */
@@ -191,12 +170,7 @@ public:
 	 */
 	std::map<std::string, std::string> iplLocations;
 	std::map<std::string, std::string> ideLocations;
-	
-	/**
-	 * Maps file names to data about the file.
-	 */
-	std::map<std::string, FileInfo> _knownFiles;
-	
+
 	/**
 	 * Map of loaded archives
 	 */
diff --git a/rwengine/include/loaders/LoaderDFF.hpp b/rwengine/include/loaders/LoaderDFF.hpp
index faaae83c..99c3cd58 100644
--- a/rwengine/include/loaders/LoaderDFF.hpp
+++ b/rwengine/include/loaders/LoaderDFF.hpp
@@ -68,7 +68,7 @@ private:
 	GameData* _gameData;
 	std::string _file;
 	ModelCallback _callback;
-	FileHandle _data;
+	FileHandle data;
 public:
 
 	LoadModelJob(WorkContext* context, GameData* gd, const std::string& file, ModelCallback cb);
diff --git a/rwengine/include/loaders/TextureLoader.hpp b/rwengine/include/loaders/TextureLoader.hpp
index 02cccc28..11a976ca 100644
--- a/rwengine/include/loaders/TextureLoader.hpp
+++ b/rwengine/include/loaders/TextureLoader.hpp
@@ -7,6 +7,9 @@
 
 #include <loaders/rwbinarystream.h>
 
+#include <WorkContext.hpp>
+#include <core/FileHandle.hpp>
+#include <functional>
 #include <string>
 #include <map>
 
@@ -15,20 +18,16 @@ class GameData;
 class TextureLoader
 {
 public:
-	bool loadFromFile(std::string filename, GameData* gameData);
-	bool loadFromMemory(char *data, GameData* gameData);
+	bool loadFromMemory(FileHandle file, GameData* gameData);
 };
 
-#include <WorkContext.hpp>
-#include <functional>
-
 // TODO: refactor this interface to be more like ModelLoader so they can be rolled into one.
 class LoadTextureArchiveJob : public WorkJob
 {
 private:
 	GameData* _gameData;
 	std::string _file;
-	char* _data;
+	FileHandle data;
 public:
 
 	LoadTextureArchiveJob(WorkContext* context, GameData* gd, const std::string& file);
diff --git a/rwengine/src/core/FileIndex.cpp b/rwengine/src/core/FileIndex.cpp
index 281cbe63..0837cc90 100644
--- a/rwengine/src/core/FileIndex.cpp
+++ b/rwengine/src/core/FileIndex.cpp
@@ -29,6 +29,27 @@ void FileIndex::indexDirectory(const std::string& directory)
 			};
 		}
 	}
+	closedir(dp);
+}
+
+void FileIndex::indexTree(const std::string& root)
+{
+	indexDirectory(root);
+	
+	DIR* dp = opendir(root.c_str());
+	dirent* ep;
+	if ( dp == NULL ) {
+		throw std::runtime_error("Unable to open directory: " + root);
+	}
+	while( (ep = readdir(dp)) )
+	{
+		if( ep->d_type == DT_DIR && ep->d_name[0] != '.' )
+		{
+			std::string path = root + "/" + ep->d_name;
+			indexTree(path);
+		}
+	}
+	closedir(dp);
 }
 
 void FileIndex::indexArchive(const std::string& archive)
@@ -120,11 +141,16 @@ FileHandle FileIndex::openFile(const std::string& filename)
 		}
 
 		dfile.seekg(0, std::ios_base::end);
-		size_t length = dfile.tellg();
+		length = dfile.tellg();
 		dfile.seekg(0);
-		char *data = new char[length];
+		data = new char[length];
 		dfile.read(data, length);
 	}
 	
+	if( data == nullptr )
+	{
+		return nullptr;
+	}
+	
 	return FileHandle( new FileContentsInfo{ data, length } );
 }
diff --git a/rwengine/src/engine/GameData.cpp b/rwengine/src/engine/GameData.cpp
index 5bb0bd11..3703f24d 100644
--- a/rwengine/src/engine/GameData.cpp
+++ b/rwengine/src/engine/GameData.cpp
@@ -41,7 +41,7 @@ std::string findPathRealCase(const std::string& base, const std::string& path)
 	// Open the current "base" path (i.e. the real path)
 	DIR* dp = opendir(base.c_str());
 	dirent* ep;
-		
+	
 	if( dp != NULL) {
 		while( (ep = readdir(dp)) ) {
 			realName = ep->d_name;
@@ -96,25 +96,12 @@ GameData::~GameData()
 
 void GameData::load()
 {
+	index.indexTree(datpath);
+	
 	parseDAT(datpath+"/data/default.dat");
 	parseDAT(datpath+"/data/gta3.dat");
 	
-	_knownFiles.insert({"wheels.DFF", {false, datpath+"/models/Generic/wheels.DFF"}});
-	_knownFiles.insert({"loplyguy.dff", {false, datpath+"/models/Generic/loplyguy.dff"}});
-	_knownFiles.insert({"weapons.dff", {false, datpath+"/models/Generic/weapons.dff"}});
-	_knownFiles.insert({"arrow.dff", {false, datpath+"/models/Generic/arrow.DFF"}});
-	_knownFiles.insert({"particle.txd", {false, datpath+"/models/particle.txd"}});
-	_knownFiles.insert({"hud.txd", {false, datpath+"/models/hud.txd"}});
-	_knownFiles.insert({"english.gxt", {false, datpath+"/TEXT/english.gxt"}});
-	_knownFiles.insert({"ped.ifp", {false, datpath+"/anim/ped.ifp"}});
-	_knownFiles.insert({"fonts.txd", {false, datpath+"/models/fonts.txd"}});
-
-	_knownFiles.insert({"news.txd", {false, datpath+"/txd/NEWS.TXD"}});
-	_knownFiles.insert({"splash1.txd", {false, datpath+"/txd/SPLASH1.TXD"}});
-	_knownFiles.insert({"splash2.txd", {false, datpath+"/txd/SPLASH2.TXD"}});
-	_knownFiles.insert({"splash3.txd", {false, datpath+"/txd/SPLASH3.TXD"}});
-
-	loadDFF("wheels.DFF");
+	loadDFF("wheels.dff");
 	loadDFF("weapons.dff");
 	loadDFF("arrow.dff");
 	loadTXD("particle.txd");
@@ -179,9 +166,7 @@ void GameData::parseDAT(const std::string& path)
 							texpath[t] = '/';
 						}
 					}
-					texpath = findPathRealCase(datpath, texpath);
 					std::string texname = texpath.substr(texpath.find_last_of("/")+1);
-					_knownFiles.insert({ texname, { false, texpath }});
 					loadTXD(texname);
 				}
 			}
@@ -219,32 +204,7 @@ void GameData::loadCOL(const size_t zone, const std::string& name)
 
 void GameData::loadIMG(const std::string& name)
 {
-	LoaderIMG imgLoader; 
-	std::string archivePath = datpath + name;
-
-	if (imgLoader.load(archivePath)) {
-		for (size_t i = 0; i < imgLoader.getAssetCount(); i++) {
-			auto &asset = imgLoader.getAssetInfoByIndex(i);
-
-			std::string filename = asset.name;
-			
-			if(asset.size == 0)
-			{
-				engine->logger.warning("Data", "Ignoring asset " + filename + " with size 0");
-			}
-			else
-            {
-				// Enter the asset twice.. 
-				_knownFiles.insert({ filename, { true, archivePath }});
-				for(size_t t = 0; t < filename.size(); ++t)
-				{
-					filename[t] = tolower(filename[t]);
-				}
-				_knownFiles.insert({ filename, { true, archivePath }});
-			}
-		}
-		archives.insert({archivePath, imgLoader});
-	}
+	index.indexArchive(datpath + name);
 }
 
 void GameData::loadIPL(const std::string& name)
@@ -364,7 +324,7 @@ SCMFile *GameData::loadSCM(const std::string &path)
 
 void GameData::loadGXT(const std::string &name)
 {
-	auto d = openFile2(name);
+	auto d = openFile(name);
 
 	LoaderGXT loader;
 
@@ -470,7 +430,7 @@ void GameData::loadDFF(const std::string& name, bool async)
 
 void GameData::loadIFP(const std::string &name)
 {
-	auto f = openFile2(name);
+	auto f = openFile(name);
 
 	if(f)
 	{
@@ -563,109 +523,14 @@ void GameData::loadSplash(const std::string &name)
 	engine->state.currentSplash = lower;
 }
 
-char* GameData::openFile(const std::string& name)
+FileHandle GameData::openFile(const std::string &name)
 {
-	auto i = _knownFiles.find(name);
-	if(i != _knownFiles.end())
+	auto file = index.openFile(name);
+	if( file == nullptr )
 	{
-		if(i->second.archived)
-		{
-			// Find the archive
-			auto ai = archives.find(i->second.path);
-			if(ai != archives.end())
-			{
-				return ai->second.loadToMemory(name);
-			}
-			else 
-			{
-				engine->logger.error("Data", "Archive not found " + i->second.path);
-			}
-		}
-		else
-		{
-			std::ifstream dfile(i->second.path);
-			if ( ! dfile.is_open()) {
-				engine->logger.error("Data", "Error opening file " + i->second.path);
-				return nullptr;
-			}
-
-			dfile.seekg(0, std::ios_base::end);
-			size_t length = dfile.tellg();
-			dfile.seekg(0);
-			char *data = new char[length];
-			dfile.read(data, length);
-			
-			return data;
-		}
+		engine->logger.error("Data", "Unable to open file: " + name);
 	}
-	else 
-	{
-		engine->logger.error("Data", "Unable to locate file: " + name);
-	}
-	
-	return nullptr;
-}
-
-FileHandle GameData::openFile2(const std::string &name)
-{
-	auto i = _knownFiles.find(name);
-	if(i != _knownFiles.end())
-	{
-		char* data = nullptr;
-		size_t length = 0;
-
-		if(i->second.archived)
-		{
-			// Find the archive
-			auto ai = archives.find(i->second.path);
-			if(ai != archives.end())
-			{
-				LoaderIMGFile asset;
-				if( ai->second.findAssetInfo(name, asset) )
-				{
-					data = ai->second.loadToMemory(name);
-					length = asset.size * 2048;
-				}
-			}
-			else
-			{
-				engine->logger.error("Data", "Archive not found " + i->second.path);
-			}
-		}
-		else
-		{
-			std::ifstream dfile(i->second.path);
-			if ( ! dfile.is_open()) {
-				engine->logger.error("Data", "Error opening file " + i->second.path);
-				return nullptr;
-			}
-
-			dfile.seekg(0, std::ios_base::end);
-			length = dfile.tellg();
-			dfile.seekg(0);
-			data = new char[length];
-			dfile.read(data, length);
-		}
-
-		return FileHandle( new FileContentsInfo{ data, length } );
-	}
-	else
-	{
-		engine->logger.error("Data", "Unable to locate file: " + name);
-	}
-	return nullptr;
-}
-
-char* GameData::loadFile(const std::string &name)
-{
-	auto it = loadedFiles.find(name);
-	 if( it != loadedFiles.end() ) {
-		engine->logger.warning("Data", "File " + name + " already loaded");
-	}
-
-	loadedFiles[name] = true;
-
-	return openFile(name);
+	return file;
 }
 
 TextureAtlas* GameData::getAtlas(size_t i)
diff --git a/rwengine/src/engine/GameWorld.cpp b/rwengine/src/engine/GameWorld.cpp
index b6227a54..e5c115ad 100644
--- a/rwengine/src/engine/GameWorld.cpp
+++ b/rwengine/src/engine/GameWorld.cpp
@@ -215,6 +215,9 @@ InstanceObject *GameWorld::createInstance(const uint16_t id, const glm::vec3& po
 		std::string modelname = oi->modelName;
 		std::string texturename = oi->textureName;
 
+		std::transform(std::begin(modelname), std::end(modelname), std::begin(modelname), tolower);
+		std::transform(std::begin(texturename), std::end(texturename), std::begin(texturename), tolower);
+
 		// Ensure the relevant data is loaded.
 		if(! oi->modelName.empty()) {
 			if( modelname != "null" ) {
@@ -660,7 +663,7 @@ void GameWorld::loadCutscene(const std::string &name)
 	std::string lowerName(name);
 	std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
 
-	auto datfile = gameData.openFile2(lowerName + ".dat");
+	auto datfile = gameData.openFile(lowerName + ".dat");
 
 	CutsceneData* cutscene = new CutsceneData;
 
diff --git a/rwengine/src/loaders/LoaderDFF.cpp b/rwengine/src/loaders/LoaderDFF.cpp
index 241a5cbb..e10f4a92 100644
--- a/rwengine/src/loaders/LoaderDFF.cpp
+++ b/rwengine/src/loaders/LoaderDFF.cpp
@@ -456,26 +456,26 @@ Model* LoaderDFF::loadFromMemory(FileHandle file, GameData *gameData)
 }
 
 LoadModelJob::LoadModelJob(WorkContext *context, GameData* gd, const std::string &file, ModelCallback cb)
-	: WorkJob(context), _gameData(gd), _file(file), _callback(cb), _data(nullptr)
+	: WorkJob(context), _gameData(gd), _file(file), _callback(cb)
 {
 
 }
 
 void LoadModelJob::work()
 {
-	_data = _gameData->openFile2(_file);
+	data = _gameData->openFile(_file);
 }
 
 void LoadModelJob::complete()
 {
 	Model* m = nullptr;
 	// TODO error status
-	if( _data ) {
+	if( data ) {
 
 		// TODO allow some of the loading to process in a seperate thread.
 		LoaderDFF loader;
 
-		m = loader.loadFromMemory(_data, _gameData);
+		m = loader.loadFromMemory(data, _gameData);
 	}
 
 	_callback(m);
diff --git a/rwengine/src/loaders/TextureLoader.cpp b/rwengine/src/loaders/TextureLoader.cpp
index 01cda201..48401e9d 100644
--- a/rwengine/src/loaders/TextureLoader.cpp
+++ b/rwengine/src/loaders/TextureLoader.cpp
@@ -7,23 +7,6 @@
 #include <iostream>
 #include <algorithm>
 
-bool TextureLoader::loadFromFile(std::string filename, GameData* gameData)
-{
-	std::ifstream dfile(filename);
-	if ( ! dfile.is_open()) {
-		std::cerr << "Error opening file " << filename << std::endl;
-		return false;
-	}
-
-	dfile.seekg(0, std::ios_base::end);
-	size_t length = dfile.tellg();
-	dfile.seekg(0);
-	char *data = new char[length];
-	dfile.read(data, length);
-
-	return loadFromMemory(data, gameData);
-}
-
 GLuint gErrorTextureData[] = { 0xFFFF00FF, 0xFF000000, 0xFF000000, 0xFFFF00FF };
 GLuint gDebugTextureData[] = {0xFF0000FF, 0xFF00FF00};
 GLuint gTextureRed[] = {0xFF0000FF};
@@ -201,8 +184,9 @@ TextureData::Handle createTexture(RW::BSTextureNative& texNative, RW::BinaryStre
 	return TextureData::create( textureName, { texNative.width, texNative.height }, transparent );
 }
 
-bool TextureLoader::loadFromMemory(char *data, GameData *gameData)
+bool TextureLoader::loadFromMemory(FileHandle file, GameData *gameData)
 {
+	auto data = file->data;
 	RW::BinaryStreamSection root(data);
 	/*auto texDict =*/ root.readStructure<RW::BSTextureDictionary>();
 
@@ -218,7 +202,7 @@ bool TextureLoader::loadFromMemory(char *data, GameData *gameData)
 		std::string alpha = std::string(texNative.alphaName);
 		std::transform(name.begin(), name.end(), name.begin(), ::tolower );
 		std::transform(alpha.begin(), alpha.end(), alpha.begin(), ::tolower );
-
+		
 		auto texture = createTexture(texNative, rootSection);
 
 		gameData->textures[{name, alpha}] = texture;
@@ -233,23 +217,21 @@ bool TextureLoader::loadFromMemory(char *data, GameData *gameData)
 
 
 LoadTextureArchiveJob::LoadTextureArchiveJob(WorkContext *context, GameData *gd, const std::string &file)
-	: WorkJob(context), _gameData(gd), _file(file), _data(nullptr)
+	: WorkJob(context), _gameData(gd), _file(file)
 {
 
 }
 
 void LoadTextureArchiveJob::work()
 {
-	_data = _gameData->openFile(_file);
+	data = _gameData->openFile(_file);
 }
 
 void LoadTextureArchiveJob::complete()
 {
 	// TODO error status
-	if(_data) {
+	if(data) {
 		TextureLoader loader;
-		loader.loadFromMemory(_data, _gameData);
+		loader.loadFromMemory(data, _gameData);
 	}
-
-	delete[] _data;
 }
diff --git a/rwengine/src/render/GameRenderer.cpp b/rwengine/src/render/GameRenderer.cpp
index 7a1d5b22..0d28ca15 100644
--- a/rwengine/src/render/GameRenderer.cpp
+++ b/rwengine/src/render/GameRenderer.cpp
@@ -647,7 +647,7 @@ void GameRenderer::renderInstance(InstanceObject *instance)
 	}
 
 	glm::mat4 matrixModel;
-	if( instance->body ) {
+	if( instance->body && instance->body->body ) {
 		instance->body->body->getWorldTransform().getOpenGLMatrix(glm::value_ptr(matrixModel));
 	}
 	else {
@@ -917,6 +917,10 @@ void GameRenderer::renderGeometry(Model* model, size_t g, const glm::mat4& model
 					auto& tC = mat.textures[0].name;
 					auto& tA = mat.textures[0].alphaName;
 					tex = engine->gameData.findTexture(tC, tA);
+					if( ! tex )
+					{
+						//engine->logger.warning("Renderer", "Missing texture: " + tC + " " + tA);
+					}
 					mat.textures[0].texture = tex;
 				}
 				if( tex )
diff --git a/rwengine/src/render/GameShaders.cpp b/rwengine/src/render/GameShaders.cpp
index 17ae395c..c46d32ce 100644
--- a/rwengine/src/render/GameShaders.cpp
+++ b/rwengine/src/render/GameShaders.cpp
@@ -257,6 +257,6 @@ void main()
 {
 	vec4 c = texture2D(texture, TexCoords);
 	// Set colour to 0, 0, 0, 1 for textured mode.
-	outColour = colour + c.rgba;
+	outColour = vec4(colour.rgb + c.rgb, colour.a);
 })";
 }
diff --git a/rwgame/RWGame.cpp b/rwgame/RWGame.cpp
index 4de22fde..b94e306f 100644
--- a/rwgame/RWGame.cpp
+++ b/rwgame/RWGame.cpp
@@ -63,7 +63,7 @@ RWGame::RWGame(const std::string& gamepath, int argc, char* argv[])
 	
 	// Initalize all the archives.
 	engine->gameData.loadIMG("/models/gta3");
-	engine->gameData.loadIMG("/models/txd");
+	//engine->gameData.loadIMG("/models/txd");
 	engine->gameData.loadIMG("/anim/cuts");
 	
 	// Initialize renderer