Data Loader interface

rwengine/include/loaders/DataLoader.hpp

@@ -0,0 +1,41 @@
+#pragma once
+#ifndef _DATALOADER_HPP_
+#define _DATALOADER_HPP_
+
+#include <iostream>
+
+class LoadContext;
+
+class DataLoader
+{
+	LoadContext* _context;
+
+public:
+
+	DataLoader(LoadContext* context)
+		: _context(context) {}
+
+	virtual ~DataLoader() {}
+
+	/**
+	 * @brief getContext
+	 * @return The loading context for this Loader
+	 */
+	LoadContext* getContext() const { return _context; }
+
+	/**
+	 * @brief load the data contained in a set of bytes
+	 * @param data The bytes from which to load data.
+	 * @param size The number of bytes.
+	 * @return true if the data was valid and loaded, false otherwise.
+	 */
+	virtual bool load( const char* data, size_t size ) = 0;
+
+	/**
+	 * @brief create perform any after-load activitiy that is required.
+	 */
+	virtual void create() = 0;
+};
+
+
+#endif

rwengine/include/loaders/LoadContext.hpp

@@ -0,0 +1,86 @@
+#pragma once
+#ifndef _LOADCONTEXT_HPP_
+#define _LOADCONTEXT_HPP_
+
+#include <queue>
+#include <thread>
+#include <functional>
+#include <fstream>
+
+class DataLoader;
+class LoadContext;
+
+struct LoadRequest
+{
+	DataLoader* loader;
+	const char* data;
+	size_t size;
+	std::string filename;
+};
+
+class LoadWorker
+{
+	LoadContext* _context;
+
+public:
+
+	bool _started;
+	bool _running;
+	std::thread _thread;
+	void start();
+
+	LoadWorker( LoadContext* context )
+		: _context( context ), _started(false), _running(true),
+	_thread( std::bind(&LoadWorker::start, this) ) { }
+
+	~LoadWorker( )
+	{
+		_started = true;
+		_running = false;
+		_thread.join();
+	}
+};
+
+class LoadContext
+{
+	std::queue<LoadRequest> _loadQueue;
+	std::queue<LoadRequest> _createQueue;
+	unsigned int _loaded;
+	unsigned int _loading;
+
+	LoadWorker _worker;
+
+public:
+
+	LoadContext()
+		: _loaded(0), _loading(0), _worker(this) { }
+
+	void add(DataLoader* loader, const char* data, size_t size)
+	{
+		_loadQueue.push({loader, data, size, ""});
+	}
+
+	void add(DataLoader* loader, const std::string& filename)
+	{
+		_loadQueue.push({loader, nullptr, 0, filename});
+	}
+
+	void loadNext();
+
+	const std::queue<LoadRequest> getLoadQueue() const { return _loadQueue; }
+
+	unsigned int getLoaded() const { return _createQueue.size(); }
+
+	unsigned int getIncomplete() const { return _loadQueue.size() + _loading + _createQueue.size(); }
+
+	unsigned int getComplete() const { return _loaded; }
+
+	unsigned int getTotal() const { return getIncomplete() + getComplete(); }
+
+	void flushCreation();
+
+	void wait();
+
+};
+
+#endif

rwengine/src/loaders/DataLoader.cpp

@@ -0,0 +1,2 @@
+#include <loaders/DataLoader.hpp>
+

rwengine/src/loaders/LoadContext.cpp

@@ -0,0 +1,69 @@
+#include <loaders/LoadContext.hpp>
+#include <loaders/DataLoader.hpp>
+
+void LoadWorker::start()
+{
+	while( !_started ) {
+		std::this_thread::yield();
+	}
+	while( _running ) {
+		_context->loadNext();
+		std::this_thread::yield();
+	}
+}
+
+
+void LoadContext::loadNext()
+{
+	// MUTEX
+	if( _loadQueue.size() == 0 ) {
+		// UMUTEX
+		return;
+	}
+	_loading++;
+	auto tl = _loadQueue.front();
+	_loadQueue.pop();
+	// UMUTEX
+
+	if( tl.data == nullptr ) {
+		// TODO: instigate opening of the data bytes.
+	}
+
+	bool result = tl.loader->load(tl.data, tl.size);
+
+	delete tl.data;
+
+	// MUTEX
+
+	if( result ) {
+		_createQueue.push(tl);
+	}
+	else {
+		// Create is rip
+		_loaded++;
+	}
+	_loading--;
+
+	// UMUTEX
+}
+
+void LoadContext::flushCreation()
+{
+	// MUTEX
+
+	while( ! _createQueue.empty() ) {
+		auto tl = _createQueue.front();
+		_createQueue.pop();
+
+		tl.loader->create();
+		_loaded++;
+	}
+
+	// UMUTEX
+}
+
+void LoadContext::wait()
+{
+	while( _loadQueue.size() > 0 || _loading > 0 ) std::this_thread::yield();
+}
+

tests/test_archive.cpp

@@ -6,13 +6,13 @@ BOOST_AUTO_TEST_SUITE(ArchiveTests)
 
 BOOST_AUTO_TEST_CASE(test_open_archive)
 {
-	LoaderIMG loader;
+	LoaderIMG archive;
 	
-	BOOST_REQUIRE( loader.load(Global::getGamePath() + "/models/gta3") );
+	BOOST_REQUIRE( archive.load(Global::getGamePath() + "/models/gta3") );
 	
-	BOOST_CHECK( loader.getAssetCount() > 0 );
+	BOOST_CHECK( archive.getAssetCount() > 0 );
 	
-	auto& f = loader.getAssetInfoByIndex(0);
+	auto& f = archive.getAssetInfoByIndex(0);
 	
 	// A few assumptions..
 	
@@ -20,7 +20,7 @@ BOOST_AUTO_TEST_CASE(test_open_archive)
 	BOOST_CHECK_EQUAL( f.offset, 0);
 	BOOST_CHECK_EQUAL( f.size, 33);
 	
-	auto& f2 = loader.getAssetInfo("radar00.txd");
+	auto& f2 = archive.getAssetInfo("radar00.txd");
 	
 	BOOST_CHECK_EQUAL( f2.name, f.name );
 	BOOST_CHECK_EQUAL( f2.offset, f.offset );

tests/test_loader.cpp

@@ -0,0 +1,147 @@
+#include <boost/test/unit_test.hpp>
+#include <loaders/LoadContext.hpp>
+#include <loaders/DataLoader.hpp>
+
+
+
+class TestLoader : public DataLoader
+{
+public:
+	bool _loaded, _loadResult, _created;
+
+	TestLoader(LoadContext* context, bool result )
+		: DataLoader(context), _loaded(false), _loadResult(result),
+		_created(false) { }
+
+	bool load( const char* bytes, size_t size )
+	{ std::this_thread::sleep_for(std::chrono::milliseconds(5)); _loaded = true; return _loadResult; }
+
+	void create() { _created = true; }
+};
+
+
+BOOST_AUTO_TEST_SUITE(LoaderTests)
+
+
+BOOST_AUTO_TEST_CASE(test_interface)
+{
+	{
+		LoadContext context;
+
+		TestLoader loader( &context, true );
+
+		BOOST_CHECK_EQUAL( loader.getContext(), &context );
+
+		// Test that the TestLoader works as expected.
+		const char* databytes = new const char[1]{ 0x00 };
+
+		BOOST_CHECK( ! loader._loaded );
+		BOOST_CHECK( ! loader._created );
+
+		BOOST_CHECK( loader.load(databytes, 1) );
+
+		BOOST_CHECK( loader._loaded );
+		BOOST_CHECK( ! loader._created );
+
+		loader.create();
+
+		BOOST_CHECK( loader._created );
+	}
+}
+
+BOOST_AUTO_TEST_CASE(test_context)
+{
+	{
+		LoadContext context;
+
+		TestLoader test(&context, true);
+		const char* databytes = new const char[1]{ 0x00 };
+
+		context.add(&test, databytes, 1);
+
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 1 );
+		BOOST_CHECK_EQUAL( context.getTotal(), 1 );
+		BOOST_CHECK_EQUAL( context.getComplete(), 0 );
+
+		auto& queue = context.getLoadQueue();
+		BOOST_REQUIRE( queue.size() == 1 );
+		BOOST_CHECK_EQUAL( queue.front().loader, &test );
+		BOOST_CHECK_EQUAL( queue.front().data, databytes );
+		BOOST_CHECK_EQUAL( queue.front().size, 1 );
+
+		// Worker thread tier.
+		context.loadNext();
+
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 1 );
+		BOOST_CHECK_EQUAL( context.getTotal(), 1 );
+		BOOST_CHECK_EQUAL( context.getComplete(), 0 );
+
+		// Back on main thread
+		BOOST_CHECK_EQUAL( context.getLoaded(), 1 );
+
+		context.flushCreation();
+
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 0 );
+		BOOST_CHECK_EQUAL( context.getTotal(), 1 );
+		BOOST_CHECK_EQUAL( context.getComplete(), 1 );
+
+	}
+}
+
+BOOST_AUTO_TEST_CASE(test_thread)
+{
+	{
+		LoadContext context;
+
+		LoadWorker worker(&context);
+
+		TestLoader test(&context, true);
+		const char* databytes = new const char[1]{ 0x00 };
+
+		context.add(&test, databytes, 1);
+
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 1 );
+		BOOST_CHECK_EQUAL( context.getLoaded(), 0 );
+
+		worker._started = true;
+
+		std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+		BOOST_CHECK_EQUAL( context.getLoaded(), 1 );
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 1 );
+
+		context.flushCreation();
+
+		BOOST_CHECK_EQUAL( context.getLoaded(), 0 );
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 0 );
+		BOOST_CHECK_EQUAL( context.getComplete(), 1 );
+	}
+	{
+		LoadContext context;
+
+		LoadWorker worker(&context);
+
+		TestLoader test(&context, true);
+		const char* databytes = new const char[1]{ 0x00 };
+
+		context.add(&test, databytes, 1);
+
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 1 );
+		BOOST_CHECK_EQUAL( context.getLoaded(), 0 );
+
+		worker._started = true;
+
+		context.wait();
+
+		BOOST_CHECK_EQUAL( context.getLoaded(), 1 );
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 1 );
+
+		context.flushCreation();
+
+		BOOST_CHECK_EQUAL( context.getLoaded(), 0 );
+		BOOST_CHECK_EQUAL( context.getIncomplete(), 0 );
+		BOOST_CHECK_EQUAL( context.getComplete(), 1 );
+	}
+}
+BOOST_AUTO_TEST_SUITE_END()
+

tests/test_loaderdff.cpp

@@ -7,7 +7,7 @@ BOOST_AUTO_TEST_SUITE(LoaderDFFTests)
 BOOST_AUTO_TEST_CASE(test_open_dff)
 {
 	LoaderIMG loader;
-	
+
 	BOOST_REQUIRE( loader.load(Global::getGamePath() + "/models/gta3") );
 
 	auto d = loader.loadToMemory("landstal.dff");