Replace mb strings with 16-bit char strings

The game indexes into the font map directly, it doesn't use any font
 encoding scheme like UTF-16. This corrects the behaviour and removes
 dependancy on iconv.

rwengine/CMakeLists.txt

@@ -33,6 +33,7 @@ set(RWENGINE_SOURCES
 	src/data/Chase.hpp
 	src/data/CollisionModel.hpp
 	src/data/CutsceneData.hpp
+	src/data/GameTexts.cpp
 	src/data/GameTexts.hpp
 	src/data/InstanceData.hpp
 	src/data/ObjectData.cpp

rwengine/src/data/GameTexts.cpp

@@ -0,0 +1,10 @@
+#include "GameTexts.hpp"
+
+GameString GameStringUtil::fromString(const std::string& str)
+{
+	GameString s;
+	for (std::string::size_type i = 0u; i < str.size();  ++i) {
+		s += str[i];
+	}
+	return s;
+}

rwengine/src/data/GameTexts.hpp

@@ -3,21 +3,47 @@
 #include <string>
 #include <unordered_map>
 
+/**
+ * Each GXT char is just a 16-bit index into the font map.
+ */
+using GameStringChar = uint16_t;
+/**
+ * The game stores strings as 16-bit indexes into the font
+ * texture, which is something simllar to ASCII.
+ */
+using GameString = std::basic_string<GameStringChar>;
+/**
+ * GXT keys are just 8 single byte chars.
+ * Keys are small so should be subject to SSO
+ */
+using GameStringKey = std::string;
+
+namespace GameStringUtil
+{
+/**
+ * @brief fromString Converts a string to a GameString
+ *
+ * Encoding of GameStrings depends on the font, only simple ASCII chars will map well
+ */
+GameString fromString(const std::string& str);
+}
+
 class GameTexts
 {
-	std::unordered_map<std::string, std::string> _textDB;
+	using StringTable = std::unordered_map<GameStringKey, GameString>;
+	StringTable m_strings;
 public:
 
-	void addText(const std::string& id, const std::string& text) {
+	void addText(const GameStringKey& id, GameString&& text) {
-		_textDB.insert({ id, text });
+		m_strings.emplace(id, text);
 	}
 
-	std::string text(const std::string& id) {
+	GameString text(const GameStringKey& id) {
-		auto a = _textDB.find(id);
+		auto a = m_strings.find(id);
-		if( a != _textDB.end() ) {
+		if( a != m_strings.end() ) {
 			return a->second;
 		}
-		return id;
+		return GameStringUtil::fromString("MISSING: " + id);
 	}
 
 };

rwengine/src/engine/GameState.hpp

@@ -162,7 +162,7 @@ struct GameStats
 struct TextDisplayData
 {
 	// This is set by the final display text command.
-	std::string text;
+	GameString text;
 	glm::vec2 position;
 
 	glm::vec4 colourFG;

rwengine/src/engine/ScreenText.cpp

@@ -26,7 +26,7 @@ void ScreenText::tick(float dt)
 	}
 }
 
-ScreenTextEntry ScreenTextEntry::makeBig(const std::string& id, const std::string& str, int style, int durationMS)
+ScreenTextEntry ScreenTextEntry::makeBig(const GameStringKey& id, const GameString& str, int style, int durationMS)
 {
 	switch(style) {
 
@@ -142,7 +142,7 @@ ScreenTextEntry ScreenTextEntry::makeBig(const std::string& id, const std::strin
 	}
 
 	return {
-		"Error, style " + std::to_string(style),
+		GameStringUtil::fromString("Error, style " + std::to_string(style)),
 		{320.f, 400.f},
 		2,
 		50,
@@ -156,7 +156,7 @@ ScreenTextEntry ScreenTextEntry::makeBig(const std::string& id, const std::strin
 	};
 }
 
-ScreenTextEntry ScreenTextEntry::makeHighPriority(const std::string& id, const std::string& str, int durationMS)
+ScreenTextEntry ScreenTextEntry::makeHighPriority(const GameStringKey& id, const GameString& str, int durationMS)
 {
 	// Color: ?
 	// Font: Arial
@@ -179,7 +179,7 @@ ScreenTextEntry ScreenTextEntry::makeHighPriority(const std::string& id, const s
 	};
 }
 
-ScreenTextEntry ScreenTextEntry::makeHelp(const std::string& id, const std::string& str)
+ScreenTextEntry ScreenTextEntry::makeHelp(const GameStringKey& id, const GameString& str)
 {
 	return {
 		str,

rwengine/src/engine/ScreenText.hpp

@@ -1,7 +1,7 @@
 #ifndef RWENGINE_SCREENTEXT_HPP
 #define RWENGINE_SCREENTEXT_HPP
 #include <rw/types.hpp>
-#include <string>
+#include <data/GameTexts.hpp>
 #include <vector>
 #include <array>
 #include <algorithm>
@@ -33,7 +33,7 @@ constexpr unsigned int ScreenTypeTextCount = static_cast<unsigned int>(ScreenTex
 struct ScreenTextEntry
 {
 	/// After processing numbers
-	std::string text;
+	GameString text;
 	/// in the virtual 640x480 screen space
 	glm::vec2 position;
 	/// Font number
@@ -53,19 +53,19 @@ struct ScreenTextEntry
 	/// Wrap width
 	int wrapX;
 	/// ID used to reference the text.
-	std::string id;
+	GameStringKey id;
 
-	static ScreenTextEntry makeBig(const std::string& id,
+	static ScreenTextEntry makeBig(const GameStringKey& id,
-							const std::string& str,
+							const GameString& str,
 							int style,
 							int durationMS);
 
-	static ScreenTextEntry makeHighPriority(const std::string& id,
+	static ScreenTextEntry makeHighPriority(const GameStringKey& id,
-							const std::string& str,
+							const GameString& str,
 							int durationMS);
 
-	static ScreenTextEntry makeHelp(const std::string& id,
+	static ScreenTextEntry makeHelp(const GameStringKey& id,
-									const std::string& str);
+									const GameString& str);
 };
 
 /**
@@ -141,12 +141,13 @@ public:
 	}
 
 	template<class...Args>
-	static std::string format(std::string format, Args&&...args)
+	static GameString format(GameString format, Args&&...args)
 	{
-		const std::array<std::string, sizeof...(args)> vals = { args... };
+		static auto kReplacementMarker = GameStringUtil::fromString("~1~");
+		const std::array<GameString, sizeof...(args)> vals = { args... };
 		size_t x = 0, val = 0;
 		// We're only looking for numerical replacement markers
-		while ((x = format.find("~1~")) != std::string::npos && val < vals.size())
+		while ((x = format.find(kReplacementMarker)) != GameString::npos && val < vals.size())
 		{
 			format = format.substr(0, x) + vals[val++] + format.substr(x + 3);
 		}

rwengine/src/loaders/LoaderGXT.cpp

@@ -1,5 +1,4 @@
 #include <loaders/LoaderGXT.hpp>
-#include <iconv.h>
 
 void LoaderGXT::load(GameTexts &texts, FileHandle &file)
 {
@@ -13,40 +12,12 @@ void LoaderGXT::load(GameTexts &texts, FileHandle &file)
 
 	auto tdata = data+blocksize+8;
 
-	// This is not supported in GCC 4.8.1
-	//std::wstring_convert<std::codecvt<char16_t,char,std::mbstate_t>,char16_t> convert;
-
-	auto icv = iconv_open("UTF-8", "UTF-16");
-
 	for( size_t t = 0; t < blocksize/12; ++t ) {
 		size_t offset = *(std::uint32_t*)(data+(t * 12 + 0));
 		std::string id(data+(t * 12 + 4));
 
-		// Find the terminating bytes
+		GameStringChar* stringSrc = reinterpret_cast<GameStringChar*>(tdata+offset);
-		size_t bytes = 0;
+		GameString string(stringSrc);
-		for(;; bytes++ ) {
+		texts.addText(id, std::move(string));
-			if(tdata[offset+bytes-1] == 0 && tdata[offset+bytes] == 0) break;
-		}
-		size_t len = bytes/2;
-
-		size_t outSize = 1024;
-		char u8buff[1024];
-		char *uwot = u8buff;
-
-		char* strbase = tdata+offset;
-
-#if defined(RW_NETBSD)
-		iconv(icv, (const char**)&strbase, &bytes, &uwot, &outSize);
-#else
-		iconv(icv, &strbase, &bytes, &uwot, &outSize);
-#endif
-
-		u8buff[len] = '\0';
-
-		std::string message(u8buff);
-
-		texts.addText(id, message);
 	}
-
-	iconv_close(icv);
 }

rwengine/src/render/TextRenderer.cpp

@@ -5,8 +5,7 @@
 #include <algorithm>
 #include <cctype>
 
-/// @todo This is very rough
+int charToIndex(uint16_t g)
-int charToIndex(char g)
 {
 	// Correct for the default font maps
 	/// @todo confirm for JA / RU font maps
@@ -171,7 +170,7 @@ void TextRenderer::renderText(const TextRenderer::TextInfo& ti, bool forceColour
 	
 	for (size_t i = 0; i < text.length(); ++i)
 	{
-		char c = text[i];
+		char16_t c = text[i];
 
 		// Handle any markup changes.
 		if( c == '~' && text.length() > i + 1 )

rwengine/src/render/TextRenderer.hpp

@@ -32,7 +32,7 @@ public:
 		/// Font index @see TextRenderer::setFontTexture
 		int font;
 		/// Message to be displayed (including markup)
-		std::string text;
+		GameString text;
 		/// On screen position
 		glm::vec2 screenPosition;
 		/// font size

rwengine/src/script/modules/GameModule.cpp

@@ -29,21 +29,21 @@
 
 // Helper function to format script numbers to strings
 // for use in the text printing opcodes.
-std::string formatValue(const SCMOpcodeParameter& p)
+GameString formatValue(const SCMOpcodeParameter& p)
 {
 	switch (p.type) {
 	case TFloat16:
-		return std::to_string(p.real);
+		return GameStringUtil::fromString(std::to_string(p.real));
 	default:
-		return std::to_string(p.integerValue());
+		return GameStringUtil::fromString(std::to_string(p.integerValue()));
 	}
-	return "";
+	return {0};
 }
 
 void game_print_big(const ScriptArguments& args)
 {
 	std::string id(args[0].string);
-	std::string str = args.getWorld()->data->texts.text(id);
+	auto str = args.getWorld()->data->texts.text(id);
 	unsigned short time = args[1].integer;
 	unsigned short style = args[2].integer;
 	args.getWorld()->state->text.addText<ScreenTextType::Big>(
@@ -55,7 +55,7 @@ void game_print_big(const ScriptArguments& args)
 void game_print_now(const ScriptArguments& args)
 {
 	std::string id(args[0].string);
-	std::string str = args.getWorld()->data->texts.text(id);
+	auto str = args.getWorld()->data->texts.text(id);
 	int time = args[1].integer;
 	int flags = args[2].integer;
 	RW_UNUSED(flags);
@@ -390,7 +390,7 @@ void game_get_runtime(const ScriptArguments& args)
 void game_print_big_with_number(const ScriptArguments& args)
 {
 	std::string id(args[0].string);
-	std::string str =
+	auto str =
 			ScreenText::format(
 				args.getWorld()->data->texts.text(id),
 				formatValue(args[1]));
@@ -873,8 +873,8 @@ bool game_has_respray_happened(const ScriptArguments& args)
 void game_display_text(const ScriptArguments& args)
 {
 	glm::vec2 pos(args[0].real, args[1].real);
-	std::string str(args[2].string);
+	std::string id(args[2].string);
-	str = args.getWorld()->data->texts.text(str);
+	auto str = args.getWorld()->data->texts.text(id);
 	args.getWorld()->state->nextText.text = str;
 	args.getWorld()->state->nextText.position = pos;
 	args.getWorld()->state->texts.push_back(args.getWorld()->state->nextText);
@@ -946,7 +946,7 @@ void game_print_big_with_2_numbers(const ScriptArguments& args)
 	int time = args[3].integerValue();
 	unsigned short style = args[4].integerValue();
 
-	std::string str = ScreenText::format(world->data->texts.text(id),
+	auto str = ScreenText::format(world->data->texts.text(id),
 	                                     formatValue(args[1]),
 	                                     formatValue(args[2]));
 
@@ -1113,7 +1113,7 @@ void game_get_found_hidden_packages(const ScriptArguments& args)
 void game_display_help(const ScriptArguments& args)
 {
 	std::string id(args[0].string);
-	std::string str = args.getWorld()->data->texts.text(id);
+	auto str = args.getWorld()->data->texts.text(id);
 
 	args.getWorld()->state->text.addText<ScreenTextType::Big>(
 				ScreenTextEntry::makeHelp(

rwgame/DrawUI.cpp

@@ -76,7 +76,7 @@ void drawPlayerInfo(PlayerController* player, GameWorld* world, GameRenderer* re
 		   << std::setw(0) << ":"
 		   << std::setw(2) << world->getMinute();
 
-		ti.text = ss.str();
+		ti.text = GameStringUtil::fromString(ss.str());
 	}
 	ti.baseColour = ui_shadowColour;
 	ti.screenPosition = glm::vec2(infoTextX + 1.f, infoTextY+1.f);
@@ -88,7 +88,7 @@ void drawPlayerInfo(PlayerController* player, GameWorld* world, GameRenderer* re
 
 	infoTextY += ui_textHeight;
 
-	ti.text = "$" + std::to_string(world->state->playerInfo.money);
+	ti.text = GameStringUtil::fromString("[") + GameStringUtil::fromString(std::to_string(world->state->playerInfo.money));
 	ti.baseColour = ui_shadowColour;
 	ti.screenPosition = glm::vec2(infoTextX + 1.f, infoTextY+1.f);
 	render->text.renderText(ti);
@@ -103,7 +103,7 @@ void drawPlayerInfo(PlayerController* player, GameWorld* world, GameRenderer* re
 		std::stringstream ss;
 		ss << "{" << std::setw(3) << std::setfill('0')
 		   << (int)player->getCharacter()->getCurrentState().health;
-		ti.text = ss.str();
+		ti.text = GameStringUtil::fromString(ss.str());
 	}
 	ti.baseColour = ui_shadowColour;
 	ti.screenPosition = glm::vec2(infoTextX + 1.f, infoTextY+1.f);
@@ -118,7 +118,7 @@ void drawPlayerInfo(PlayerController* player, GameWorld* world, GameRenderer* re
 		std::stringstream ss;
 		ss << "[" << std::setw(3) << std::setfill('0')
 		   << (int)player->getCharacter()->getCurrentState().armour;
-		ti.text = ss.str();
+		ti.text = GameStringUtil::fromString(ss.str());
 		ti.baseColour = ui_shadowColour;
 		ti.screenPosition = glm::vec2(infoTextX + 1.f - ui_armourOffset, infoTextY+1.f);
 		render->text.renderText(ti);
@@ -128,9 +128,9 @@ void drawPlayerInfo(PlayerController* player, GameWorld* world, GameRenderer* re
 		render->text.renderText(ti);
 	}
 
-	std::string s;
+	GameString s;
 	for (size_t i = 0; i < ui_maxWantedLevel; ++i) {
-		s += "]";
+		s += GameStringUtil::fromString("]");
 	}
 	ti.text = s;
 	ti.baseColour = ui_shadowColour;
@@ -183,8 +183,9 @@ void drawPlayerInfo(PlayerController* player, GameWorld* world, GameRenderer* re
 		if (wep->getWeaponData()->fireType != WeaponData::MELEE) {
 			const CharacterState& cs = player->getCharacter()->getCurrentState();
 			const CharacterWeaponSlot& slotInfo = cs.weapons[cs.currentWeapon];
-			ti.text = std::to_string(slotInfo.bulletsClip) + "-"
+			ti.text = GameStringUtil::fromString(
-					+ std::to_string(slotInfo.bulletsTotal);
+			            std::to_string(slotInfo.bulletsClip) + "-"
+			          + std::to_string(slotInfo.bulletsTotal));
 
 			ti.baseColour = ui_shadowColour;
 			ti.font = 2;

rwgame/MenuSystem.hpp

@@ -32,7 +32,7 @@ public:
 			TextRenderer::TextInfo ti;
 			ti.font = font;
 			ti.screenPosition = basis;
-			ti.text = name;
+			ti.text = GameStringUtil::fromString(name);
 			ti.size = getHeight();
 			if( ! active )
 			{

rwgame/RWGame.cpp

@@ -627,7 +627,7 @@ void RWGame::renderDebugStats(float time, Renderer::ProfileInfo& worldRenderTime
 	}
 	
 	TextRenderer::TextInfo ti;
-	ti.text = ss.str();
+	ti.text = GameStringUtil::fromString(ss.str());
 	ti.font = 2;
 	ti.screenPosition = glm::vec2( 10.f, 10.f );
 	ti.size = 15.f;

rwgame/states/DebugState.cpp

@@ -368,7 +368,7 @@ void DebugState::draw(GameRenderer* r)
 	ss << "Camera Position: " << glm::to_string(_debugCam.position);
 
 	TextRenderer::TextInfo ti;
-	ti.text = ss.str();
+	ti.text = GameStringUtil::fromString(ss.str());
 	ti.font = 2;
 	ti.screenPosition = glm::vec2( 10.f, 10.f );
 	ti.size = 15.f;

rwgame/states/LoadingState.cpp

@@ -50,9 +50,10 @@ void LoadingState::handleEvent(const SDL_Event& e)
 
 void LoadingState::draw(GameRenderer* r)
 {
+	static auto kLoadingString = GameStringUtil::fromString("Loading...");
 	// Display some manner of loading screen.
 	TextRenderer::TextInfo ti;
-	ti.text = "Loading...";
+	ti.text = kLoadingString;
 	auto size = r->getRenderer()->getViewport();
 	ti.size = 25.f;
 	ti.screenPosition = glm::vec2( 50.f, size.y - ti.size - 50.f );

tests/test_globals.hpp

@@ -42,6 +42,18 @@ struct print_log_value<std::nullptr_t> {
 };
 }} BOOST_NS_MAGIC_CLOSING
 
+namespace boost { namespace test_tools { BOOST_NS_MAGIC
+template<>
+struct print_log_value<GameString> {
+	void operator()( std::ostream& s , GameString const& v )
+	{
+		for (GameString::size_type i = 0u; i<v.size(); ++i) {
+			s << (char)v[i];
+		}
+	}
+};
+}} BOOST_NS_MAGIC_CLOSING
+
 #undef BOOST_NS_MAGIC
 #undef BOOST_NS_MAGIC_CLOSING
 

tests/test_text.cpp

@@ -4,6 +4,8 @@
 #include <loaders/LoaderGXT.hpp>
 #include <engine/ScreenText.hpp>
 
+#define T(x) GameStringUtil::fromString(x)
+
 BOOST_AUTO_TEST_SUITE(TextTests)
 
 #if RW_TEST_WITH_DATA
@@ -18,7 +20,7 @@ BOOST_AUTO_TEST_CASE(load_test)
 
 		loader.load( texts, d );
 
-		BOOST_CHECK_EQUAL( texts.text("1008"), "BUSTED" );
+		BOOST_CHECK_EQUAL( texts.text("1008"), T("BUSTED") );
 	}
 }
 
@@ -28,12 +30,12 @@ BOOST_AUTO_TEST_CASE(big_test)
 	{
 		auto big = ScreenTextEntry::makeBig(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					1,
 					5000);
 
 		BOOST_CHECK_EQUAL("TEST_1", big.id);
-		BOOST_CHECK_EQUAL("Test String", big.text);
+		BOOST_CHECK_EQUAL(T("Test String"), big.text);
 		BOOST_CHECK_EQUAL(5000, big.durationMS);
 		BOOST_CHECK_EQUAL(0, big.displayedMS);
 		BOOST_CHECK_EQUAL(1, big.alignment);
@@ -42,11 +44,11 @@ BOOST_AUTO_TEST_CASE(big_test)
 	{
 		auto big = ScreenTextEntry::makeBig(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					2,
 					5000);
 
-		BOOST_CHECK_EQUAL("Test String", big.text);
+		BOOST_CHECK_EQUAL(T("Test String"), big.text);
 		BOOST_CHECK_EQUAL(5000, big.durationMS);
 		BOOST_CHECK_EQUAL(0, big.displayedMS);
 		BOOST_CHECK_EQUAL(2, big.alignment);
@@ -58,9 +60,9 @@ BOOST_AUTO_TEST_CASE(help_test)
 {
 	auto help = ScreenTextEntry::makeHelp(
 				"TEST_1",
-				"Test Help");
+				T("Test Help"));
 
-	BOOST_CHECK_EQUAL("Test Help", help.text);
+	BOOST_CHECK_EQUAL(T("Test Help"), help.text);
 	BOOST_CHECK_EQUAL(5000, help.durationMS);
 	BOOST_CHECK_EQUAL(0, help.displayedMS);
 	BOOST_CHECK_EQUAL(18, help.size);
@@ -75,17 +77,17 @@ BOOST_AUTO_TEST_CASE(queue_test)
 	st.addText<ScreenTextType::Big>(
 				ScreenTextEntry::makeBig(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					2,
 					5000));
 	st.addText<ScreenTextType::HighPriority>(
 				ScreenTextEntry::makeHighPriority(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					5000));
 
 	BOOST_REQUIRE(st.getText<ScreenTextType::Big>().size() == 1);
-	BOOST_CHECK_EQUAL("Test String", st.getText<ScreenTextType::Big>()[0].text);
+	BOOST_CHECK_EQUAL(T("Test String"), st.getText<ScreenTextType::Big>()[0].text);
 	BOOST_CHECK_EQUAL(5000, st.getText<ScreenTextType::Big>()[0].durationMS);
 	BOOST_CHECK_EQUAL(0, st.getText<ScreenTextType::Big>()[0].displayedMS);
 
@@ -104,7 +106,7 @@ BOOST_AUTO_TEST_CASE(clear_test)
 	st.addText<ScreenTextType::Big>(
 				ScreenTextEntry::makeBig(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					2,
 					5000));
 
@@ -118,21 +120,21 @@ BOOST_AUTO_TEST_CASE(clear_test)
 BOOST_AUTO_TEST_CASE(format_test)
 {
 	// Test formating of string codes into strings.
-	const auto codeStr1 = "Hello ~1~ world";
+	const auto codeStr1 = T("Hello ~1~ world");
-	const auto codeStr2 = "~1~Hello ~1~ world~1~";
+	const auto codeStr2 = T("~1~Hello ~1~ world~1~");
-	const auto codeStr3 = "Hello world~1~";
+	const auto codeStr3 = T("Hello world~1~");
 
-	auto f1 = ScreenText::format(codeStr1, "r");
+	auto f1 = ScreenText::format(codeStr1, T("r"));
-	BOOST_CHECK_EQUAL(f1, "Hello r world");
+	BOOST_CHECK_EQUAL(f1, T("Hello r world"));
 
-	auto f2 = ScreenText::format(codeStr2, "k", "h");
+	auto f2 = ScreenText::format(codeStr2, T("k"), T("h"));
-	BOOST_CHECK_EQUAL(f2, "kHello h world~1~");
+	BOOST_CHECK_EQUAL(f2, T("kHello h world~1~"));
 
-	auto f3 = ScreenText::format(codeStr3, "x");
+	auto f3 = ScreenText::format(codeStr3, T("x"));
-	BOOST_CHECK_EQUAL(f3, "Hello worldx");
+	BOOST_CHECK_EQUAL(f3, T("Hello worldx"));
 
-	auto f4  = ScreenText::format(codeStr3, "x", "k");
+	auto f4  = ScreenText::format(codeStr3, T("x"), T("k"));
-	BOOST_CHECK_EQUAL(f4, "Hello worldx");
+	BOOST_CHECK_EQUAL(f4, T("Hello worldx"));
 }
 
 BOOST_AUTO_TEST_CASE(format_remove)
@@ -143,21 +145,21 @@ BOOST_AUTO_TEST_CASE(format_remove)
 	st.addText<ScreenTextType::Big>(
 				ScreenTextEntry::makeBig(
 					"TEST_2",
-					"Test String",
+					T("Test String"),
 					2,
 					5000));
 
 	st.addText<ScreenTextType::Big>(
 				ScreenTextEntry::makeBig(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					2,
 					5000));
 
 	st.addText<ScreenTextType::Big>(
 				ScreenTextEntry::makeBig(
 					"TEST_1",
-					"Test String",
+					T("Test String"),
 					2,
 					5000));