Merge pull request #1988 from momoko-h/IEDT

Improve loading strings from EDT files

assets/mods/test-json-dialogs/data/binarydata/aimbios.edt.json

@@ -0,0 +1,4 @@
+{
+ "0": { "1": "This is the additional info for Barry who usually does not have any. The above text still comes from the original aimbios.edt file. Please note that this text is longer than the old limit of 160 characters." },
+ "5": { "0": "Main description for Trevor. Unicode seems to work partially (it depends on the font, of course): ô£ÇБÿÄщµ¢ć. Error messages about invalid characters in the console are expected." }
+}

assets/mods/test-json-dialogs/data/binarydata/help.edt.json

@@ -0,0 +1,3 @@
+{
+	"0": { "0": "This is the footer of the help screen." }
+}

assets/mods/test-json-dialogs/data/binarydata/mercbios.edt.json

@@ -0,0 +1,6 @@
+{
+  "0": {
+    "0": "This is Biff's description.",
+    "1": "This is Biff's additional information."
+  }
+}

assets/mods/test-json-dialogs/manifest.json

@@ -1,5 +1,5 @@
 {
-    "name": "Test JSON dialogs",
-    "description": "This mod replaces all of Igor's dialog quotes with 'Test quote XXX'",
-    "version": "1.0.0"
+    "name": "Test JSON edt replacements",
+    "description": "This is a development helper mod that replaces all of Igor's dialog quotes with 'Test quote XXX' as well as several other srings stored in .edt files. End users should not enable this mod.",
+    "version": "2.0.2"
 }

src/externalized/ContentManager.h

@@ -6,11 +6,11 @@
 #include "ItemSystem.h"
 #include "MercSystem.h"
 #include "DirFs.h"
+#include "IEDT.h"
 
 #include <string_theory/string>
-
 #include <map>
-#include <stdint.h>
+#include <string_view>
 #include <vector>
 
 
@@ -98,7 +98,7 @@ class ContentManager : public ItemSystem, public MercSystem
 	virtual ST::string loadEncryptedString(const ST::string& fileName, uint32_t seek_chars, uint32_t read_chars) const = 0;
 
 	/** Load dialogue quote from file. */
-	virtual ST::string* loadDialogQuoteFromFile(const ST::string& filename, int quote_number) = 0;
+	virtual ST::string loadDialogQuoteFromFile(const ST::string& filename, unsigned quote_number) = 0;
 
 	/** Get weapons with the give index. */
 	virtual const WeaponModel* getWeapon(uint16_t index) = 0;
@@ -198,4 +198,6 @@ class ContentManager : public ItemSystem, public MercSystem
 
 	/* Gets the enabled mods and their version strings as a map */
 	virtual const std::vector<std::pair<ST::string, ST::string>> getEnabledMods() const = 0;
+
+	virtual IEDT::uptr openEDT(std::string_view filename, IEDT::column_list columns) const = 0;
 };

src/externalized/DefaultContentManager.cc

@@ -343,18 +343,15 @@ ST::string DefaultContentManager::loadEncryptedString(const ST::string& fileName
 }
 
 /** Load dialogue quote from file. */
-ST::string* DefaultContentManager::loadDialogQuoteFromFile(const ST::string& fileName, int quote_number)
+ST::string DefaultContentManager::loadDialogQuoteFromFile(const ST::string& fileName, unsigned quote_number)
 {
-	AutoSGPFile File(openGameResForReading(fileName));
-	ST::string err_msg;
-	ST::string quote = LoadEncryptedData(err_msg, getStringEncType(), File, quote_number * DIALOGUESIZE, DIALOGUESIZE);
-	if (!err_msg.empty())
-	{
-		SLOGW("DefaultContentManager::loadDialogQuoteFromFile '{}' {}: {}", fileName, quote_number, err_msg);
-	}
-	return new ST::string{std::move(quote)};
+	// Using the qualified name because we do not want a virtual function call here.
+	return DefaultContentManager::openEDT(fileName.view(), { DIALOGUESIZE })->at(quote_number);
 }
 
+#if 0
+/* This function is only used when someone wants to extract all quotes to JSON,
+   see the commented out code in SGP.cc */
 /** Load all dialogue quotes for a character. */
 void DefaultContentManager::loadAllDialogQuotes(STRING_ENC_TYPE encType, const ST::string& fileName, std::vector<ST::string*> &quotes) const
 {
@@ -373,6 +370,7 @@ void DefaultContentManager::loadAllDialogQuotes(STRING_ENC_TYPE encType, const S
 		quotes.push_back(new ST::string{std::move(quote)});
 	}
 }
+#endif
 
 /** Get weapons with the give index. */
 const WeaponModel* DefaultContentManager::getWeapon(uint16_t itemIndex)

src/externalized/DefaultContentManager.h

@@ -8,6 +8,7 @@
 
 #include "ContentManager.h"
 #include "ContentMusic.h"
+#include "IEDT.h"
 #include "IGameDataLoader.h"
 #include "StringEncodingTypes.h"
 #include "RustInterface.h"
@@ -17,7 +18,7 @@
 
 #include <map>
 #include <memory>
-#include <stdexcept>
+#include <string_view>
 #include <vector>
 
 class DefaultContentManager : public ContentManager, public IGameDataLoader
@@ -69,7 +70,7 @@ class DefaultContentManager : public ContentManager, public IGameDataLoader
 	virtual ST::string loadEncryptedString(const ST::string& fileName, uint32_t seek_chars, uint32_t read_chars) const override;
 
 	/** Load dialogue quote from file. */
-	virtual ST::string* loadDialogQuoteFromFile(const ST::string& filename, int quote_number) override;
+	ST::string loadDialogQuoteFromFile(const ST::string& filename, unsigned quote_number) override;
 
 	/** Load all dialogue quotes for a character. */
 	void loadAllDialogQuotes(STRING_ENC_TYPE encType, const ST::string& filename, std::vector<ST::string*> &quotes) const;
@@ -163,6 +164,11 @@ class DefaultContentManager : public ContentManager, public IGameDataLoader
 
 	/* Gets the enabled mods and their version strings */
 	virtual const std::vector<std::pair<ST::string, ST::string>> getEnabledMods() const override;
+
+	/* Opens an EDT file. The columns parameter describes the layout of the data
+	   inside the EDT file. */
+	IEDT::uptr openEDT(std::string_view filename, IEDT::column_list columns) const override;
+
 protected:
 	RustPointer<EngineOptions> m_engineOptions;
 	RustPointer<ModManager> m_modManager;

src/externalized/ModPackContentManager.cc

@@ -18,10 +18,6 @@ ModPackContentManager::ModPackContentManager(RustPointer<EngineOptions> engineOp
 	m_modNames = modNames;
 }
 
-ModPackContentManager::~ModPackContentManager()
-{
-}
-
 void ModPackContentManager::logConfiguration() const {
 	ST::string joinedModList;
     for(const auto &s : m_modNames) {
@@ -34,14 +30,14 @@ void ModPackContentManager::logConfiguration() const {
 }
 
 /** Load dialogue quote from file. */
-ST::string* ModPackContentManager::loadDialogQuoteFromFile(const ST::string& filename, int quote_number)
+ST::string ModPackContentManager::loadDialogQuoteFromFile(const ST::string& filename, unsigned quote_number)
 {
 	ST::string jsonFileName = ST::string(filename) + ".json";
 	std::map<ST::string, std::vector<ST::string> >::iterator it = m_dialogQuotesMap.find(jsonFileName);
 	if(it != m_dialogQuotesMap.end())
 	{
 		SLOGD("cached quote {} {}", quote_number, jsonFileName);
-		return new ST::string(it->second[quote_number]);
+		return it->second[quote_number];
 	}
 	else
 	{
@@ -52,7 +48,7 @@ ST::string* ModPackContentManager::loadDialogQuoteFromFile(const ST::string& fil
 			std::vector<ST::string> quotes;
 			JsonUtility::parseJsonToListStrings(jsonQuotes.c_str(), quotes);
 			m_dialogQuotesMap[jsonFileName] = quotes;
-			return new ST::string(quotes[quote_number]);
+			return quotes[quote_number];
 		}
 		else
 		{

src/externalized/ModPackContentManager.h

@@ -12,10 +12,12 @@ class ModPackContentManager : public DefaultContentManager
 public:
 	ModPackContentManager(RustPointer<EngineOptions> engineOptions);
 
-	virtual ~ModPackContentManager() override;
+	~ModPackContentManager() override = default;
 
 	/** Load dialogue quote from file. */
-	virtual ST::string* loadDialogQuoteFromFile(const ST::string& filename, int quote_number) override;
+	ST::string loadDialogQuoteFromFile(const ST::string& filename, unsigned quote_number) override;
+
+	IEDT::uptr openEDT(std::string_view filename, IEDT::column_list columns) const override;
 
 protected:
 	void logConfiguration() const override;

src/externalized/strings/CMakeLists.txt

@@ -2,6 +2,7 @@ file(GLOB LOCAL_JA2_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
 set(JA2_SOURCES
     ${JA2_SOURCES}
     ${LOCAL_JA2_HEADERS}
+    ${CMAKE_CURRENT_SOURCE_DIR}/EDT.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/EncryptedString.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/ItemStrings.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/Localization.cc

src/externalized/strings/EDT.cc

@@ -0,0 +1,113 @@
+#include "ContentManager.h"
+#include "DefaultContentManager.h"
+#include "EncryptedString.h"
+#include "IEDT.h"
+#include "Json.h"
+#include "ModPackContentManager.h"
+#include "SGPFile.h"
+#include <charconv>
+#include <memory>
+#include <numeric>
+#include <optional>
+#include <string_view>
+#include <vector>
+#include "string_theory/string"
+
+namespace {
+
+class ClassicEDT final : public IEDT
+{
+	std::unique_ptr<SGPFile> underlyingFile;
+	std::vector<IEDT::column_list::value_type> columns;
+	unsigned rowLength;
+
+public:
+	ClassicEDT(ContentManager const& cm,
+	           std::string_view const filename,
+	           IEDT::column_list columnsList) :
+		underlyingFile{ cm.openGameResForReading(filename) },
+		columns{ columnsList },
+		rowLength{ std::accumulate(columns.begin(), columns.end(), 0U) }
+	{
+	}
+
+	ST::string at(unsigned const row, unsigned const column) const override
+	{
+		return LoadEncryptedString(underlyingFile.get(),
+			row * rowLength + std::accumulate
+				(columns.begin(), columns.begin() + column, 0U),
+			columns.at(column));
+	}
+
+	~ClassicEDT() override = default;
+};
+
+
+class JsonEDT final : public IEDT
+{
+	template<std::size_t TN>
+	static char * conv(char (&buf)[TN], unsigned const value)
+	{
+		auto const convResult{ std::to_chars(buf, buf + TN, value) };
+		// to_chars does not NUL terminate its result, fix that here.
+		*convResult.ptr = 0;
+		return buf;
+	}
+
+	std::optional<ST::string> attemptToGet(unsigned const row, unsigned const column) const
+	{
+		char buf[64];
+		if (jsonObject.has(conv(buf, row)))
+		{
+			auto const rowObject{ jsonObject.GetValue(buf).toObject() };
+			if (rowObject.has(conv(buf, column)))
+			{
+				return rowObject.GetString(buf);
+			}
+		}
+
+		return std::nullopt;
+	}
+
+	ClassicEDT backupEDT;
+	JsonObject jsonObject;
+
+public:
+	JsonEDT(ContentManager const& cm,
+	        std::string_view const filename,
+	        IEDT::column_list columnsList)
+		: backupEDT{ cm, filename, columnsList }
+	{
+		auto const jsonFilename{ ST::string{filename} + ".json" };
+		if (cm.doesGameResExists(jsonFilename))
+		{
+			std::unique_ptr<SGPFile> jsonfile{ cm.openGameResForReading(jsonFilename) };
+			jsonObject = JsonValue::deserialize(jsonfile->readStringToEnd()).toObject();
+		}
+	}
+
+	ST::string at(unsigned const row, unsigned const column) const override
+	{
+		auto const result{ attemptToGet(row, column) };
+		// Not using result.value_or here because that would make us
+		// always call backupEDT.at().
+		return result ? *result : backupEDT.at(row, column);
+	}
+
+	~JsonEDT() override = default;
+};
+} // anonymous namespace
+
+
+IEDT::uptr DefaultContentManager::openEDT(std::string_view const filename,
+	IEDT::column_list columns) const
+{
+	return IEDT::uptr{ new ClassicEDT(*this, filename, columns) };
+}
+
+
+IEDT::uptr ModPackContentManager::openEDT(std::string_view const filename,
+	IEDT::column_list columns) const
+{
+	return IEDT::uptr{ new JsonEDT(*this, filename, columns) };
+}

src/externalized/strings/EDT.h

@@ -0,0 +1,97 @@
+#pragma once
+
+#include "ContentManager.h"
+#include "Directories.h"
+#include "GameInstance.h"
+#include "IEDT.h"
+#include <string_view>
+
+/*
+    EDTFile: support class for reading strings from EDT files.
+
+    EDT files are essentially just one big blob of slightly obfuscated
+    UTF-16 (or UCS2?) characters. They contain absolutely no metadata to
+    indicate where one string ends and the next one begins. It is entirely
+    up to the code reading the file to break up the blob into single strings.
+
+    This interpretation is done in two ways in JA2:
+
+    - The entire file is seen as an array of fixed width strings.
+    - The file is seen as a table where each row has the same column layout.
+
+    The first case could also be interpreted as a table where each row has
+    just one column. To keep things simple, the IEDT interface treats all EDT
+    files as tables.
+
+    The default content manager uses the IEDT implementation provided by the
+    class ClassicEDT. This implementation can only read files in the original
+    .edt file format. It does not do much; it simply opens the file and
+	calculates the file offset from the row and column number.
+
+    The mod content manager uses the IEDT implementation provided by the class
+    JsonEDT, where each table entry can be provided by a JSON file. The file
+	name of this file is the original file name with .json appended. Strings
+	that are not found in the JSON file are taken from an original .edt file
+	as a fallback option.
+
+	As already mentioned, EDT contains no metadata, therefore ClassicEDT
+	requires some details about each individual file. These details are
+	contained in the EDTFilesTable below which also shows which files can
+	currently be overriden.
+	
+	To support a new file add an entry to the table then change the source
+	file to use EDTFile instead of ContentManager::loadEncryptedString.
+*/
+
+class EDTFile
+{
+	struct EDTFilesTable
+	{
+		std::string_view     filename;
+		IEDT::column_list    columns;
+	};
+
+	static constexpr EDTFilesTable EDTFilesTable[]
+	{
+		/* Description strings of the A.I.M. members screen.
+		   One row per merc (40 in total) with two columns each:
+		   Column 0: Long description (original limit 400 characters)
+		   Column 1: Additional information (160 characters)
+		*/
+		{ BINARYDATADIR "/aimbios.edt", { 400, 160 } },
+
+		/* Description strings of the M.E.R.C. files.
+		   One row per merc (10 in total) with two columns each:
+		   Column 0: Long description (original limit 400 characters)
+		   Column 1: Additional information (160 characters)
+		*/
+		{ BINARYDATADIR "/mercbios.edt", { 400, 160 } },
+
+		/* Strings of the help screen.
+		   123 rows, each with one column of 640 characters.
+		   See HelpScreenText.h to get a rough overview of the row contents.
+		*/
+		{ BINARYDATADIR "/help.edt", { 640 } },
+	};
+
+	IEDT::uptr mIEDT;
+
+public:
+	enum EDTFilesList
+	{
+		AIMBIOS,
+		MERCBIOS,
+		HELP
+	};
+
+	EDTFile(EDTFilesList const whichFile)
+	{
+		auto const& fileMetaData{ EDTFilesTable[whichFile] };
+		mIEDT = GCM->openEDT(fileMetaData.filename, fileMetaData.columns);
+	}
+
+	auto at(unsigned const row, unsigned const column = 0) const
+	{
+		return mIEDT->at(row, column);
+	}
+};