Basic storage and loading/saving for extradata

src/ass_dialogue.cpp

@@ -44,6 +44,7 @@
 #include <boost/algorithm/string/replace.hpp>
 #include <boost/algorithm/string/trim.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/regex.hpp>
 #include <boost/spirit/include/karma_generate.hpp>
 #include <boost/spirit/include/karma_int.hpp>
 
@@ -120,7 +121,28 @@ void AssDialogue::Parse(std::string const& raw) {
 	for (int& margin : Margin)
 		margin = mid(0, boost::lexical_cast<int>(tkn.next_str()), 9999);
 	Effect = tkn.next_str_trim();
-	Text = std::string(tkn.next_tok().begin(), str.end());
+
+	std::string text{tkn.next_tok().begin(), str.end()};
+
+	static const boost::regex extradata_test("^\\{(=\\d+)+\\}");
+	boost::match_results<std::string::iterator> rematch;
+	if (boost::regex_search(text.begin(), text.end(), rematch, extradata_test)) {
+		std::string extradata_str = rematch.str(0);
+		text = rematch.suffix().str();
+
+		static const boost::regex idmatcher("=(\\d+)");
+		auto start = extradata_str.begin();
+		auto end = extradata_str.end();
+		std::vector<uint32_t> ids;
+		while (boost::regex_search(start, end, rematch, idmatcher)) {
+			auto id = boost::lexical_cast<uint32_t>(rematch.str(1));
+			ids.push_back(id);
+			start = rematch.suffix().second;
+		}
+		ExtradataIds = ids;
+	}
+
+	Text = text;
 }
 
 void append_int(std::string &str, int v) {
@@ -156,6 +178,16 @@ std::string AssDialogue::GetData(bool ssa) const {
 	for (auto margin : Margin)
 		append_int(str, margin);
 	append_unsafe_str(str, Effect);
+
+	if (ExtradataIds.get().size() > 0) {
+		str += "{";
+		for (auto id : ExtradataIds.get()) {
+			str += "=";
+			boost::spirit::karma::generate(back_inserter(str), boost::spirit::karma::int_, id);
+		}
+		str += "}";
+	}
+
 	str += Text.get();
 
 	if (str.find('\n') != str.npos || str.find('\r') != str.npos) {

src/ass_dialogue.h

@@ -146,6 +146,8 @@ struct AssDialogueBase {
 	boost::flyweight<std::string> Actor;
 	/// Effect name
 	boost::flyweight<std::string> Effect;
+	/// IDs of extradata entries for line
+	boost::flyweight<std::vector<uint32_t>> ExtradataIds;
 	/// Raw text data
 	boost::flyweight<std::string> Text;
 };
@@ -183,3 +185,4 @@ public:
 	AssDialogue(std::string const& data);
 	~AssDialogue();
 };
+

src/ass_entry.cpp

@@ -28,6 +28,7 @@ std::string const& AssEntry::GroupHeader(bool ssa) const {
 		"[Fonts]",
 		"[Graphics]",
 		"[Events]",
+		"[Aegisub Extradata]",
 		""
 	};
 
@@ -37,6 +38,7 @@ std::string const& AssEntry::GroupHeader(bool ssa) const {
 		"[Fonts]",
 		"[Graphics]",
 		"[Events]",
+		"[Aegisub Extradata]",
 		""
 	};
 

src/ass_entry.h

@@ -43,6 +43,7 @@ enum class AssEntryGroup {
 	FONT,
 	GRAPHIC,
 	DIALOGUE,
+	EXTRADATA,
 	GROUP_MAX
 };
 

src/ass_file.cpp

@@ -26,6 +26,7 @@
 #include <algorithm>
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/filesystem/path.hpp>
+#include <cassert>
 
 AssFile::AssFile() { }
 
@@ -54,6 +55,8 @@ void AssFile::LoadDefault(bool include_dialogue_line) {
 AssFile::AssFile(const AssFile &from)
 : Info(from.Info)
 , Attachments(from.Attachments)
+, Extradata(from.Extradata)
+, next_extradata_id(from.next_extradata_id)
 {
 	Styles.clone_from(from.Styles,
 		[](AssStyle const& e) { return new AssStyle(e); },
@@ -68,6 +71,8 @@ void AssFile::swap(AssFile& from) throw() {
 	Styles.swap(from.Styles);
 	Events.swap(from.Events);
 	Attachments.swap(from.Attachments);
+	Extradata.swap(from.Extradata);
+	std::swap(next_extradata_id, from.next_extradata_id);
 }
 
 AssFile& AssFile::operator=(AssFile from) {
@@ -229,3 +234,58 @@ void AssFile::Sort(EntryList<AssDialogue> &lst, CompFunc comp, std::set<AssDialo
 		begin = --end;
 	}
 }
+
+
+uint32_t AssFile::AddExtradata(std::string const& key, std::string const& value) {
+	// next_extradata_id must not exist
+	assert(Extradata.find(next_extradata_id) == Extradata.end());
+	Extradata[next_extradata_id] = std::make_pair(key, value);
+	return next_extradata_id++; // return old value, then post-increment
+}
+
+std::map<std::string, std::string> AssFile::GetExtradata(std::vector<uint32_t> const& id_list) const {
+	// If multiple IDs have the same key name, the last ID wins
+	std::map<std::string, std::string> result;
+	for (auto id : id_list) {
+		auto it = Extradata.find(id);
+		if (it != Extradata.end())
+			result[it->second.first] = it->second.second;
+	}
+	return result;
+}
+
+void AssFile::CleanExtradata() {
+	// Collect all IDs existing in the database
+	// Then remove all IDs found to be in use from this list
+	// Remaining is then all garbage IDs
+	std::vector<uint32_t> ids;
+	for (auto& it : Extradata) {
+		ids.push_back(it.first);
+	}
+
+	// For each line, find which IDs it actually uses and remove them from the unused-list
+	for (auto& line : Events) {
+		// Find the ID for each unique key in the line
+		std::map<std::string, uint32_t> key_ids;
+		for (auto id : line.ExtradataIds.get()) {
+			auto ed_it = Extradata.find(id);
+			if (ed_it == Extradata.end())
+				continue;
+			key_ids[ed_it->second.first] = id;
+		}
+		// Update the line's ID list to only contain the actual ID for any duplicate keys
+		// Also mark found IDs as used in the cleaning list
+		std::vector<uint32_t> new_ids;
+		for (auto& keyid : key_ids) {
+			new_ids.push_back(keyid.second);
+			ids.erase(std::remove(ids.begin(), ids.end(), keyid.second));
+		}
+		line.ExtradataIds = new_ids;
+	}
+
+	// The ids list should contain only unused IDs now
+	for (auto id : ids) {
+		Extradata.erase(id);
+	}
+}
+

src/ass_file.h

@@ -50,6 +50,8 @@ class wxString;
 template<typename T>
 using EntryList = typename boost::intrusive::make_list<T, boost::intrusive::constant_time_size<false>, boost::intrusive::base_hook<AssEntryListHook>>::type;
 
+using AegisubExtradataMap = std::map<uint32_t, std::pair<std::string, std::string>>;
+
 struct AssFileCommit {
 	wxString const& message;
 	int *commit_id;
@@ -66,6 +68,9 @@ public:
 	EntryList<AssStyle> Styles;
 	EntryList<AssDialogue> Events;
 	std::vector<AssAttachment> Attachments;
+	AegisubExtradataMap Extradata;
+
+	uint32_t next_extradata_id = 0;
 
 	AssFile();
 	AssFile(const AssFile &from);
@@ -102,6 +107,16 @@ public:
 	int GetUIStateAsInt(std::string const& key) const;
 	void SaveUIState(std::string const& key, std::string const& value);
 
+	/// @brief Add a new extradata entry
+	/// @param key Class identifier/owner for the extradata
+	/// @param value Data for the extradata
+	/// @return ID of the created entry
+	uint32_t AddExtradata(std::string const& key, std::string const& value);
+	/// Fetch all extradata entries from a list of IDs
+	std::map<std::string, std::string> GetExtradata(std::vector<uint32_t> const& id_list) const;
+	/// Remove unreferenced extradata entries
+	void CleanExtradata();
+
 	/// Type of changes made in a commit
 	enum CommitType {
 		/// Potentially the entire file has been changed; any saved information
@@ -129,7 +144,9 @@ public:
 		COMMIT_DIAG_TIME   = 0x40,
 		/// The text of existing dialogue lines have changed
 		COMMIT_DIAG_TEXT   = 0x80,
-		COMMIT_DIAG_FULL   = COMMIT_DIAG_META | COMMIT_DIAG_TIME | COMMIT_DIAG_TEXT
+		COMMIT_DIAG_FULL   = COMMIT_DIAG_META | COMMIT_DIAG_TIME | COMMIT_DIAG_TEXT,
+		/// Extradata entries were added/modified/removed
+		COMMIT_EXTRADATA   = 0x100,
 	};
 
 	DEFINE_SIGNAL_ADDERS(AnnounceCommit, AddCommitListener)
@@ -168,3 +185,4 @@ public:
 	/// @param limit If non-empty, only lines in this set are sorted
 	static void Sort(EntryList<AssDialogue>& lst, CompFunc comp = CompStart, std::set<AssDialogue*> const& limit = std::set<AssDialogue*>());
 };
+

src/ass_parser.cpp

@@ -19,12 +19,15 @@
 #include "ass_file.h"
 #include "ass_info.h"
 #include "ass_style.h"
+#include "string_codec.h"
 #include "subtitle_format.h"
 
 #include <algorithm>
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/trim.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/regex.hpp>
 
 AssParser::AssParser(AssFile *target, int version)
 : target(target)
@@ -111,6 +114,21 @@ void AssParser::ParseGraphicsLine(std::string const& data) {
 		attach.reset(new AssAttachment(data, AssEntryGroup::GRAPHIC));
 }
 
+void AssParser::ParseExtradataLine(std::string const &data) {
+	static const boost::regex matcher("Data:[[:space:]]*(\\d+),([^,]+),(.*)");
+	boost::match_results<std::string::const_iterator> mr;
+
+	if (boost::regex_match(data, mr, matcher)) {
+		auto id = boost::lexical_cast<uint32_t>(mr.str(1));
+		auto key = inline_string_decode(mr.str(2));
+		auto value = inline_string_decode(mr.str(3));
+
+		// ensure next_extradata_id is always at least 1 more than the largest existing id
+		target->next_extradata_id = std::max(id+1, target->next_extradata_id);
+		target->Extradata[id] = std::make_pair(key, value);
+	}
+}
+
 void AssParser::AddLine(std::string const& data) {
 	// Special-case for attachments since a line could theoretically be both a
 	// valid attachment data line and a valid section header, and if an
@@ -142,6 +160,8 @@ void AssParser::AddLine(std::string const& data) {
 			state = &AssParser::ParseGraphicsLine;
 		else if (low == "[fonts]")
 			state = &AssParser::ParseFontLine;
+		else if (low == "[aegisub extradata]")
+			state = &AssParser::ParseExtradataLine;
 		else
 			state = &AssParser::UnknownLine;
 		return;

src/ass_parser.h

@@ -31,6 +31,7 @@ class AssParser {
 	void ParseScriptInfoLine(std::string const& data);
 	void ParseFontLine(std::string const& data);
 	void ParseGraphicsLine(std::string const& data);
+	void ParseExtradataLine(std::string const &data);
 	void UnknownLine(std::string const&) { }
 public:
 	AssParser(AssFile *target, int version);

src/subtitle_format_ass.cpp

@@ -22,6 +22,7 @@
 #include "ass_file.h"
 #include "ass_style.h"
 #include "ass_parser.h"
+#include "string_codec.h"
 #include "text_file_reader.h"
 #include "text_file_writer.h"
 #include "version.h"
@@ -114,6 +115,24 @@ struct Writer {
 			file.WriteLineToFile(ssa ? line.GetSSAText() : line.GetEntryData());
 		}
 	}
+
+	void WriteExtradata(AegisubExtradataMap const& extradata) {
+		if (extradata.size() == 0)
+			return;
+
+		group = AssEntryGroup::EXTRADATA;
+		file.WriteLineToFile("");
+		file.WriteLineToFile("[Aegisub Extradata]");
+		for (auto const& edi : extradata) {
+			std::string line = "Data: ";
+			line += std::to_string(edi.first);
+			line += ",";
+			line += inline_string_encode(edi.second.first);
+			line += ",";
+			line += inline_string_encode(edi.second.second);
+			file.WriteLineToFile(line);
+		}
+	}
 };
 }
 
@@ -124,4 +143,5 @@ void AssSubtitleFormat::WriteFile(const AssFile *src, agi::fs::path const& filen
 	writer.Write(src->Styles);
 	writer.Write(src->Attachments);
 	writer.Write(src->Events);
+	writer.WriteExtradata(src->Extradata);
 }