Merge branch 'extradata'

Conflicts:
	src/ass_parser.cpp
This commit is contained in:
Niels Martin Hansen 2014-04-25 17:22:15 +02:00
commit d53c36e67f
72 changed files with 4913 additions and 15 deletions

View file

@ -28,6 +28,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libresrc", "build\libresrc\
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lua51", "build\lua51\lua51.vcxproj", "{5391A8B1-9C70-4DC4-92AD-D3E34C6B803F}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "luabins", "build\luabins\luabins.vcxproj", "{A7A30702-8162-4E1A-A010-EF51B590C121}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "respack", "build\respack\respack.vcxproj", "{08AF2BCC-FCDD-4F0E-8B41-59A6E634F2E8}"
ProjectSection(ProjectDependencies) = postProject
{FB8E8D19-A4D6-4181-943C-282075F49B41} = {FB8E8D19-A4D6-4181-943C-282075F49B41}
@ -223,6 +225,20 @@ Global
{5391A8B1-9C70-4DC4-92AD-D3E34C6B803F}.Release|x64.Build.0 = Release|x64
{5391A8B1-9C70-4DC4-92AD-D3E34C6B803F}.Release-MinDep|Win32.ActiveCfg = Release|Win32
{5391A8B1-9C70-4DC4-92AD-D3E34C6B803F}.Release-MinDep|x64.ActiveCfg = Release|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug|Win32.ActiveCfg = Debug|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug|Win32.Build.0 = Debug|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug|x64.ActiveCfg = Debug|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug|x64.Build.0 = Debug|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug-MinDep|Win32.ActiveCfg = Debug|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug-MinDep|x64.ActiveCfg = Debug|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug-Tests|Win32.ActiveCfg = Debug|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Debug-Tests|x64.ActiveCfg = Debug|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Release|Win32.ActiveCfg = Release|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Release|Win32.Build.0 = Release|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Release|x64.ActiveCfg = Release|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Release|x64.Build.0 = Release|x64
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Release-MinDep|Win32.ActiveCfg = Release|Win32
{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}.Release-MinDep|x64.ActiveCfg = Release|x64
{08AF2BCC-FCDD-4F0E-8B41-59A6E634F2E8}.Debug|Win32.ActiveCfg = Debug|Win32
{08AF2BCC-FCDD-4F0E-8B41-59A6E634F2E8}.Debug|Win32.Build.0 = Debug|Win32
{08AF2BCC-FCDD-4F0E-8B41-59A6E634F2E8}.Debug|x64.ActiveCfg = Debug|x64

View file

@ -3,6 +3,7 @@ include Makefile.inc
SUBDIRS += \
vendor/lua \
vendor/universalchardet \
vendor/luabins \
libaegisub \
tools \
src \

View file

@ -81,6 +81,9 @@
<ProjectReference Include="..\lua51\lua51.vcxproj">
<Project>{5391a8b1-9c70-4dc4-92ad-d3e34c6b803f}</Project>
</ProjectReference>
<ProjectReference Include="..\luabins\luabins.vcxproj">
<Project>{A7A30702-8162-4E1A-A010-EF51B590C121}</Project>
</ProjectReference>
<ProjectReference Include="..\universalchardet\universalchardet.vcxproj">
<Project>{7b56955d-5162-4698-aa5b-47484edc8783}</Project>
</ProjectReference>

View file

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Label="Globals">
<ProjectGuid>{A7A30702-8162-4E1A-A010-EF51B590C121}</ProjectGuid>
<RootNamespace>luabins</RootNamespace>
</PropertyGroup>
<!-- Aegisub project configuration -->
<PropertyGroup Label="AegisubConfiguration">
<AegisubProjectType>lib</AegisubProjectType>
<SrcDir>..\..\vendor\luabins\src\</SrcDir>
</PropertyGroup>
<ImportGroup Label="PropertySheets">
<Import Project="$(MSBuildThisFileDirectory)..\aegisub.props" />
</ImportGroup>
<!-- Project specific configuration -->
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>LUABINS_LUABUILTASCPP;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SrcDir)..\..\lua51\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
</ItemDefinitionGroup>
<!-- Source files -->
<ItemGroup>
<ClCompile Include="$(SrcDir)fwrite.cpp" />
<ClCompile Include="$(SrcDir)load.cpp" />
<ClCompile Include="$(SrcDir)luabins.cpp" />
<ClCompile Include="$(SrcDir)luainternals.cpp" />
<ClCompile Include="$(SrcDir)save.cpp" />
<ClCompile Include="$(SrcDir)savebuffer.cpp" />
<ClCompile Include="$(SrcDir)write.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="$(SrcDir)fwrite.h" />
<ClInclude Include="$(SrcDir)luabins.h" />
<ClInclude Include="$(SrcDir)luaheaders.h" />
<ClInclude Include="$(SrcDir)luainternals.h" />
<ClInclude Include="$(SrcDir)savebuffer.h" />
<ClInclude Include="$(SrcDir)saveload.h" />
<ClInclude Include="$(SrcDir)write.h" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,57 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Headers">
<UniqueIdentifier>{0A33FF05-970D-49a7-B722-73E8EA350084}</UniqueIdentifier>
</Filter>
<Filter Include="Source">
<UniqueIdentifier>{2C50401A-5AC6-4630-B633-DFCC190306A8}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="$(SrcDir)fwrite.cpp">
<Filter>Source</Filter>
</ClCompile>
<ClCompile Include="$(SrcDir)load.cpp">
<Filter>Source</Filter>
</ClCompile>
<ClCompile Include="$(SrcDir)luabins.cpp">
<Filter>Source</Filter>
</ClCompile>
<ClCompile Include="$(SrcDir)luainternals.cpp">
<Filter>Source</Filter>
</ClCompile>
<ClCompile Include="$(SrcDir)save.cpp">
<Filter>Source</Filter>
</ClCompile>
<ClCompile Include="$(SrcDir)savebuffer.cpp">
<Filter>Source</Filter>
</ClCompile>
<ClCompile Include="$(SrcDir)write.cpp">
<Filter>Source</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="$(SrcDir)fwrite.h">
<Filter>Headers</Filter>
</ClInclude>
<ClInclude Include="$(SrcDir)luabins.h">
<Filter>Headers</Filter>
</ClInclude>
<ClInclude Include="$(SrcDir)luaheaders.h">
<Filter>Headers</Filter>
</ClInclude>
<ClInclude Include="$(SrcDir)luainternals.h">
<Filter>Headers</Filter>
</ClInclude>
<ClInclude Include="$(SrcDir)savebuffer.h">
<Filter>Headers</Filter>
</ClInclude>
<ClInclude Include="$(SrcDir)saveload.h">
<Filter>Headers</Filter>
</ClInclude>
<ClInclude Include="$(SrcDir)write.h">
<Filter>Headers</Filter>
</ClInclude>
</ItemGroup>
</Project>

View file

@ -28,7 +28,7 @@ namespace agi { namespace ass {
/// Encode a blob of data, using ASS's nonstandard variant
template<typename RandomAccessRange>
std::string UUEncode(RandomAccessRange const& data) {
std::string UUEncode(RandomAccessRange const& data, bool insert_linebreaks=true) {
using std::begin;
using std::end;
@ -51,7 +51,7 @@ std::string UUEncode(RandomAccessRange const& data) {
for (size_t i = 0; i < std::min<size_t>(size - pos + 1, 4u); ++i) {
ret += dst[i] + 33;
if (++written == 80 && pos + 3 < size) {
if (insert_linebreaks && ++written == 80 && pos + 3 < size) {
written = 0;
ret += "\r\n";
}

View file

@ -14,6 +14,7 @@ LIBS := -L../libaegisub -laegisub $(LIBS)
LIBS += $(LIBS_GL) $(LIBS_PTHREAD) $(LIBS_WX) $(LIBS_FREETYPE)
LIBS += $(LIBS_FONTCONFIG) $(LIBS_FFTW3) $(LIBS_UCHARDET) $(LIBS_BOOST)
LIBS += $(LIBS_ICU) $(LIBS_LUA)
LIBS += ../vendor/luabins/libluabins.a
ifeq (yes, $(BUILD_DARWIN))
SRC += osx_utils.mm retina_helper.mm

View file

@ -44,6 +44,7 @@
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/regex.hpp>
#include <boost/spirit/include/karma_generate.hpp>
#include <boost/spirit/include/karma_int.hpp>
@ -120,7 +121,28 @@ void AssDialogue::Parse(std::string const& raw) {
for (int& margin : Margin)
margin = mid(0, boost::lexical_cast<int>(tkn.next_str()), 9999);
Effect = tkn.next_str_trim();
Text = std::string(tkn.next_tok().begin(), str.end());
std::string text{tkn.next_tok().begin(), str.end()};
static const boost::regex extradata_test("^\\{(=\\d+)+\\}");
boost::match_results<std::string::iterator> rematch;
if (boost::regex_search(text.begin(), text.end(), rematch, extradata_test)) {
std::string extradata_str = rematch.str(0);
text = rematch.suffix().str();
static const boost::regex idmatcher("=(\\d+)");
auto start = extradata_str.begin();
auto end = extradata_str.end();
std::vector<uint32_t> ids;
while (boost::regex_search(start, end, rematch, idmatcher)) {
auto id = boost::lexical_cast<uint32_t>(rematch.str(1));
ids.push_back(id);
start = rematch.suffix().first;
}
ExtradataIds = ids;
}
Text = text;
}
void append_int(std::string &str, int v) {
@ -156,6 +178,16 @@ std::string AssDialogue::GetData(bool ssa) const {
for (auto margin : Margin)
append_int(str, margin);
append_unsafe_str(str, Effect);
if (ExtradataIds.get().size() > 0) {
str += "{";
for (auto id : ExtradataIds.get()) {
str += "=";
boost::spirit::karma::generate(back_inserter(str), boost::spirit::karma::int_, id);
}
str += "}";
}
str += Text.get();
if (str.find('\n') != str.npos || str.find('\r') != str.npos) {

View file

@ -146,6 +146,8 @@ struct AssDialogueBase {
boost::flyweight<std::string> Actor;
/// Effect name
boost::flyweight<std::string> Effect;
/// IDs of extradata entries for line
boost::flyweight<std::vector<uint32_t>> ExtradataIds;
/// Raw text data
boost::flyweight<std::string> Text;
};
@ -183,3 +185,4 @@ public:
AssDialogue(std::string const& data);
~AssDialogue();
};

View file

@ -28,6 +28,7 @@ std::string const& AssEntry::GroupHeader(bool ssa) const {
"[Fonts]",
"[Graphics]",
"[Events]",
"[Aegisub Extradata]",
""
};
@ -37,6 +38,7 @@ std::string const& AssEntry::GroupHeader(bool ssa) const {
"[Fonts]",
"[Graphics]",
"[Events]",
"[Aegisub Extradata]",
""
};

View file

@ -43,6 +43,7 @@ enum class AssEntryGroup {
FONT,
GRAPHIC,
DIALOGUE,
EXTRADATA,
GROUP_MAX
};

View file

@ -26,6 +26,7 @@
#include <algorithm>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/filesystem/path.hpp>
#include <cassert>
AssFile::AssFile() { }
@ -54,6 +55,8 @@ void AssFile::LoadDefault(bool include_dialogue_line) {
AssFile::AssFile(const AssFile &from)
: Info(from.Info)
, Attachments(from.Attachments)
, Extradata(from.Extradata)
, next_extradata_id(from.next_extradata_id)
{
Styles.clone_from(from.Styles,
[](AssStyle const& e) { return new AssStyle(e); },
@ -68,6 +71,8 @@ void AssFile::swap(AssFile& from) throw() {
Styles.swap(from.Styles);
Events.swap(from.Events);
Attachments.swap(from.Attachments);
Extradata.swap(from.Extradata);
std::swap(next_extradata_id, from.next_extradata_id);
}
AssFile& AssFile::operator=(AssFile from) {
@ -229,3 +234,58 @@ void AssFile::Sort(EntryList<AssDialogue> &lst, CompFunc comp, std::set<AssDialo
begin = --end;
}
}
uint32_t AssFile::AddExtradata(std::string const& key, std::string const& value) {
// next_extradata_id must not exist
assert(Extradata.find(next_extradata_id) == Extradata.end());
Extradata[next_extradata_id] = std::make_pair(key, value);
return next_extradata_id++; // return old value, then post-increment
}
std::map<std::string, std::string> AssFile::GetExtradata(std::vector<uint32_t> const& id_list) const {
// If multiple IDs have the same key name, the last ID wins
std::map<std::string, std::string> result;
for (auto id : id_list) {
auto it = Extradata.find(id);
if (it != Extradata.end())
result[it->second.first] = it->second.second;
}
return result;
}
void AssFile::CleanExtradata() {
// Collect all IDs existing in the database
// Then remove all IDs found to be in use from this list
// Remaining is then all garbage IDs
std::vector<uint32_t> ids;
for (auto& it : Extradata) {
ids.push_back(it.first);
}
// For each line, find which IDs it actually uses and remove them from the unused-list
for (auto& line : Events) {
// Find the ID for each unique key in the line
std::map<std::string, uint32_t> key_ids;
for (auto id : line.ExtradataIds.get()) {
auto ed_it = Extradata.find(id);
if (ed_it == Extradata.end())
continue;
key_ids[ed_it->second.first] = id;
}
// Update the line's ID list to only contain the actual ID for any duplicate keys
// Also mark found IDs as used in the cleaning list
std::vector<uint32_t> new_ids;
for (auto& keyid : key_ids) {
new_ids.push_back(keyid.second);
ids.erase(std::remove(ids.begin(), ids.end(), keyid.second));
}
line.ExtradataIds = new_ids;
}
// The ids list should contain only unused IDs now
for (auto id : ids) {
Extradata.erase(id);
}
}

View file

@ -50,6 +50,8 @@ class wxString;
template<typename T>
using EntryList = typename boost::intrusive::make_list<T, boost::intrusive::constant_time_size<false>, boost::intrusive::base_hook<AssEntryListHook>>::type;
using AegisubExtradataMap = std::map<uint32_t, std::pair<std::string, std::string>>;
struct AssFileCommit {
wxString const& message;
int *commit_id;
@ -66,6 +68,9 @@ public:
EntryList<AssStyle> Styles;
EntryList<AssDialogue> Events;
std::vector<AssAttachment> Attachments;
AegisubExtradataMap Extradata;
uint32_t next_extradata_id = 0;
AssFile();
AssFile(const AssFile &from);
@ -102,6 +107,16 @@ public:
int GetUIStateAsInt(std::string const& key) const;
void SaveUIState(std::string const& key, std::string const& value);
/// @brief Add a new extradata entry
/// @param key Class identifier/owner for the extradata
/// @param value Data for the extradata
/// @return ID of the created entry
uint32_t AddExtradata(std::string const& key, std::string const& value);
/// Fetch all extradata entries from a list of IDs
std::map<std::string, std::string> GetExtradata(std::vector<uint32_t> const& id_list) const;
/// Remove unreferenced extradata entries
void CleanExtradata();
/// Type of changes made in a commit
enum CommitType {
/// Potentially the entire file has been changed; any saved information
@ -129,7 +144,9 @@ public:
COMMIT_DIAG_TIME = 0x40,
/// The text of existing dialogue lines have changed
COMMIT_DIAG_TEXT = 0x80,
COMMIT_DIAG_FULL = COMMIT_DIAG_META | COMMIT_DIAG_TIME | COMMIT_DIAG_TEXT
COMMIT_DIAG_FULL = COMMIT_DIAG_META | COMMIT_DIAG_TIME | COMMIT_DIAG_TEXT,
/// Extradata entries were added/modified/removed
COMMIT_EXTRADATA = 0x100,
};
DEFINE_SIGNAL_ADDERS(AnnounceCommit, AddCommitListener)
@ -168,3 +185,4 @@ public:
/// @param limit If non-empty, only lines in this set are sorted
static void Sort(EntryList<AssDialogue>& lst, CompFunc comp = CompStart, std::set<AssDialogue*> const& limit = std::set<AssDialogue*>());
};

View file

@ -19,14 +19,18 @@
#include "ass_file.h"
#include "ass_info.h"
#include "ass_style.h"
#include "string_codec.h"
#include "subtitle_format.h"
#include <libaegisub/ass/uuencode.h>
#include <libaegisub/make_unique.h>
#include <algorithm>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/regex.hpp>
AssParser::AssParser(AssFile *target, int version)
: target(target)
@ -113,6 +117,33 @@ void AssParser::ParseGraphicsLine(std::string const& data) {
attach = agi::make_unique<AssAttachment>(data, AssEntryGroup::GRAPHIC);
}
void AssParser::ParseExtradataLine(std::string const &data) {
static const boost::regex matcher("Data:[[:space:]]*(\\d+),([^,]+),(.)(.*)");
boost::match_results<std::string::const_iterator> mr;
if (boost::regex_match(data, mr, matcher)) {
auto id = boost::lexical_cast<uint32_t>(mr.str(1));
auto key = inline_string_decode(mr.str(2));
auto valuetype = mr.str(3);
auto value = mr.str(4);
if (valuetype == "e") {
// escaped/inline_string encoded
value = inline_string_decode(value);
} else if (valuetype == "u") {
// ass uuencoded
auto valuedata = agi::ass::UUDecode(value);
value = std::string(valuedata.begin(), valuedata.end());
} else {
// unknown, error?
value = "";
}
// ensure next_extradata_id is always at least 1 more than the largest existing id
target->next_extradata_id = std::max(id+1, target->next_extradata_id);
target->Extradata[id] = std::make_pair(key, value);
}
}
void AssParser::AddLine(std::string const& data) {
// Special-case for attachments since a line could theoretically be both a
// valid attachment data line and a valid section header, and if an
@ -144,6 +175,8 @@ void AssParser::AddLine(std::string const& data) {
state = &AssParser::ParseGraphicsLine;
else if (low == "[fonts]")
state = &AssParser::ParseFontLine;
else if (low == "[aegisub extradata]")
state = &AssParser::ParseExtradataLine;
else
state = &AssParser::UnknownLine;
return;

View file

@ -31,6 +31,7 @@ class AssParser {
void ParseScriptInfoLine(std::string const& data);
void ParseFontLine(std::string const& data);
void ParseGraphicsLine(std::string const& data);
void ParseExtradataLine(std::string const &data);
void UnknownLine(std::string const&) { }
public:
AssParser(AssFile *target, int version);

View file

@ -56,6 +56,7 @@
#include <libaegisub/make_unique.h>
#include <algorithm>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/predicate.hpp>
@ -274,6 +275,17 @@ namespace {
luaL_argcheck(L, lua_istable(L, 1), 1, "");
luaL_argcheck(L, lua_isstring(L, 2), 2, "");
// have to check that it looks like a style table before actually converting
// if it's a dialogue table then an active AssFile object is required
{
lua_getfield(L, 1, "class");
std::string actual_class{lua_tostring(L, -1)};
boost::to_lower(actual_class);
if (actual_class != "style")
return luaL_error(L, "Not a style entry");
lua_pop(L, 1);
}
lua_pushvalue(L, 1);
std::unique_ptr<AssEntry> et(Automation4::LuaAssFile::LuaToAssEntry(L));
auto st = dynamic_cast<AssStyle*>(et.get());
@ -295,6 +307,9 @@ namespace {
int luaopen_lpeg (lua_State *L);
// Forward-declaration for luabins library (not in any public header)
extern "C" int luaopen_luabins(lua_State * L);
namespace Automation4 {
int regex_init(lua_State *L);
@ -365,6 +380,7 @@ namespace Automation4 {
push_value(L, luaopen_package); lua_call(L, 0, 0);
push_value(L, luaopen_string); lua_call(L, 0, 0);
push_value(L, luaopen_table); lua_call(L, 0, 0);
push_value(L, luaopen_luabins); lua_call(L, 0, 0);
_stackcheck.check_stack(0);
// dofile and loadfile are replaced with include

View file

@ -124,7 +124,7 @@ namespace Automation4 {
/// makes a Lua representation of AssEntry and places on the top of the stack
void AssEntryToLua(lua_State *L, size_t idx);
/// assumes a Lua representation of AssEntry on the top of the stack, and creates an AssEntry object of it
static std::unique_ptr<AssEntry> LuaToAssEntry(lua_State *L);
static std::unique_ptr<AssEntry> LuaToAssEntry(lua_State *L, AssFile *ass=0);
/// @brief Signal that the script using this file is now done running
/// @param set_undo If there's any uncommitted changes to the file,

View file

@ -173,6 +173,15 @@ namespace Automation4 {
set_field(L, "text", dia->Text);
// create extradata table
lua_newtable(L);
for (auto const& ed : ass->GetExtradata(dia->ExtradataIds)) {
push_value(L, ed.first);
push_value(L, ed.second);
lua_settable(L, -3);
}
lua_setfield(L, -2, "extra");
set_field(L, "class", "dialogue");
}
else if (auto sty = dynamic_cast<const AssStyle*>(e)) {
@ -221,7 +230,7 @@ namespace Automation4 {
}
}
std::unique_ptr<AssEntry> LuaAssFile::LuaToAssEntry(lua_State *L)
std::unique_ptr<AssEntry> LuaAssFile::LuaToAssEntry(lua_State *L, AssFile *ass)
{
// assume an assentry table is on the top of the stack
// convert it to a real AssEntry object, and pop the table from the stack
@ -271,6 +280,7 @@ namespace Automation4 {
sty->UpdateData();
}
else if (lclass == "dialogue") {
assert(ass != 0); // since we need AssFile::AddExtradata
auto dia = new AssDialogue;
result.reset(dia);
@ -285,6 +295,18 @@ namespace Automation4 {
dia->Margin[2] = get_int_field(L, "margin_t", "dialogue");
dia->Effect = get_string_field(L, "effect", "dialogue");
dia->Text = get_string_field(L, "text", "dialogue");
lua_getfield(L, -1, "extra");
std::vector<uint32_t> new_ids;
lua_pushnil(L);
while (lua_next(L, -2)) {
auto key = get_string_or_default(L, -2);
auto value = get_string_or_default(L, -1);
new_ids.push_back(ass->AddExtradata(key, value));
lua_pop(L, 1);
}
lua_pop(L, 1);
dia->ExtradataIds = new_ids;
}
else {
luaL_error(L, "Found line with unknown class: %s", lclass.c_str());
@ -422,7 +444,7 @@ namespace Automation4 {
// insert
CheckBounds(n);
auto e = LuaToAssEntry(L);
auto e = LuaToAssEntry(L, ass);
modification_type |= modification_mask(e.get());
QueueLineForDeletion(n - 1);
AssignLine(n - 1, std::move(e));
@ -511,7 +533,7 @@ namespace Automation4 {
for (int i = 1; i <= n; i++) {
lua_pushvalue(L, i);
auto e = LuaToAssEntry(L);
auto e = LuaToAssEntry(L, ass);
modification_type |= modification_mask(e.get());
if (lines.empty()) {
@ -555,7 +577,7 @@ namespace Automation4 {
new_entries.reserve(n - 1);
for (int i = 2; i <= n; i++) {
lua_pushvalue(L, i);
auto e = LuaToAssEntry(L);
auto e = LuaToAssEntry(L, ass);
modification_type |= modification_mask(e.get());
InsertLine(new_entries, i - 2, std::move(e));
lua_pop(L, 1);
@ -594,7 +616,7 @@ namespace Automation4 {
int LuaAssFile::LuaParseKaraokeData(lua_State *L)
{
auto e = LuaToAssEntry(L);
auto e = LuaToAssEntry(L, ass);
auto dia = dynamic_cast<AssDialogue*>(e.get());
luaL_argcheck(L, dia, 1, "Subtitle line must be a dialogue line");

View file

@ -40,11 +40,12 @@ inline void push_value(lua_State *L, wxString const& value) {
}
inline void push_value(lua_State *L, agi::fs::path const& value) {
lua_pushstring(L, value.string().c_str());
std::string strval = value.string();
lua_pushlstring(L, strval.c_str(), strval.size());
}
inline void push_value(lua_State *L, std::string const& value) {
lua_pushstring(L, value.c_str());
lua_pushlstring(L, value.c_str(), value.size());
}
inline void push_value(lua_State *L, lua_CFunction value) {
@ -71,10 +72,11 @@ inline wxString check_wxstring(lua_State *L, int idx) {
}
inline std::string get_string_or_default(lua_State *L, int idx) {
const char *str = lua_tostring(L, idx);
size_t len = 0;
const char *str = lua_tolstring(L, idx, &len);
if (!str)
str = "<not a string>";
return str;
return std::string(str, len);
}
inline std::string get_global_string(lua_State *L, const char *name) {

View file

@ -45,7 +45,7 @@ std::string inline_string_encode(const std::string &input) {
auto format = boost::format("#%02X");
for (char c : input) {
if (c <= 0x1F || c == 0x23 || c == 0x2C || c == 0x3A || c == 0x7C)
output += str(format % c);
output += str(format % (int)(unsigned char)c);
else
output += c;
}

View file

@ -60,6 +60,7 @@ struct SubsController::UndoInfo {
std::vector<AssStyle> styles;
std::vector<AssDialogueBase> events;
std::vector<AssAttachment> attachments;
AegisubExtradataMap extradata;
mutable std::vector<int> selection;
int active_line_id = 0;
@ -69,6 +70,7 @@ struct SubsController::UndoInfo {
: undo_description(d)
, commit_id(commit_id)
, attachments(c->ass->Attachments)
, extradata(c->ass->Extradata)
{
script_info.reserve(c->ass->Info.size());
for (auto const& info : c->ass->Info)
@ -108,6 +110,7 @@ struct SubsController::UndoInfo {
if (binary_search(begin(selection), end(selection), copy->Id))
new_sel.insert(copy);
}
c->ass->Extradata = extradata;
c->ass->Commit("", AssFile::COMMIT_NEW);
c->selectionController->SetSelectionAndActive(std::move(new_sel), active_line);
@ -267,6 +270,8 @@ void SubsController::Save(agi::fs::path const& filename, std::string const& enco
FileSave();
context->ass->CleanExtradata();
writer->WriteFile(context->ass.get(), filename, 0, encoding);
}
catch (...) {

View file

@ -22,10 +22,12 @@
#include "ass_file.h"
#include "ass_style.h"
#include "ass_parser.h"
#include "string_codec.h"
#include "text_file_reader.h"
#include "text_file_writer.h"
#include "version.h"
#include <libaegisub/ass/uuencode.h>
#include <libaegisub/fs.h>
DEFINE_SIMPLE_EXCEPTION(AssParseError, SubtitleFormatParseError, "subtitle_io/parse/ass")
@ -114,6 +116,35 @@ struct Writer {
file.WriteLineToFile(ssa ? line.GetSSAText() : line.GetEntryData());
}
}
void WriteExtradata(AegisubExtradataMap const& extradata) {
if (extradata.size() == 0)
return;
group = AssEntryGroup::EXTRADATA;
file.WriteLineToFile("");
file.WriteLineToFile("[Aegisub Extradata]");
for (auto const& edi : extradata) {
std::string line = "Data: ";
line += std::to_string(edi.first);
line += ",";
line += inline_string_encode(edi.second.first);
line += ",";
std::string encoded_data = inline_string_encode(edi.second.second);
if (4*edi.second.second.size() < 3*encoded_data.size()) {
// the inline_string encoding grew the data by more than uuencoding would
// so base64 encode it instead
line += "u"; // marker for uuencoding
encoded_data = agi::ass::UUEncode(edi.second.second, false);
printf("did uuencoding, original size=%lu, encoded size=%lu\n", edi.second.second.size(), encoded_data.size());
line += encoded_data;
} else {
line += "e"; // marker for inline_string encoding (escaping)
line += encoded_data;
}
file.WriteLineToFile(line);
}
}
};
}
@ -124,4 +155,5 @@ void AssSubtitleFormat::WriteFile(const AssFile *src, agi::fs::path const& filen
writer.Write(src->Styles);
writer.Write(src->Attachments);
writer.Write(src->Events);
writer.WriteExtradata(src->Extradata);
}

4
vendor/luabins/AUTHORS vendored Normal file
View file

@ -0,0 +1,4 @@
Luabins authors:
----------------
Alexander Gladysh <agladysh@gmail.com>

51
vendor/luabins/BENCHMARK vendored Normal file
View file

@ -0,0 +1,51 @@
Luabins 0.2 benchmark (see etc/benchmark.lua) results on
MacBook Pro 2.4 GHz Intel Core Duo 2.66 MB DDR2 SDRAM
OS X 10.6.2
GCC 4.2.1 (Apple Inc. build 5646) (dot 1)
Lua 5.1.4 from MacPorts
Luabins built with default Makefile configuration
Note that the data used in benchmark is quite trivial. You're advised
to find out if Luabins is "fast enough" for you by yourself.
Lua
-------------------------------------------------------------------
name | rel | abs s / iter = us (1e-6 s) / iter
-------------------------------------------------------------------
luabins_save | 1.0000 | 4.21 / 1000000 = 4.210000 us
luabins_load | 1.2043 | 5.07 / 1000000 = 5.070000 us
loadstring | 4.7435 | 19.97 / 1000000 = 19.970000 us
concat | 10.6413 | 44.80 / 1000000 = 44.800000 us
===================================================================
Luabins 0.1 benchmark (see etc/benchmark.lua) results on
MacBook Pro 2.4 GHz Intel Core Duo 2.66 MB DDR2 SDRAM
OS X 10.5.6
GCC 4.0.1 (Apple Inc. build 5490)
Lua 5.1.4 from MacPorts
LuaJIT 1.1.3 built from sources with default configuration
Luabins built with default configuration
Note that the data used in benchmark is quite trivial. You're advised
to find out if Luabins is "fast enough" for you by yourself.
Lua
-------------------------------------------------------------------
name | rel | abs s / iter = us (1e-6 s) / iter
-------------------------------------------------------------------
luabins_load | 1.0000 | 6.34 / 1000000 = 6.340000 us
luabins_save | 1.7256 | 10.94 / 1000000 = 10.940000 us
loadstring | 3.6530 | 23.16 / 1000000 = 23.160000 us
concat | 10.0741 | 63.87 / 1000000 = 63.870000 us
LuaJIT -O
-------------------------------------------------------------------
name | rel | abs s / iter = us (1e-6 s) / iter
-------------------------------------------------------------------
luabins_load | 1.0000 | 5.40 / 1000000 = 5.400000 us
luabins_save | 1.6111 | 8.70 / 1000000 = 8.700000 us
concat | 6.6630 | 35.98 / 1000000 = 35.980000 us
loadstring | 23.6370 | 127.64 / 1000000 = 127.640000 us

32
vendor/luabins/COPYRIGHT vendored Normal file
View file

@ -0,0 +1,32 @@
Luabins License
---------------
Luabins is licensed under the terms of the MIT license reproduced below.
This means that luabins is free software and can be used for both academic
and commercial purposes at absolutely no cost.
===============================================================================
Copyright (C) 2009-2010 Luabins authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
===============================================================================
(end of COPYRIGHT)

47
vendor/luabins/HISTORY vendored Normal file
View file

@ -0,0 +1,47 @@
v0.3
====
Format unification for x86 and x86_64. Bug fixes.
WARNING: Format is not compatible with 0.2 and below.
Only data saved on x86_64 is affected, though.
Data, saved on x86, should load fine.
New features:
-- Format change: unified save format for x86 vs. x86_64.
-- API to save data to FILE * stream without Lua (see fwrite.h).
Bug fixes:
-- Load: fixed bug in readbyte, now it checks if we have data before read.
-- Load: fixed Lua C stack overflow bug on large data.
Misc:
-- Better module information.
Replaced luabins.VERSION with _VERSION, _DESCRIPTION and _COPYRIGHT.
-- Added some CLI tools, useful for Luabins development (see etc/).
-- Some code cleanup.
v0.2
====
Lua-less saving.
-- New, 2x faster luabins.save() (see BENCHMARK).
-- API to save data manually, without Lua (see write.h).
-- Added Luarocks rockspecs (see rockspec/).
-- Fixed Makefile for Ubuntu.
v0.1.1
======
Bugfix release.
-- Fixed handling of array holes in Lua tables.
v0.1
====
Initial release.

40
vendor/luabins/Makefile vendored Normal file
View file

@ -0,0 +1,40 @@
include ../../Makefile.inc
LIB = libluabins.a
CXXFLAGS += -I../lua/src -DLUABINS_LUABUILTASCPP
SRC = \
src/fwrite.cpp \
src/load.cpp \
src/luabins.cpp \
src/luainternals.cpp \
src/save.cpp \
src/savebuffer.cpp \
src/write.cpp
HEADER = \
src/fwrite.h \
src/luabins.h \
src/luaheaders.h \
src/luainternals.h \
src/savebuffer.h \
src/saveload.h \
src/write.h
EXTRA_DIST = \
src/lualess.c \
src/lualess.h \
etc/benchmark.lua \
etc/checkfmt.lua \
etc/dataset.lua \
etc/tolua.lua \
etc/toluabins.lua \
AUTHORS \
COPYRIGHT \
HISTORY \
README.md \
TODO
include ../../Makefile.target
-include src/*.d

112
vendor/luabins/README.md vendored Normal file
View file

@ -0,0 +1,112 @@
luabins — Lua Binary Serialization Library
==========================================
Allows to save tuples of primitive Lua types into binary chunks
and to load saved data back.
On serialization
----------------
### Luabins works with
* `nil`
* `boolean`
* `number`
* `string`
* `table` (see below)
### Luabins refuses to save
* `function`
* `thread`
* `userdata`
Luabins intentionally does not save or check any meta-information
(versions, endianness etc.) along with data. If needed, it is to be handled
elsewhere.
### Table serialization
1. Metatatables are ignored.
2. Table nesting depth should be no more than `LUABINS_MAXTABLENESTING`.
3. On table save references are not honored. Each encountered reference
becomes independent object on load:
local t = { 42 }
{ t, t }
becomes
{ { 42 }, { 42 } }
that is, three separate tables instead of two.
Lua API
-------
* `luabins.save(...)`
Saves arguments into a binary string.
* On success returns that string.
* On failure returns nil and error message.
Example:
local str = assert(luabins.save(1, "two", { "three", 4 }))
* `luabins.load(string)`
Loads a list of values from a binary string.
* On success returns true and loaded values.
* On failure returns nil and error message.
Example:
If you do not know in advance what data is stored inside a binary string,
you may put results into a table:
local values = { luabins.load(data) }
assert(values[1], values[2])
If you know how to handle stored values (for example you're sure they were
generated following some established protocol), you may want to use
something like this function to check `luabins.load()` result:
function eat_true(t, ...)
assert(t, ...)
return ...
end
my_value_handler(eat_true(luabins.load(data)))
C API
-----
* `int luabins_save(lua_State * L, int index_from, int index_to)`
Save Lua values from given state at given stack index range.
Lua value is left untouched. Note that empty range is not an error.
You may save from 0 to `LUABINS_MAXTUPLE` values.
Note only real non-negative indices work.
* On success returns 0, pushes saved data as a string on the top of stack.
* On failure returns non-zero, pushes error message on the top
of the stack.
* `int luabins_load(lua_State * L, const unsigned char * data,
size_t len, int *count)`
Load Lua values from given byte chunk.
* On success returns 0, pushes loaded values on stack.
Sets count to the number of values pushed.
Note that to have zero loaded items is a valid scenario.
* On failure returns non-zero, pushes error message on the top
of the stack.
Luabins is still an experimental volatile software.
Please see source code for more documentation.
See the copyright information in the file named `COPYRIGHT`.

11
vendor/luabins/TODO vendored Normal file
View file

@ -0,0 +1,11 @@
-- Add key-value size-less table format (see luatexts)
-- Add utf-8 char-length string format (see luatexts)
-- Add autotest targets for LuaJIT and LuaJIT 2 x86, x86_64, both for Lua and C.
-- Test on Lua with custom allocator.
-- Use Makefile in rockspec?
-- Enhance "corrupt data" message on load. Need more info on what is wrong.
Ensure every case is covered with tests.
-- Autocompact integers (especially strings -- most of them do not need size_t!)
-- Better cover new exponential growth strategy corner-cases with tests.
-- Shouldn't write and fwrite have common codebase?
-- Document write.h and fwrite.h

106
vendor/luabins/etc/benchmark.lua vendored Normal file
View file

@ -0,0 +1,106 @@
-- This benchmark is compatible with luamarca benchmarking system
-- http://github.com/agladysh/luamarca
package.cpath = "./?.so;"..package.cpath
local luabins = require("luabins")
local table_concat = table.concat
local loadstring, assert = loadstring, assert
local pairs, type, tostring = pairs, type, tostring
local luabins_save, luabins_load = luabins.save, luabins.load
local lua = ([[return {
true, false, 42, "string",
[{
true, false, 42, "string",
[true] = true, [false] = false, [42] = 42, string = "string"
}] =
{
true, false, 42, "string",
[true] = true, [false] = false, [42] = 42, string = "string"
}
}]]):gsub("[%s\n]+", "") -- Remove spaces for compactness
local data = assert(loadstring(lua))()
local saved = assert(luabins_save(data))
-- Imagine we know exact data structure.
-- We still impose some overhead on table.concat() related
-- stuff, since it is more realistic scenario.
-- Still looks a bit silly.
local concat = function(data)
local buf = {}
local function cat(v) buf[#buf + 1] = tostring(v) return cat end
-- Find table key
local tablekey, tableval
for k, v in pairs(data) do
if type(k) == "table" then
tablekey, tableval = k, v
break
end
end
cat 'return{'
cat (data[1]) ','
cat (data[2]) ','
cat (data[3]) ','
cat '"' (data[4]) '",'
cat '[{'
cat (tablekey[1]) ','
cat (tablekey[2]) ','
cat (tablekey[3]) ','
cat '"' (tablekey[4]) '",'
cat '[' (true) ']=' (tablekey[true]) ','
cat '[' (false) ']=' (tablekey[false]) ','
cat '[' (42) ']=' (tablekey[42]) ','
cat 'string' '=' '"' (tablekey["string"]) '"'
cat '}]='
cat '{'
cat (tablekey[1]) ','
cat (tablekey[2]) ','
cat (tablekey[3]) ','
cat '"' (tablekey[4]) '",'
cat '[' (true) ']=' (tablekey[true]) ','
cat '[' (false) ']=' (tablekey[false]) ','
cat '[' (42) ']=' (tablekey[42]) ','
cat 'string' '=' '"' (tablekey["string"]) '"'
cat '}'
cat '}'
return table_concat(buf, '')
end
-- Sanity check
assert(concat(data) == lua)
local bench = {}
bench.concat = function()
assert(concat(data))
end
bench.loadstring = function()
assert(loadstring(lua))()
end
bench.luabins_save = function()
assert(luabins_save(data))
end
bench.luabins_load = function()
assert(luabins_load(saved))
end
return bench

30
vendor/luabins/etc/checkfmt.lua vendored Normal file
View file

@ -0,0 +1,30 @@
--- Luabins format checker
package.cpath = "./lib/?.so;"..package.cpath
local luabins = require("luabins")
local luabins_save, luabins_load = luabins.save, luabins.load
local filename = select(1, ...)
assert(filename, "Usage: lua checkfmt.lua <out_filename>")
local file
if filename == "-" then
file = io.stdin
else
file = assert(io.open(filename, "r"))
end
assert(
luabins_load(
file:read("*a")
)
)
if file ~= io.stdin then
file:close()
end
file = nil
print("OK")

90
vendor/luabins/etc/dataset.lua vendored Normal file
View file

@ -0,0 +1,90 @@
-- Random Luabins dataset generator
package.cpath = "./lib/?.so;"..package.cpath
local luabins = require("luabins")
local luabins_save, luabins_load = luabins.save, luabins.load
math.randomseed(123456)
-- TODO: Generalize. Copy-paste from test.lua
local function gen_random_dataset(num, nesting)
num = num or math.random(0, 128)
nesting = nesting or 1
local gen_str = function()
local t = {}
local n = math.random(0, 1024)
for i = 1, n do
t[i] = string.char(math.random(0, 255))
end
return table.concat(t)
end
local gen_bool = function() return math.random() >= 0.5 end
local gen_nil = function() return nil end
local generators =
{
gen_nil;
gen_nil;
gen_nil;
gen_bool;
gen_bool;
gen_bool;
function() return math.random() end;
function() return math.random(-10000, 10000) end;
function() return math.random() * math.random(-10000, 10000) end;
gen_str;
gen_str;
gen_str;
function()
if nesting >= 24 then
return nil
end
local t = {}
local n = math.random(0, 24 - nesting)
for i = 1, n do
local k = gen_random_dataset(1, nesting + 1)
if k == nil then
k = "(nil)"
end
t[ k ] = gen_random_dataset(
1,
nesting + 1
)
end
return t
end;
}
local t = {}
for i = 1, num do
local n = math.random(1, #generators)
t[i] = generators[n]()
end