mia/Aegisub
1
0
Fork 1
Code Issues Pull requests Projects Releases Wiki Activity

Rewrite encoding detection

Browse source 
Now feeds all data to uchardet, when uchardet is available. The file
size limit is removed.

When uchardet is not available, we look for UTF-8 BOM.
This should make loading UTF-8-BOM files faster.
Because Aegisub always save file in UTF-8-BOM, this should also
guarentee Aegisub will load large (>100MB) file saved by itself.

See Aegisub/Aegisub#110
This commit is contained in:
wangqr 2019-05-18 20:10:32 -04:00
parent 4c6d370d51
commit f733297499
1 changed files with 21 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
libaegisub/common/charset.cpp
View file

@ -29,37 +29,33 @@ namespace agi { namespace charset {
std::string Detect(agi::fs::path const& file) {
agi::read_file_mapping fp(file);
#ifdef WITH_UCHARDET
agi::scoped_holder<uchardet_t> ud(uchardet_new(), uchardet_delete);
for (uint64_t offset = 0; offset < fp.size(); ) {
auto read = std::min<uint64_t>(65536, fp.size() - offset);
auto buf = fp.read(offset, read);
uchardet_handle_data(ud, buf, read);
offset += read;
}
uchardet_data_end(ud);
std::string encoding = uchardet_get_charset(ud);
return encoding.empty() ? "binary" : encoding;
#else
// Look for utf-8 BOM
if (fp.size() >= 3) {
const char* buf = fp.read(0, 3);
if (!strncmp(buf, "\xef\xbb\xbf", 3))
return "utf-8";
}
// If it's over 100 MB it's either binary or big enough that we won't
// be able to do anything useful with it anyway
if (fp.size() > 100 * 1024 * 1024)
return "binary";
uint64_t binaryish = 0;
#ifdef WITH_UCHARDET
agi::scoped_holder<uchardet_t> ud(uchardet_new(), uchardet_delete);
for (uint64_t offset = 0; offset < fp.size(); ) {
auto read = std::min<uint64_t>(4096, fp.size() - offset);
auto buf = fp.read(offset, read);
uchardet_handle_data(ud, buf, read);
uchardet_data_end(ud);
if (*uchardet_get_charset(ud))
return uchardet_get_charset(ud);
offset += read;
// A dumb heuristic to detect binary files
for (size_t i = 0; i < read; ++i) {
if ((unsigned char)buf[i] < 32 && (buf[i] != '\r' && buf[i] != '\n' && buf[i] != '\t'))
++binaryish;
}
if (binaryish > offset / 8)
return "binary";
}
return uchardet_get_charset(ud);
#else
auto read = std::min<uint64_t>(4096, fp.size());
auto read = std::min<uint64_t>(65536, fp.size());
auto buf = fp.read(0, read);
for (size_t i = 0; i < read; ++i) {
if ((unsigned char)buf[i] < 32 && (buf[i] != '\r' && buf[i] != '\n' && buf[i] != '\t'))