2008-03-08 08:52:20 +01:00
|
|
|
// Copyright (c) 2005, Rodrigo Braz Monteiro
|
|
|
|
// All rights reserved.
|
|
|
|
//
|
|
|
|
// Redistribution and use in source and binary forms, with or without
|
|
|
|
// modification, are permitted provided that the following conditions are met:
|
|
|
|
//
|
|
|
|
// * Redistributions of source code must retain the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer.
|
|
|
|
// * Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer in the documentation
|
|
|
|
// and/or other materials provided with the distribution.
|
|
|
|
// * Neither the name of the Aegisub Group nor the names of its contributors
|
|
|
|
// may be used to endorse or promote products derived from this software
|
|
|
|
// without specific prior written permission.
|
|
|
|
//
|
|
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
|
|
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
|
|
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
|
|
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
|
|
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
|
|
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
|
|
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
|
|
// POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
//
|
|
|
|
// -----------------------------------------------------------------------------
|
|
|
|
//
|
|
|
|
// AEGISUB
|
|
|
|
//
|
|
|
|
// Website: http://aegisub.cellosoft.com
|
|
|
|
// Contact: mailto:zeratul@cellosoft.com
|
|
|
|
//
|
|
|
|
|
|
|
|
|
|
|
|
///////////
|
|
|
|
// Headers
|
|
|
|
#include <algorithm>
|
|
|
|
#include <string>
|
|
|
|
#include <wx/wfstream.h>
|
|
|
|
#include "text_file_reader.h"
|
2008-03-13 22:28:22 +01:00
|
|
|
using namespace Gorgonsub;
|
2008-03-08 08:52:20 +01:00
|
|
|
|
|
|
|
#ifdef WITH_UNIVCHARDET
|
|
|
|
#include "charset_detect.h"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
///////////////
|
|
|
|
// Constructor
|
2008-03-14 09:00:55 +01:00
|
|
|
TextFileReader::TextFileReader(wxInputStream &stream,Gorgonsub::String enc,bool _trim,bool prefetch)
|
2008-03-08 08:52:20 +01:00
|
|
|
: file(stream)
|
|
|
|
{
|
|
|
|
// Setup
|
|
|
|
trim = _trim;
|
2008-03-14 09:00:55 +01:00
|
|
|
threaded = prefetch;
|
|
|
|
thread = NULL;
|
2008-03-08 08:52:20 +01:00
|
|
|
|
|
|
|
// Set encoding
|
|
|
|
encoding = enc.c_str();
|
|
|
|
if (encoding == _T("binary")) return;
|
|
|
|
SetEncodingConfiguration();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//////////////
|
|
|
|
// Destructor
|
2008-03-14 09:00:55 +01:00
|
|
|
TextFileReader::~TextFileReader()
|
|
|
|
{
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//////////////////////////////
|
|
|
|
// Set encoding configuration
|
2008-03-14 09:00:55 +01:00
|
|
|
void TextFileReader::SetEncodingConfiguration()
|
|
|
|
{
|
2008-03-08 08:52:20 +01:00
|
|
|
// Set encoding configuration
|
|
|
|
swap = false;
|
|
|
|
Is16 = false;
|
2008-03-13 19:06:13 +01:00
|
|
|
//conv = shared_ptr<wxMBConv>();
|
2008-03-08 08:52:20 +01:00
|
|
|
if (encoding == _T("UTF-8")) {
|
2008-03-13 04:42:27 +01:00
|
|
|
conv = shared_ptr<wxMBConv> (new wxMBConvUTF8);
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
else if (encoding == _T("UTF-16LE")) {
|
|
|
|
Is16 = true;
|
|
|
|
}
|
|
|
|
else if (encoding == _T("UTF-16BE")) {
|
|
|
|
Is16 = true;
|
|
|
|
swap = true;
|
|
|
|
}
|
|
|
|
else if (encoding == _T("UTF-7")) {
|
2008-03-13 04:42:27 +01:00
|
|
|
conv = shared_ptr<wxMBConv>(new wxCSConv(encoding));
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
else if (encoding == _T("Local")) {
|
2008-03-13 04:42:27 +01:00
|
|
|
conv = shared_ptr<wxMBConv> (wxConvCurrent,NullDeleter());
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
else {
|
2008-03-13 04:42:27 +01:00
|
|
|
conv = shared_ptr<wxMBConv> (new wxCSConv(encoding));
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//////////////////////////
|
|
|
|
// Reads a line from file
|
2008-03-14 09:00:55 +01:00
|
|
|
Gorgonsub::String TextFileReader::ActuallyReadLine()
|
|
|
|
{
|
2008-03-08 08:52:20 +01:00
|
|
|
wxString wxbuffer;
|
|
|
|
size_t bufAlloc = 1024;
|
|
|
|
wxbuffer.Alloc(bufAlloc);
|
|
|
|
std::string buffer = "";
|
|
|
|
|
|
|
|
// Read UTF-16 line from file
|
|
|
|
if (Is16) {
|
|
|
|
char charbuffer[3];
|
|
|
|
charbuffer[2] = 0;
|
|
|
|
wchar_t ch = 0;
|
|
|
|
size_t len = 0;
|
|
|
|
while (ch != L'\n' && !file.Eof()) {
|
|
|
|
// Read two chars from file
|
|
|
|
charbuffer[0] = 0;
|
|
|
|
charbuffer[1] = 0;
|
|
|
|
file.Read(charbuffer,2);
|
|
|
|
|
|
|
|
// Swap bytes for big endian
|
|
|
|
if (swap) {
|
|
|
|
register char aux = charbuffer[0];
|
|
|
|
charbuffer[0] = charbuffer[1];
|
|
|
|
charbuffer[1] = aux;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Convert two chars into a widechar and append to string
|
|
|
|
ch = *((wchar_t*)charbuffer);
|
|
|
|
if (len >= bufAlloc - 1) {
|
|
|
|
bufAlloc *= 2;
|
|
|
|
wxbuffer.Alloc(bufAlloc);
|
|
|
|
}
|
|
|
|
wxbuffer += ch;
|
|
|
|
len++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read ASCII/UTF-8 line from file
|
|
|
|
else {
|
|
|
|
//getline(file,buffer);
|
|
|
|
//wxbuffer.Clear();
|
|
|
|
//if (buffer.length()) wxbuffer = wxString(buffer.c_str(),*conv);
|
|
|
|
char temp = 0;
|
2008-03-09 05:16:52 +01:00
|
|
|
std::string buff;
|
2008-03-08 08:52:20 +01:00
|
|
|
while (temp != '\n' && !file.Eof()) {
|
|
|
|
file.Read(&temp,1);
|
|
|
|
if (temp != '\r') {
|
2008-03-09 05:16:52 +01:00
|
|
|
buff += temp;
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
}
|
2008-03-09 05:16:52 +01:00
|
|
|
if (buff.size()) wxbuffer = wxString(buff.c_str(),*conv);
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Remove line breaks
|
|
|
|
//wxbuffer.Replace(_T("\r"),_T("\0"));
|
|
|
|
//wxbuffer.Replace(_T("\n"),_T("\0"));
|
|
|
|
size_t len=wxbuffer.Length();
|
|
|
|
for (size_t i=0;i<len;i++) {
|
|
|
|
if (wxbuffer[i] == _T('\r') || wxbuffer[i] == _T('\n')) wxbuffer[i] = _T(' ');
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove BOM
|
|
|
|
if (wxbuffer.Length() > 0 && wxbuffer[0] == 0xFEFF) {
|
|
|
|
wxbuffer = wxbuffer.Mid(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trim
|
|
|
|
if (trim) {
|
|
|
|
wxbuffer.Trim(true);
|
|
|
|
wxbuffer.Trim(false);
|
|
|
|
}
|
2008-03-13 22:28:22 +01:00
|
|
|
return Gorgonsub::String(wxbuffer.c_str());
|
2008-03-08 08:52:20 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//////////////////////////////////
|
|
|
|
// Checks if there's more to read
|
2008-03-14 09:00:55 +01:00
|
|
|
bool TextFileReader::HasMoreLines()
|
|
|
|
{
|
|
|
|
wxCriticalSectionLocker locker(mutex);
|
2008-03-08 08:52:20 +01:00
|
|
|
return (!file.Eof());
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
////////////////////////////////
|
|
|
|
// Ensure that charset is valid
|
2008-03-14 09:00:55 +01:00
|
|
|
void TextFileReader::EnsureValid(Gorgonsub::String enc)
|
|
|
|
{
|
2008-03-08 08:52:20 +01:00
|
|
|
if (enc == _T("unknown") || enc == _T("UTF-32BE") || enc == _T("UTF-32LE")) {
|
|
|
|
wxString error = _T("Character set ");
|
|
|
|
error += enc;
|
|
|
|
error += _T(" is not supported.");
|
|
|
|
throw error.c_str();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
///////////////////////////
|
|
|
|
// Get encoding being used
|
2008-03-14 09:00:55 +01:00
|
|
|
String TextFileReader::GetCurrentEncoding()
|
|
|
|
{
|
2008-03-08 08:52:20 +01:00
|
|
|
return encoding.c_str();
|
|
|
|
}
|
2008-03-14 09:00:55 +01:00
|
|
|
|
|
|
|
|
|
|
|
///////////////////////////
|
|
|
|
// Reads a line from cache
|
|
|
|
String TextFileReader::ReadLineFromFile()
|
|
|
|
{
|
|
|
|
// Not threaded, just return it
|
|
|
|
if (!threaded) return ActuallyReadLine();
|
|
|
|
|
|
|
|
// Load into cache if needed
|
|
|
|
String final;
|
|
|
|
if (cache.size() == 0) {
|
|
|
|
wxCriticalSectionLocker locker(mutex);
|
|
|
|
cache.push_back(ActuallyReadLine());
|
|
|
|
}
|
|
|
|
|
|
|
|
bool startThread = false;
|
|
|
|
{
|
|
|
|
// Retrieve from cache
|
|
|
|
wxCriticalSectionLocker locker(mutex);
|
|
|
|
if (cache.size()) {
|
|
|
|
final = cache.front();
|
|
|
|
cache.pop_front();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Start the thread to prefetch more
|
|
|
|
if (cache.size() < 3 && thread == NULL) {
|
|
|
|
thread = new PrefetchThread(this);
|
|
|
|
startThread = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Starts the thread
|
|
|
|
if (startThread) {
|
|
|
|
thread->Create();
|
|
|
|
thread->Run();
|
|
|
|
}
|
|
|
|
|
|
|
|
return final;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
////////////////
|
|
|
|
// Thread entry
|
|
|
|
wxThread::ExitCode PrefetchThread::Entry()
|
|
|
|
{
|
|
|
|
// Lock
|
|
|
|
wxCriticalSectionLocker locker(parent->mutex);
|
|
|
|
while (parent->cache.size() < 3 && !parent->file.Eof()) {
|
|
|
|
// So I need to do this for whatever reason
|
|
|
|
if (TestDestroy()) {
|
|
|
|
parent->thread = NULL;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get line
|
|
|
|
parent->cache.push_back(parent->ActuallyReadLine());
|
|
|
|
}
|
|
|
|
|
|
|
|
// Die
|
|
|
|
//wxCriticalSectionLocker locker(parent->mutex);
|
|
|
|
parent->thread = NULL;
|
|
|
|
Delete();
|
|
|
|
return 0;
|
|
|
|
}
|