Update universalchardet to 29631:4dd0cf2ba86f (2009-06-27). Closes #866.

Originally committed to SVN as r3662.
This commit is contained in:
Amar Takhar 2009-10-09 15:37:55 +00:00
parent 61c3a3149e
commit 2623b8380f
6 changed files with 14 additions and 14 deletions

View file

@ -45,11 +45,10 @@
class CharDistributionAnalysis class CharDistributionAnalysis
{ {
public: public:
CharDistributionAnalysis() {Reset();}; CharDistributionAnalysis() {Reset();}
virtual ~CharDistributionAnalysis() {};
//feed a block of data and do distribution analysis //feed a block of data and do distribution analysis
void HandleData(const char* aBuf, PRUint32 aLen) {}; void HandleData(const char* aBuf, PRUint32 aLen) {}
//Feed a character with known length //Feed a character with known length
void HandleOneChar(const char* aStr, PRUint32 aCharLen) void HandleOneChar(const char* aStr, PRUint32 aCharLen)

View file

@ -39,7 +39,7 @@
#include "JpCntx.h" #include "JpCntx.h"
//This is hiragana 2-char sequence table, the number in each cell represents its frequency category //This is hiragana 2-char sequence table, the number in each cell represents its frequency category
const char jp2CharContext[83][83] = const PRUint8 jp2CharContext[83][83] =
{ {
{ 0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,}, { 0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,},
{ 2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4,}, { 2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4,},

View file

@ -46,13 +46,12 @@
#define MAX_REL_THRESHOLD 1000 #define MAX_REL_THRESHOLD 1000
//hiragana frequency category table //hiragana frequency category table
extern const char jp2CharContext[83][83]; extern const PRUint8 jp2CharContext[83][83];
class JapaneseContextAnalysis class JapaneseContextAnalysis
{ {
public: public:
JapaneseContextAnalysis() {Reset();}; JapaneseContextAnalysis() {Reset();}
virtual ~JapaneseContextAnalysis() {};
void HandleData(const char* aBuf, PRUint32 aLen); void HandleData(const char* aBuf, PRUint32 aLen);

View file

@ -56,21 +56,22 @@ nsSBCSGroupProber::nsSBCSGroupProber()
mProbers[7] = new nsSingleByteCharSetProber(&Win1253Model); mProbers[7] = new nsSingleByteCharSetProber(&Win1253Model);
mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel); mProbers[8] = new nsSingleByteCharSetProber(&Latin5BulgarianModel);
mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel); mProbers[9] = new nsSingleByteCharSetProber(&Win1251BulgarianModel);
mProbers[10] = new nsSingleByteCharSetProber(&TIS620ThaiModel);
nsHebrewProber *hebprober = new nsHebrewProber(); nsHebrewProber *hebprober = new nsHebrewProber();
// Notice: Any change in these indexes - 10,11,12 must be reflected // Notice: Any change in these indexes - 10,11,12 must be reflected
// in the code below as well. // in the code below as well.
mProbers[10] = hebprober; mProbers[11] = hebprober;
mProbers[11] = new nsSingleByteCharSetProber(&Win1255Model, PR_FALSE, hebprober); // Logical Hebrew mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, PR_FALSE, hebprober); // Logical Hebrew
mProbers[12] = new nsSingleByteCharSetProber(&Win1255Model, PR_TRUE, hebprober); // Visual Hebrew mProbers[13] = new nsSingleByteCharSetProber(&Win1255Model, PR_TRUE, hebprober); // Visual Hebrew
// Tell the Hebrew prober about the logical and visual probers // Tell the Hebrew prober about the logical and visual probers
if (mProbers[10] && mProbers[11] && mProbers[12]) // all are not null if (mProbers[11] && mProbers[12] && mProbers[13]) // all are not null
{ {
hebprober->SetModelProbers(mProbers[11], mProbers[12]); hebprober->SetModelProbers(mProbers[12], mProbers[13]);
} }
else // One or more is null. avoid any Hebrew probing, null them all else // One or more is null. avoid any Hebrew probing, null them all
{ {
for (PRUint32 i = 10; i <= 12; ++i) for (PRUint32 i = 11; i <= 13; ++i)
{ {
delete mProbers[i]; delete mProbers[i];
mProbers[i] = 0; mProbers[i] = 0;

View file

@ -40,7 +40,7 @@
#define nsSBCSGroupProber_h__ #define nsSBCSGroupProber_h__
#define NUM_OF_SBCS_PROBERS 13 #define NUM_OF_SBCS_PROBERS 14
class nsCharSetProber; class nsCharSetProber;
class nsSBCSGroupProber: public nsCharSetProber { class nsSBCSGroupProber: public nsCharSetProber {

View file

@ -119,6 +119,7 @@ extern const SequenceModel Win1251BulgarianModel;
extern const SequenceModel Latin2HungarianModel; extern const SequenceModel Latin2HungarianModel;
extern const SequenceModel Win1250HungarianModel; extern const SequenceModel Win1250HungarianModel;
extern const SequenceModel Win1255Model; extern const SequenceModel Win1255Model;
extern const SequenceModel TIS620ThaiModel;
#endif /* nsSingleByteCharSetProber_h__ */ #endif /* nsSingleByteCharSetProber_h__ */