LCOV - app.info - extensions/universalchardet/src/base/CharDistribution.h

LCOV - code coverage report
| Current view: | directory - extensions/universalchardet/src/base - CharDistribution.h (source / functions) |  | Found | Hit | Coverage |
| Test: | app.info | Lines: | 49 | 0 | 0.0 %
| Date: | 2012-06-02 | Functions: | 11 | 0 | 0.0 %

           1                 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
           2                 : /* ***** BEGIN LICENSE BLOCK *****
           3                 :  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
           4                 :  *
           5                 :  * The contents of this file are subject to the Mozilla Public License Version
           6                 :  * 1.1 (the "License"); you may not use this file except in compliance with
           7                 :  * the License. You may obtain a copy of the License at
           8                 :  * http://www.mozilla.org/MPL/
           9                 :  *
          10                 :  * Software distributed under the License is distributed on an "AS IS" basis,
          11                 :  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
          12                 :  * for the specific language governing rights and limitations under the
          13                 :  * License.
          14                 :  *
          15                 :  * The Original Code is Mozilla Communicator client code.
          16                 :  *
          17                 :  * The Initial Developer of the Original Code is
          18                 :  * Netscape Communications Corporation.
          19                 :  * Portions created by the Initial Developer are Copyright (C) 1998
          20                 :  * the Initial Developer. All Rights Reserved.
          21                 :  *
          22                 :  * Contributor(s):
          23                 :  *
          24                 :  * Alternatively, the contents of this file may be used under the terms of
          25                 :  * either the GNU General Public License Version 2 or later (the "GPL"), or
          26                 :  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
          27                 :  * in which case the provisions of the GPL or the LGPL are applicable instead
          28                 :  * of those above. If you wish to allow use of your version of this file only
          29                 :  * under the terms of either the GPL or the LGPL, and not to allow others to
          30                 :  * use your version of this file under the terms of the MPL, indicate your
          31                 :  * decision by deleting the provisions above and replace them with the notice
          32                 :  * and other provisions required by the GPL or the LGPL. If you do not delete
          33                 :  * the provisions above, a recipient may use your version of this file under
          34                 :  * the terms of any one of the MPL, the GPL or the LGPL.
          35                 :  *
          36                 :  * ***** END LICENSE BLOCK ***** */
          37                 :
          38                 : #ifndef CharDistribution_h__
          39                 : #define CharDistribution_h__
          40                 :
          41                 : #include "nscore.h"
          42                 :
          43                 : #define ENOUGH_DATA_THRESHOLD 1024
          44                 :
          45                 : #define MINIMUM_DATA_THRESHOLD  4
          46                 :
          47                 : class CharDistributionAnalysis
          48                 : {
          49                 : public:
          50               0 :   CharDistributionAnalysis() {Reset(false);}
          51                 :
          52                 :   //feed a block of data and do distribution analysis
          53                 :   void HandleData(const char* aBuf, PRUint32 aLen) {}
          54                 :
          55                 :   //Feed a character with known length
          56               0 :   void HandleOneChar(const char* aStr, PRUint32 aCharLen)
          57                 :   {
          58                 :     PRInt32 order;
          59                 :
          60                 :     //we only care about 2-bytes character in our distribution analysis
          61               0 :     order = (aCharLen == 2) ? GetOrder(aStr) : -1;
          62                 :
          63               0 :     if (order >= 0)
          64                 :     {
          65               0 :       mTotalChars++;
          66                 :       //order is valid
          67               0 :       if ((PRUint32)order < mTableSize)
          68                 :       {
          69               0 :         if (512 > mCharToFreqOrder[order])
          70               0 :           mFreqChars++;
          71                 :       }
          72                 :     }
          73               0 :   }
          74                 :
          75                 :   //return confidence base on existing data
          76                 :   float GetConfidence(void);
          77                 :
          78                 :   //Reset analyser, clear any state
          79               0 :   void      Reset(bool aIsPreferredLanguage)
          80                 :   {
          81               0 :     mDone = false;
          82               0 :     mTotalChars = 0;
          83               0 :     mFreqChars = 0;
          84               0 :     mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD;
          85               0 :   }
          86                 :
          87                 :   //This function is for future extension. Caller can use this function to control
          88                 :   //analyser's behavior
          89                 :   void      SetOpion(){}
          90                 :
          91                 :   //It is not necessary to receive all data to draw conclusion. For charset detection,
          92                 :   // certain amount of data is enough
          93               0 :   bool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;}
          94                 :
          95                 : protected:
          96                 :   //we do not handle character base on its original encoding string, but
          97                 :   //convert this encoding string to a number, here called order.
          98                 :   //This allow multiple encoding of a language to share one frequency table
          99               0 :   virtual PRInt32 GetOrder(const char* str) {return -1;}
         100                 :
         101                 :   //If this flag is set to true, detection is done and conclusion has been made
         102                 :   bool     mDone;
         103                 :
         104                 :   //The number of characters whose frequency order is less than 512
         105                 :   PRUint32 mFreqChars;
         106                 :
         107                 :   //Total character encounted.
         108                 :   PRUint32 mTotalChars;
         109                 :
         110                 :   //Number of hi-byte characters needed to trigger detection
         111                 :   PRUint32 mDataThreshold;
         112                 :
         113                 :   //Mapping table to get frequency order from char order (get from GetOrder())
         114                 :   const PRInt16  *mCharToFreqOrder;
         115                 :
         116                 :   //Size of above table
         117                 :   PRUint32 mTableSize;
         118                 :
         119                 :   //This is a constant value varies from language to language, it is used in
         120                 :   //calculating confidence. See my paper for further detail.
         121                 :   float    mTypicalDistributionRatio;
         122                 : };
         123                 :
         124                 :
         125                 : class EUCTWDistributionAnalysis: public CharDistributionAnalysis
         126                 : {
         127                 : public:
         128                 :   EUCTWDistributionAnalysis();
         129                 : protected:
         130                 :
         131                 :   //for euc-TW encoding, we are interested
         132                 :   //  first  byte range: 0xc4 -- 0xfe
         133                 :   //  second byte range: 0xa1 -- 0xfe
         134                 :   //no validation needed here. State machine has done that
         135               0 :   PRInt32 GetOrder(const char* str)
         136               0 :   { if ((unsigned char)*str >= (unsigned char)0xc4)
         137               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1;
         138                 :     else
         139               0 :       return -1;
         140                 :   }
         141                 : };
         142                 :
         143                 :
         144                 : class EUCKRDistributionAnalysis : public CharDistributionAnalysis
         145                 : {
         146                 : public:
         147                 :   EUCKRDistributionAnalysis();
         148                 : protected:
         149                 :   //for euc-KR encoding, we are interested
         150                 :   //  first  byte range: 0xb0 -- 0xfe
         151                 :   //  second byte range: 0xa1 -- 0xfe
         152                 :   //no validation needed here. State machine has done that
         153               0 :   PRInt32 GetOrder(const char* str)
         154               0 :   { if ((unsigned char)*str >= (unsigned char)0xb0)
         155               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
         156                 :     else
         157               0 :       return -1;
         158                 :   }
         159                 : };
         160                 :
         161                 : class GB2312DistributionAnalysis : public CharDistributionAnalysis
         162                 : {
         163                 : public:
         164                 :   GB2312DistributionAnalysis();
         165                 : protected:
         166                 :   //for GB2312 encoding, we are interested
         167                 :   //  first  byte range: 0xb0 -- 0xfe
         168                 :   //  second byte range: 0xa1 -- 0xfe
         169                 :   //no validation needed here. State machine has done that
         170               0 :   PRInt32 GetOrder(const char* str)
         171               0 :   { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1)
         172               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
         173                 :     else
         174               0 :       return -1;
         175                 :   }
         176                 : };
         177                 :
         178                 :
         179                 : class Big5DistributionAnalysis : public CharDistributionAnalysis
         180                 : {
         181                 : public:
         182                 :   Big5DistributionAnalysis();
         183                 : protected:
         184                 :   //for big5 encoding, we are interested
         185                 :   //  first  byte range: 0xa4 -- 0xfe
         186                 :   //  second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
         187                 :   //no validation needed here. State machine has done that
         188               0 :   PRInt32 GetOrder(const char* str)
         189               0 :   { if ((unsigned char)*str >= (unsigned char)0xa4)
         190               0 :       if ((unsigned char)str[1] >= (unsigned char)0xa1)
         191               0 :         return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63;
         192                 :       else
         193               0 :         return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40;
         194                 :     else
         195               0 :       return -1;
         196                 :   }
         197                 : };
         198                 :
         199                 : class SJISDistributionAnalysis : public CharDistributionAnalysis
         200                 : {
         201                 : public:
         202                 :   SJISDistributionAnalysis();
         203                 : protected:
         204                 :   //for sjis encoding, we are interested
         205                 :   //  first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
         206                 :   //  second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
         207                 :   //no validation needed here. State machine has done that
         208               0 :   PRInt32 GetOrder(const char* str)
         209                 :   {
         210                 :     PRInt32 order;
         211               0 :     if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f)
         212               0 :       order = 188 * ((unsigned char)str[0]-(unsigned char)0x81);
         213               0 :     else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef)
         214               0 :       order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31);
         215                 :     else
         216               0 :       return -1;
         217               0 :     order += (unsigned char)*(str+1) - 0x40;
         218               0 :     if ((unsigned char)str[1] > (unsigned char)0x7f)
         219               0 :       order--;
         220               0 :     return order;
         221                 :   }
         222                 : };
         223                 :
         224                 : class EUCJPDistributionAnalysis : public CharDistributionAnalysis
         225                 : {
         226                 : public:
         227                 :   EUCJPDistributionAnalysis();
         228                 : protected:
         229                 :   //for euc-JP encoding, we are interested
         230                 :   //  first  byte range: 0xa0 -- 0xfe
         231                 :   //  second byte range: 0xa1 -- 0xfe
         232                 :   //no validation needed here. State machine has done that
         233               0 :   PRInt32 GetOrder(const char* str)
         234               0 :   { if ((unsigned char)*str >= (unsigned char)0xa0)
         235               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1;
         236                 :     else
         237               0 :       return -1;
         238                 :   }
         239                 : };
         240                 :
         241                 : #endif //CharDistribution_h__
         242                 :

Generated by: LCOV version 1.7