kdecore Library API Documentation

kcharsets.cpp

00001 /* This file is part of the KDE libraries
00002     Copyright (C) 1999 Lars Knoll (knoll@kde.org)
00003     $Id: kcharsets.cpp,v 1.134.2.1 2002/11/28 12:44:23 kevlo Exp $
00004 
00005     This library is free software; you can redistribute it and/or
00006     modify it under the terms of the GNU Library General Public
00007     License as published by the Free Software Foundation; either
00008     version 2 of the License, or (at your option) any later version.
00009 
00010     This library is distributed in the hope that it will be useful,
00011     but WITHOUT ANY WARRANTY; without even the implied warranty of
00012     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00013     Library General Public License for more details.
00014 
00015     You should have received a copy of the GNU Library General Public License
00016     along with this library; see the file COPYING.LIB.  If not, write to
00017     the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
00018     Boston, MA 02111-1307, USA.
00019 */
00020 #include "kcharsets.h"
00021 
00022 #include "kentities.c"
00023 
00024 #include <kapplication.h>
00025 #include <kglobal.h>
00026 #include <klocale.h>
00027 #include <kconfig.h>
00028 
00029 #include <qfontinfo.h>
00030 #include <qstrlist.h>
00031 #include <qfontdatabase.h>
00032 #include <kdebug.h>
00033 
00034 #include <qtextcodec.h>
00035 #include <qmap.h>
00036 #include <qcstring.h>
00037 
00038 #include <assert.h>
00039 
00040 #define CHARSETS_COUNT 33
00041 
00042 static const char * const language_names[] = {
00043     I18N_NOOP( "Other" ),
00044     I18N_NOOP( "Arabic" ),
00045     I18N_NOOP( "Baltic" ),
00046     I18N_NOOP( "Central European" ),
00047     I18N_NOOP( "Chinese Simplified" ),
00048     I18N_NOOP( "Chinese Traditional" ),
00049     I18N_NOOP( "Cyrillic" ),
00050     I18N_NOOP( "Greek" ),
00051     I18N_NOOP( "Hebrew" ),
00052     I18N_NOOP( "Japanese" ),
00053     I18N_NOOP( "Korean" ),
00054     I18N_NOOP( "Thai" ),
00055     I18N_NOOP( "Turkish" ),
00056     I18N_NOOP( "Western European" ),
00057     I18N_NOOP( "Tamil" ),
00058     I18N_NOOP( "Unicode" ),
00059     I18N_NOOP( "Northern Saami" )
00060 };
00061 
00062 // this list gives the charsets that can be used to display a file given in a certain encoding.
00063 // the list should be in order of preference
00064 // left side is the name returned by the codec used, right side the name of the charset as
00065 // used in kcharsets.cpp
00066 // 'unicode' will always be chosen as last resort, so it only needs to be added to the list,
00067 // if it should get a higher priority
00068 // every line must end with 0
00069 
00070 static const char* const charsets_for_encoding[] = {
00071     "koi8-r",                "koi8-r","cp 1251","koi8-u","iso-8859-5", 0,
00072     "koi8-u",                "koi8-u","cp 1251","iso-8859-5","koi8-r", 0,
00073     "iso 8859-1",            "iso8859-1","iso8859-15", 0,
00074     "iso 8859-2",            "iso8859-2","unicode","iso8859-1", 0,
00075     "iso 8859-3",            "iso8859-3","unicode","iso8859-1", 0,
00076     "iso 8859-4",            "iso8859-4","unicode","iso8859-13", "iso8859-1", 0,
00077     "iso 8859-5",            "iso8859-5","koi8-u","koi8-r", 0,
00078     "iso 8859-6",            "unicode","iso8859-6", 0,
00079     "iso 8859-7",            "iso8859-7", 0,
00080     "iso 8859-8",            "iso8859-8", 0,
00081     "iso 8859-8-i",          "iso8859-8", 0,
00082     "iso 8859-9",            "iso8859-9","unicode","iso8859-1", 0,
00083     "iso 8859-11",           "iso8859-11", 0,
00084     "iso 8859-13",           "iso8859-13","unicode","iso8859-4", "iso8859-1", 0,
00085     "iso 8859-15",           "iso8859-15","unicode","iso8859-1", 0,
00086     "utf8",                  "unicode","iso8859-1", 0,
00087     "utf16",                 "unicode","iso8859-1", 0,
00088     "iso-10646-ucs-2",       "unicode","iso8859-1", 0,
00089     "cp 1250",               "iso8859-2", 0,
00090     "cp 1251",               "cp 1251","koi8-u","koi8-r","iso8859-5", 0,
00091     "cp 1252",               "iso8859-1", 0,
00092     "cp 1253",               "iso8859-7", 0,
00093     "cp 1254",               "iso8859-9", 0,
00094     "cp 1255",               "iso8859-8", 0,
00095     "cp 1256",               "unicode","iso8859-6", 0,
00096     "cp 1257",               "iso8859-13", "iso8859-4", 0,
00097     "ibm852",                "unicode","iso-8859-2", 0,
00098     "tis620",                "iso8859-11", 0,
00099     "eucjp",                 "eucjp","unicode","iso8859-1", 0,
00100     "sjis",                  "eucjp","unicode","iso8859-1", 0,
00101     "jis7",                  "eucjp","unicode","iso8859-1", 0,
00102     "big5",                  "big5","unicode","iso8859-1", 0,
00103     "gbk",                   "gb2312.1980-0","gbk-0","unicode","iso8859-1", 0,
00104     "gb18030",               "gb18030.2000-1", "gb18030.2000-0", "unicode", "gbk-0", "gb2313.1980-0", "iso8859-1", 0,
00105     "gb2312",                "gb2312.1980-0","unicode","iso8859-1", 0,
00106     "euckr",                 "euckr","unicode","iso8859-1", 0,
00107     "tscii",                 "tscii", 0,
00108     "pt 154",                "pt 154","cp 1251","koi8-u","koi8-r","iso8859-5", 0,
00109     "winsami2",              "winsami2", "cp1252", "unicode", 0,
00110     0 }; // extra 0 for end
00111 
00112 // 0 other
00113 // 1 Arabic
00114 // 2 Baltic
00115 // 3 Central European
00116 // 4 Chinese Simplified
00117 // 5 Chinese Traditional
00118 // 6 Cyrillic
00119 // 7 Greek
00120 // 8 Hebrew
00121 // 9 Japanese
00122 // 10 Korean
00123 // 11 Thai
00124 // 12 Turkish
00125 // 13 Western European
00126 // 14 Tamil
00127 // 15 Unicode
00128 // 16 Northern Sami
00129 static struct LanguageForEncoding
00130     {
00131     const char* index;
00132     int data;
00133     } const language_for_encoding[] = {
00134     { "iso 8859-1", 13 },
00135     { "iso 8859-15", 13 },
00136     { "cp 1252", 13 },
00137     { "iso 8859-2", 3 },
00138     { "iso 8859-3", 3 },
00139     { "iso 8859-4", 2 },
00140     { "iso 8859-13", 2 },
00141     { "cp 1250", 3 },
00142     { "cp 1254", 12 },
00143     { "cp 1257", 2 },
00144     { "ibm852", 3 },
00145     { "koi8-r", 6 },
00146     { "iso 8859-5", 6 },
00147     { "cp 1251", 6 },
00148     { "koi8-u", 6 },
00149     { "pt 154", 6 },
00150     { "big5", 5 },
00151     { "gb18030", 4 },
00152     { "gbk", 4 },
00153     { "gb2312", 4 },
00154     { "euckr", 10 },
00155     { "sjis", 9 },
00156     { "jis7", 9 },
00157     { "eucjp", 9 },
00158     { "iso 8859-7", 7 },
00159     { "cp 1253", 7 },
00160     { "iso 8859-6", 1 },
00161     { "cp 1256", 1 },
00162     { "iso 8859-8", 8 },
00163     { "iso 8859-8-i", 8 },
00164     { "cp 1255", 8 },
00165     { "iso 8859-9", 12 },
00166     { "tis620", 11 },
00167     { "iso 8859-11", 11 },
00168     { "utf8", 15 },
00169     { "utf16", 15 },
00170     { "utf7", 15 },
00171     { "ucs2", 15 },
00172     { "iso-10646-ucs-2", 15 },
00173     { "winsami2", 16},
00174     { 0, 0 } };
00175 
00176 // defines some different names for codecs that are built into Qt.
00177 static struct Builtin
00178     {
00179     const char* index;
00180     const char* data;
00181     } const builtin[] = {
00182     { "iso-ir-111", "koi8-r" },
00183     { "koi8-ru", "koi8-u" },
00184     { "koi8r", "koi8-r" },
00185     { "koi8u", "koi8-u" },
00186     { "koi unified", "koi8-r" },
00187     { "us-ascii", "iso 8859-1" },
00188     { "usascii", "iso 8859-1" },
00189     { "x-utf-8", "utf-8" },
00190     { "x-utf-7", "utf-7" },
00191     { "unicode-1-1-utf-7", "utf-7" },
00192     { "ucs2", "iso-10646-ucs-2" },
00193     { "iso10646-1", "iso-10646-ucs-2" },
00194     { "big5-0", "big5" },
00195     { "gb18030.2000-1", "gb18030" },
00196     { "gb18030.2000-0", "gb18030" },
00197     { "gbk-0", "gbk" },
00198     { "gb2312", "gbk" },
00199     { "gb2312.1980-0", "gbk" },
00200     { "euc-kr", "euckr" },
00201     { "x-euc-kr", "euckr" },
00202     { "euc-jp", "eucjp" },
00203     { "x-euc-jp", "eucjp" },
00204     { "jisx0201.1976-0", "eucjp" },
00205     { "jisx0208.1983-0", "eucjp" },
00206     { "jisx0208.1990-0", "eucjp" },
00207     { "jisx0208.1997-0", "eucjp" },
00208     { "jisx0212.1990-0", "eucjp" },
00209     { "jisx0213.2000-1", "eucjp" },
00210     { "jisx0213.2000-2", "eucjp" },
00211     { "shift_jis", "sjis" },
00212     { "shift-jis", "sjis" },
00213     { "x-sjis", "sjis" },
00214     { "iso-2022-jp", "jis7" },
00215     { "windows1250", "cp 1250" },
00216     { "windows1251", "cp 1251" },
00217     { "windows1252", "cp 1252" },
00218     { "windows1253", "cp 1253" },
00219     { "windows1254", "cp 1254" },
00220     { "windows1255", "cp 1255" },
00221     { "windows1256", "cp 1256" },
00222     { "windows1257", "cp 1257" },
00223     { "windows-1250", "cp 1250" },
00224     { "windows-1251", "cp 1251" },
00225     { "windows-1252", "cp 1252" },
00226     { "windows-1253", "cp 1253" },
00227     { "windows-1254", "cp 1254" },
00228     { "windows-1255", "cp 1255" },
00229     { "windows-1256", "cp 1256" },
00230     { "windows-1257", "cp 1257" },
00231     { "x-windows-1250", "cp 1250" },
00232     { "x-windows-1251", "cp 1251" },
00233     { "x-windows-1252", "cp 1252" },
00234     { "x-windows-1253", "cp 1253" },
00235     { "x-windows-1254", "cp 1254" },
00236     { "x-windows-1255", "cp 1255" },
00237     { "x-windows-1256", "cp 1256" },
00238     { "x-windows-1257", "cp 1257" },
00239     { "cp-1250", "cp 1250" },
00240     { "cp-1251", "cp 1251" },
00241     { "cp-1252", "cp 1252" },
00242     { "cp-1253", "cp 1253" },
00243     { "cp-1254", "cp 1254" },
00244     { "cp-1255", "cp 1255" },
00245     { "cp-1256", "cp 1256" },
00246     { "cp-1257", "cp 1257" },
00247     { "x-cp-1250", "cp 1250" },
00248     { "x-cp-1251", "cp 1251" },
00249     { "x-cp-1252", "cp 1252" },
00250     { "x-cp-1253", "cp 1253" },
00251     { "x-cp-1254", "cp 1254" },
00252     { "x-cp-1255", "cp 1255" },
00253     { "x-cp-1256", "cp 1256" },
00254     { "x-cp-1257", "cp 1257" },
00255     { "tis620", "iso 8859-11" },
00256     { "tis-620", "iso 8859-11" },
00257     { "windows-874", "iso 8859-11" },
00258     { "windows874", "iso 8859-11" },
00259     { "x-windows-874", "iso 8859-11" },
00260     { "cp874", "iso 8859-11" },
00261     { "cp-874", "iso 8859-11" },
00262     { "x-cp-874", "iso 8859-11" },
00263     { "ksc5601.1987-0", "euckr" },
00264     { "ks_c_5601-1987", "euckr" },
00265     { "iso-8859-1", "iso 8859-1" },
00266     { "iso-8859-2", "iso 8859-2" },
00267     { "iso-8859-3", "iso 8859-3" },
00268     { "iso-8859-4", "iso 8859-4" },
00269     { "iso-8859-5", "iso 8859-5" },
00270     { "iso-8859-6", "iso 8859-6" },
00271     { "iso-8859-7", "iso 8859-7" },
00272     { "iso-8859-8", "iso 8859-8" },
00273     { "iso-8859-9", "iso 8859-9" },
00274     { "iso-8859-10", "iso 8859-10" },
00275     { "iso-8859-11", "iso 8859-11" },
00276     { "iso-8859-12", "iso 8859-12" },
00277     { "iso-8859-13", "iso 8859-13" },
00278     { "iso-8859-14", "iso 8859-14" },
00279     { "iso-8859-15", "iso 8859-15" },
00280     { "tscii", "tscii" },
00281     { "paratype-154", "pt 154" },
00282     { "pt-154", "pt 154" },
00283     { "x-winsami2", "winsami2" },
00284     { 0, 0 }};
00285 
00286 // some different names for the encodings defined in the charmaps files.
00287 // even though the charmap file names are all uppercase, the names are all lowercase here.
00288 static struct Aliases
00289     {
00290     const char* index;
00291     const char* data;
00292     } const aliases[] = {
00293     { "cp852", "ibm852" },
00294     { 0, 0 }};
00295 
00296 // some last resort hints in case the charmap file couldn't be found. This gives at least a partial conversion
00297 // and helps making things readable.
00298 // the name used as input here is already converted to the more canonical name as defined in the aliases array.
00299 static struct ConversionHints
00300     {
00301     const char* index;
00302     const char* data;
00303     } const conversion_hints[] = {
00304     { "cp1250", "iso-8859-2" },
00305     { "koi8-r", "iso-8859-5" },
00306     { "koi8-u", "koi8-r" },
00307     { "utf16", "iso-10646-ucs-2" },
00308     { 0, 0 }};
00309 
00310 
00311 // search an array of items index/data, index is const char*, data is T, find first matching index
00312 // and return data, or return 0
00313 template< typename T, typename Data >
00314 Data kcharsets_array_search( const T* start, const char* entry )
00315 {
00316     for( const T* pos = start;
00317          pos->index != 0;
00318          ++pos )
00319         if( qstrcmp( pos->index, entry ) == 0 )
00320             return pos->data;
00321     return 0;
00322 }
00323 
00324 
00325 class KCharsetsPrivate
00326 {
00327 public:
00328     KCharsetsPrivate(KCharsets* _kc)
00329         : codecForNameDict(43, false) // case insensitive
00330     {
00331         db = 0;
00332         kc = _kc;
00333     }
00334     ~KCharsetsPrivate()
00335     {
00336         delete db;
00337     }
00338     QFontDatabase *db;
00339     QAsciiDict<QTextCodec> codecForNameDict;
00340     KCharsets* kc;
00341 };
00342 
00343 // --------------------------------------------------------------------------
00344 
00345 KCharsets::KCharsets()
00346 {
00347     d = new KCharsetsPrivate(this);
00348 }
00349 
00350 KCharsets::~KCharsets()
00351 {
00352     delete d;
00353 }
00354 
00355 QChar KCharsets::fromEntity(const QString &str) 
00356 {
00357     QChar res = QChar::null;
00358 
00359     int pos = 0;
00360     if(str[pos] == '&') pos++;
00361 
00362     // Check for '&#000' or '&#x0000' sequence
00363     if (str[pos] == '#' && str.length()-pos > 1) {
00364         bool ok;
00365         pos++;
00366         if (str[pos] == 'x' || str[pos] == 'X') {
00367             pos++;
00368             // '&#x0000', hexadeciaml character reference
00369             QString tmp(str.unicode()+pos, str.length()-pos);
00370             res = tmp.toInt(&ok, 16);
00371         } else {
00372             //  '&#0000', deciaml character reference
00373             QString tmp(str.unicode()+pos, str.length()-pos);
00374             res = tmp.toInt(&ok, 10);
00375         }
00376         return res;
00377     }
00378 
00379     const entity *e = findEntity(str.ascii(), str.length());
00380 
00381     if(!e)
00382     {
00383         //kdDebug( 0 ) << "unknown entity " << str <<", len = " << str.length() << endl;
00384         return QChar::null;
00385     }
00386     //kdDebug() << "got entity " << str << " = " << e->code << endl;
00387 
00388     return QChar(e->code);
00389 }
00390 
00391 QChar KCharsets::fromEntity(const QString &str, int &len) 
00392 {
00393     // entities are never longer than 8 chars... we start from
00394     // that length and work backwards...
00395     len = 8;
00396     while(len > 0)
00397     {
00398         QString tmp = str.left(len);
00399         QChar res = fromEntity(tmp);
00400         if( res != QChar::null ) return res;
00401         len--;
00402     }
00403     return QChar::null;
00404 }
00405 
00406 
00407 QString KCharsets::toEntity(const QChar &ch) 
00408 {
00409     QString ent;
00410     ent.sprintf("&#0x%x;", ch.unicode());
00411     return ent;
00412 }
00413 
00414 QString KCharsets::resolveEntities( const QString &input )
00415 {
00416     QString text = input;
00417     const QChar *p = text.unicode();
00418     const QChar *end = p + text.length();
00419     const QChar *ampersand = 0;
00420     bool scanForSemicolon = false;
00421 
00422     for ( ; p < end; ++p ) {
00423         QChar ch = *p;
00424 
00425         if ( ch == '&' ) {
00426             ampersand = p;
00427             scanForSemicolon = true;
00428             continue;
00429         }
00430 
00431         if ( ch != ';' || scanForSemicolon == false )
00432             continue;
00433 
00434         assert( ampersand );
00435 
00436         scanForSemicolon = false;
00437 
00438         const QChar *entityBegin = ampersand + 1;
00439 
00440         uint entityLength = p - entityBegin;
00441         if ( entityLength == 0 )
00442             continue;
00443 
00444         QChar entityValue = KCharsets::fromEntity( QConstString( entityBegin, entityLength ).string() );
00445         if ( entityValue.isNull() )
00446             continue;
00447 
00448         uint ampersandPos = ampersand - text.unicode();
00449 
00450         text[ ampersandPos ] = entityValue;
00451         text.remove( ampersandPos + 1, entityLength + 1 );
00452         p = text.unicode() + ampersandPos;
00453         end = text.unicode() + text.length();
00454         ampersand = 0;
00455     }
00456 
00457     return text;
00458 }
00459 
00460 QStringList KCharsets::availableEncodingNames()
00461 {
00462     QStringList available;
00463 
00464     const char* const* pos = charsets_for_encoding;
00465     while( *pos != 0 ) {
00466         //kdDebug(0) << "key = " << *pos << endl;
00467 
00468 
00469         // iterate thorugh the list and find the first charset that is available
00470         for( const char* const* charsets = pos + 1;
00471              *charsets != 0;
00472              ++charsets ) {
00473             //kdDebug(0) << "checking for " << *charsets << endl;
00474 #ifdef __GNUC__
00475 #warning FIXME?
00476 #endif
00477             if( true ) {
00478                 //kdDebug(0) << *charsets << " available" << endl;
00479                 available.append( QString::fromLatin1( *pos ));
00480                 break;
00481             }
00482         }
00483         while( *pos != 0 ) // find end of line
00484             ++pos;
00485         ++pos; // move to the next line
00486     }
00487     return available;
00488 }
00489 
00490 QString KCharsets::languageForEncoding( const QString &encoding )
00491 {
00492     int lang = kcharsets_array_search< LanguageForEncoding, int >
00493         ( language_for_encoding, encoding.latin1());
00494     return i18n( language_names[lang] );
00495 }
00496 
00497 QString KCharsets::encodingForName( const QString &descriptiveName )
00498 {
00499     int left = descriptiveName.find( "( " );
00500     return descriptiveName.mid( left + 2, descriptiveName.find( " )" )
00501       - left - 2 );
00502 }
00503 
00504 QStringList KCharsets::descriptiveEncodingNames()
00505 {
00506   QStringList encodings = availableEncodingNames();
00507   QStringList::Iterator it;
00508   for( it = encodings.begin(); it != encodings.end(); ++it ) {
00509       QString lang = KGlobal::charsets()->languageForEncoding( *it );
00510       *it = lang + " ( " + *it + " )";
00511   }
00512   encodings.sort();
00513   return encodings;
00514 }
00515 
00516 QTextCodec *KCharsets::codecForName(const QString &n) const
00517 {
00518     bool b;
00519     return codecForName( n, b );
00520 }
00521 
00522 QTextCodec *KCharsets::codecForName(const QString &n, bool &ok) const
00523 {
00524     ok = true;
00525 
00526     QTextCodec* codec = 0;
00527     // dict lookup is case insensitive anyway
00528     if((codec = d->codecForNameDict[n.isEmpty() ? "->locale<-" : n.latin1()]))
00529         return codec; // cache hit, return
00530 
00531     if (n.isEmpty()) {
00532         codec = KGlobal::locale()->codecForEncoding();
00533         d->codecForNameDict.replace("->locale<-", codec);
00534         return codec;
00535     }
00536 
00537     QCString name = n.lower().latin1();
00538     QCString key = name;
00539     if (name.right(8) == "_charset")
00540        name.truncate(name.length()-8);
00541 
00542     if (name.isEmpty()) {
00543       ok = false;
00544       return QTextCodec::codecForName("iso8859-1");
00545     }
00546 
00547     codec = QTextCodec::codecForName(name);
00548 
00549     if(codec) {
00550         d->codecForNameDict.replace(key, codec);
00551         return codec;
00552     }
00553 
00554     // these codecs are built into Qt, but the name given for the codec is different,
00555     // so QTextCodec did not recognise it.
00556     QCString cname = kcharsets_array_search< Builtin, const char* >( builtin, name.data());
00557 
00558     if(!cname.isEmpty())
00559         codec = QTextCodec::codecForName(cname);
00560 
00561     if(codec)
00562     {
00563         d->codecForNameDict.replace(key, codec);
00564         return codec;
00565     }
00566 
00567     QString dir;
00568     {
00569     KConfigGroupSaver cfgsav( KGlobal::config(), "i18n" );
00570     dir = KGlobal::config()->readEntry("i18ndir", QString::fromLatin1("/usr/share/i18n/charmaps"));
00571     dir += "/";
00572     }
00573 
00574     // these are codecs not included in Qt. They can be build up if the corresponding charmap
00575     // is available in the charmap directory.
00576     cname = kcharsets_array_search< Aliases, const char* >( aliases, name.data());
00577 
00578     if(cname.isEmpty())
00579         cname = name;
00580     cname = cname.upper();
00581 
00582     codec = QTextCodec::loadCharmapFile((QString)(dir + cname.data()));
00583 
00584     if(codec) {
00585         d->codecForNameDict.replace(key, codec);
00586         return codec;
00587     }
00588 
00589     // this also failed, the last resort is now to take some compatibility charmap
00590 
00591     cname = cname.lower();
00592     cname = kcharsets_array_search< ConversionHints, const char* >( conversion_hints, (const char*)cname );
00593 
00594     if(!cname.isEmpty())
00595         codec = QTextCodec::codecForName(cname);
00596 
00597     if(codec) {
00598         d->codecForNameDict.replace(key, codec);
00599         return codec;
00600     }
00601 
00602     // could not assign a codec, let's return Latin1
00603     ok = false;
00604     return QTextCodec::codecForName("iso8859-1");
00605 }
KDE Logo
This file is part of the documentation for kdelibs Version 3.1.0.
Documentation copyright © 1996-2002 the KDE developers.
Generated on Wed Oct 8 12:20:39 2003 by doxygen 1.2.18 written by Dimitri van Heesch, © 1997-2001