Add Qt3 development HEAD version

1 files changed, 611 insertions, 0 deletions

diff --git a/src/codecs/qrtlcodec.cpp b/src/codecs/qrtlcodec.cpp
new file mode 100644
index 0000000..f77b0e6
--- /dev/null
+++ b/src/codecs/qrtlcodec.cpp
@@ -0,0 +1,611 @@
+/****************************************************************************
+**
+** Implementation of QTextCodec class
+**
+** Created : 981015
+**
+** Copyright (C) 1998-2008 Trolltech ASA.  All rights reserved.
+**
+** This file is part of the tools module of the Qt GUI Toolkit.
+**
+** This file may be used under the terms of the GNU General
+** Public License versions 2.0 or 3.0 as published by the Free
+** Software Foundation and appearing in the files LICENSE.GPL2
+** and LICENSE.GPL3 included in the packaging of this file.
+** Alternatively you may (at your option) use any later version
+** of the GNU General Public License if such license has been
+** publicly approved by Trolltech ASA (or its successors, if any)
+** and the KDE Free Qt Foundation.
+**
+** Please review the following information to ensure GNU General
+** Public Licensing requirements will be met:
+** http://trolltech.com/products/qt/licenses/licensing/opensource/.
+** If you are unsure which license is appropriate for your use, please
+** review the following information:
+** http://trolltech.com/products/qt/licenses/licensing/licensingoverview
+** or contact the sales department at [email protected].
+**
+** This file may be used under the terms of the Q Public License as
+** defined by Trolltech ASA and appearing in the file LICENSE.QPL
+** included in the packaging of this file.  Licensees holding valid Qt
+** Commercial licenses may use this file in accordance with the Qt
+** Commercial License Agreement provided with the Software.
+**
+** This file is provided "AS IS" with NO WARRANTY OF ANY KIND,
+** INCLUDING THE WARRANTIES OF DESIGN, MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE. Trolltech reserves all rights not granted
+** herein.
+**
+**********************************************************************/
+
+#include "qrtlcodec.h"
+#include <private/qtextengine_p.h>
+
+#ifndef QT_NO_CODEC_HEBREW
+
+// NOT REVISED
+
+static const uchar unkn = '?'; // BLACK SQUARE (94) would be better
+
+static const ushort heb_to_unicode[128] = {
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
+    0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x203E,
+    0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,
+    0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD,
+    0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2017,
+    0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7,
+    0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
+    0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7,
+    0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD
+};
+
+static const uchar unicode_to_heb_00[32] = {
+    0xA0, unkn, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
+    0xA8, 0xA9, 0xD7, 0xAB, 0xAC, 0xAD, 0xAE, unkn,
+    0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7,
+    0xB8, 0xB9, 0xF7, 0xBB, 0xBC, 0xBD, 0xBE, unkn,
+};
+
+static const uchar unicode_to_heb_05[32] = {
+    0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+    0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
+    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+    0xF8, 0xF9, 0xFA, unkn, unkn, unkn, unkn, unkn
+};
+
+static bool to8bit(const QChar ch, QCString *rstr)
+{
+    bool converted = FALSE;
+
+    if( ch.isMark() ) return TRUE; // ignore marks for conversion
+
+    if ( ch.row() ) {
+	if ( ch.row() == 0x05 ) {
+	    if ( ch.cell() > 0x91 )
+		converted = TRUE;
+	    // 0x0591 - 0x05cf: Hebrew punctuation... dropped
+	    if ( ch.cell() >= 0xD0 )
+		*rstr += (char)unicode_to_heb_05[ch.cell()- 0xD0];
+	} else if ( ch.row() == 0x20 ) {
+	    if ( ch.cell() == 0x3E ) {
+		*rstr += (char)0xAF;
+		converted = TRUE;
+	    } else if ( ch.cell() == 0x17 ) {
+		*rstr += (char)0xCF;
+		converted = TRUE;
+	    }
+	} else {
+	    converted = FALSE;
+	}
+    } else {
+	if ( ch.cell() < 0x80 ) {
+	    *rstr += (char)ch.cell();
+	    converted = TRUE;
+	} else if( ch.cell() < 0xA0 ) {
+	    *rstr += (char)unicode_to_heb_00[ch.cell() - 0x80];
+	    converted = TRUE;
+	}
+    }
+
+    if(converted) return TRUE;
+
+    // couldn't convert the char... lets try its decomposition
+    QString d = ch.decomposition();
+    if(d.isNull())
+	return FALSE;
+
+    int l = d.length();
+    for (int i=0; i<l; i++) {
+	const QChar ch = d[i];
+
+	if(to8bit(ch, rstr))
+	    converted = TRUE;
+    }
+
+    return converted;
+}
+
+#if 0
+static QString run(const QString &input, unsigned int from, unsigned int to, QChar::Direction runDir)
+{
+    if ( to <= from )
+	return QString::null;
+
+    QString out;
+    if ( runDir == QChar::DirR ) {
+	const QChar *ch = input.unicode() + to - 1;
+	int len = to - from;
+	while (len--) {
+	    out += *ch;
+	    ch--;
+	}
+    } else {
+	out = input.mid(from, to - from );
+    }
+    return out;
+}
+
+/*
+  we might do better here, but I'm currently not sure if it's worth the effort. It will hopefully convert
+  90% of the visually ordered Hebrew correctly.
+*/
+static QString reverseLine(const QString &str, unsigned int from, unsigned int to, QChar::Direction dir)
+{
+    QString out;
+
+    if ( to <= from ) {
+	out += str.at(from);
+	return out;
+    }
+
+    // since we don't have embedding marks, we get around with bidi levels up to 2.
+
+    // simple case: dir = RTL:
+    // go through the line from right to left, and reverse all continuous Hebrew strings.
+    if ( dir == QChar::DirR ) {
+	unsigned int pos = to;
+	to = from;
+	from = pos;
+	QChar::Direction runDir = QChar::DirON;
+
+	while ( pos > to ) {
+	    QChar::Direction d = str.at(pos).direction();
+	    switch ( d ) {
+		case QChar::DirL:
+		case QChar::DirAN:
+		case QChar::DirEN:
+		    if ( runDir != QChar::DirL ) {
+			out += run( str, pos, from, runDir );
+			from = pos - 1;
+		    }
+		    runDir = QChar::DirL;
+		    break;
+		case QChar::DirON:
+		    if ( runDir == QChar::DirON ) {
+			runDir = QChar::DirR;
+			break;
+		    }
+		    // fall through
+		case QChar::DirR:
+		    if ( runDir != QChar::DirR ) {
+			out += run( str, pos, from, runDir );
+			from = pos - 1;
+		    }
+		    runDir = QChar::DirR;
+		default:
+		    break;
+	    }
+	    pos--;
+	}
+	out += run( str, pos, from, runDir );
+    } else {
+	// basicDir == DirL. A bit more complicated, as we might need to reverse two times for numbers.
+	unsigned int pos = from;
+	QChar::Direction runDir = QChar::DirON;
+
+	// first reversing. Ignore numbers
+	while ( pos < to ) {
+	    QChar::Direction d = str.at(pos).direction();
+	    switch ( d ) {
+		case QChar::DirL:
+		    if ( runDir != QChar::DirL && runDir != QChar::DirON ) {
+			out += run( str, from, pos, runDir );
+			qDebug( "out = %s", out.latin1() );
+			from = pos;
+		    }
+		    runDir = QChar::DirL;
+		    break;
+		case QChar::DirON:
+		    if ( runDir == QChar::DirON ) {
+			runDir = QChar::DirL;
+			break;
+		    }
+		    // fall through
+		case QChar::DirR:
+		case QChar::DirAN:
+		case QChar::DirEN:
+		    if ( runDir != QChar::DirR && runDir != QChar::DirON ) {
+			out += run( str, from, pos, runDir );
+			qDebug( "out = %s", out.latin1() );
+			from = pos;
+		    }
+		    runDir = QChar::DirR;
+		default:
+		    break;
+	    }
+	    pos++;
+	}
+	out += run( str, from, pos, runDir );
+	qDebug( "out = %s", out.latin1() );
+	// second reversing for numbers
+	QString in = out;
+	out = "";
+	pos = 0;
+	from = 0;
+	to = in.length() - 1;
+	runDir = QChar::DirON;
+	while ( pos < to ) {
+	    QChar::Direction d = str.at(pos).direction();
+	    switch ( d ) {
+		case QChar::DirL:
+		case QChar::DirON:
+		case QChar::DirR:
+		    if ( runDir == QChar::DirEN && runDir != QChar::DirON ) {
+			out += run( in, from, pos, QChar::DirR ); //DirR ensures reversing
+			qDebug( "out = %s", out.latin1() );
+			runDir = QChar::DirR;
+			from = pos;
+		    }
+		    runDir = QChar::DirL;
+		    break;
+		case QChar::DirAN:
+		case QChar::DirEN:
+		    if ( runDir != QChar::DirEN && runDir != QChar::DirON ) {
+			out += in.mid(from, pos-from+1);
+			qDebug( "out = %s", out.latin1() );
+			from = pos;
+		    }
+		    runDir = QChar::DirEN;
+		default:
+		    break;
+	    }
+	    pos++;
+	}
+	out += run( str, from, pos, runDir );
+
+    }
+    return out;
+}
+#endif
+
+/* this function assuems the QString is still visually ordered.
+ * Finding the basic direction of the text is not easy in this case, since
+ * a string like "my friend MOLAHS" could (in logical order) mean aswell
+ * "SHALOM my friend" or "my friend SHALOM", depending on the basic direction
+ * one assumes for the text.
+ *
+ * So this function uses some heuristics to find the right answer...
+ */
+static QChar::Direction findBasicDirection(QString str)
+{
+    unsigned int pos;
+    unsigned int len = str.length();
+    QChar::Direction dir1 = QChar::DirON;
+    QChar::Direction dir2 = QChar::DirON;
+
+    unsigned int startLine = 0;
+    // If the visual representation of the first line starts and ends with the same
+    // directionality, we know the answer.
+    pos = 0;
+    while (pos < len) {
+	if ( str.at(pos) == '\n' )
+	    startLine = pos;
+	if (str.at(pos).direction() < 2) { // DirR or DirL
+	    dir1 = str.at(pos).direction();
+	    break;
+	}
+	pos++;
+    }
+
+    if( pos == len ) // no directional chars, assume QChar::DirL
+	return QChar::DirL;
+
+    // move to end of line
+    while( pos < len && str.at(pos) != '\n' )
+	pos++;
+
+    while (pos > startLine) {
+	if (str.at(pos).direction() < 2) { // DirR or DirL
+	    dir2 = str.at(pos).direction();
+	    break;
+	}
+	pos--;
+    }
+
+    // both are the same, so we have the direction!
+    if ( dir1 == dir2 ) return dir1;
+
+    // guess with the help of punktuation marks...
+    // if the sentence ends with a punktuation, we should have a mark
+    // at one side of the text...
+
+    pos = 0;
+    while (pos < len-1 ) {
+	if(str.at(pos).category() == QChar::Punctuation_Other) {
+	    if( str.at(pos) != (char)0xbf && str.at(pos) != (char)0xa1 ) // spanish inverted question and exclamation mark
+		if( str.at(pos+1).direction() < 2 ) return QChar::DirR;
+	}
+	pos++;
+    }
+
+    pos = len;
+    while (pos < 1 && str.at(pos).direction() < 2 ) {
+	if(str.at(pos).category() == QChar::Punctuation_Other) {
+	    if( str.at(pos-1).direction() < 2 ) return QChar::DirL;
+	}
+	pos--;
+    }
+
+    // don't know try DirR...
+    return QChar::DirR;
+}
+
+
+/*!
+    \class QHebrewCodec qrtlcodec.h
+    \reentrant
+    \ingroup i18n
+
+    \brief The QHebrewCodec class provides conversion to and from
+    visually ordered Hebrew.
+
+    Hebrew as a semitic language is written from right to left.
+    Because older computer systems couldn't handle reordering a string
+    so that the first letter appears on the right, many older
+    documents were encoded in visual order, so that the first letter
+    of a line is the rightmost one in the string.
+
+    In contrast to this, Unicode defines characters to be in logical
+    order (the order you would read the string). This codec tries to
+    convert visually ordered Hebrew (8859-8) to Unicode. This might
+    not always work perfectly, because reversing the \e bidi
+    (bi-directional) algorithm that transforms from logical to visual
+    order is non-trivial.
+
+    Transformation from Unicode to visual Hebrew (8859-8) is done
+    using the bidi algorithm in Qt, and will produce correct results,
+    so long as the codec is given the text a whole paragraph at a
+    time. Places where newlines are supposed to go can be indicated by
+    a newline character ('\n'). Note that these newline characters
+    change the reordering behaviour of the algorithm, since the bidi
+    reordering only takes place within one line of text, whereas
+    line breaks are determined in visual order.
+
+    Visually ordered Hebrew is still used quite often in some places,
+    mainly in email communication (since most email programs still
+    don't understand logically ordered Hebrew) and on web pages. The
+    use on web pages is rapidly decreasing, due to the availability of
+    browsers that correctly support logically ordered Hebrew.
+
+    This codec has the name "iso8859-8". If you don't want any bidi
+    reordering to happen during conversion, use the "iso8859-8-i"
+    codec, which assumes logical order for the 8-bit string.
+*/
+
+/*! \reimp */
+int QHebrewCodec::mibEnum() const
+{
+    return 11;
+}
+
+/*! \reimp */
+const char* QHebrewCodec::name() const
+{
+    return "ISO 8859-8";
+}
+
+/*!
+    Returns the codec's mime name.
+*/
+const char* QHebrewCodec::mimeName() const
+{
+    return "ISO-8859-8";
+}
+
+static QString visualOrder(QString logical, QChar::Direction basicDir)
+{
+    logical.replace(QChar('\n'), QChar(0x2028));
+
+    QTextEngine e(logical, 0);
+    e.direction = basicDir;
+    e.itemize();
+    Q_UINT8 l[256];
+    Q_UINT8 *levels = l;
+    int vo[256];
+    int *visualOrder = vo;
+    int nitems = e.items.size();
+    if (nitems > 255) {
+	levels = new Q_UINT8[nitems];
+	visualOrder = new int[nitems];
+    }
+    int i;
+    for (i = 0; i < nitems; ++i) {
+	//qDebug("item %d bidiLevel=%d", i,  e.items[i].analysis.bidiLevel);
+	levels[i] = e.items[i].analysis.bidiLevel;
+    }
+    e.bidiReorder(nitems, levels, visualOrder);
+
+    QString visual;
+    for (i = 0; i < nitems; ++i) {
+	QScriptItem &si = e.items[visualOrder[i]];
+	QString sub = logical.mid(si.position, e.length(visualOrder[i]));
+	if (si.analysis.bidiLevel % 2) {
+	    // reverse sub
+	    QChar *a = (QChar *)sub.unicode();
+	    QChar *b = a + sub.length() - 1;
+	    while (a < b) {
+		QChar tmp = *a;
+		*a = *b;
+		*b = tmp;
+		++a;
+		--b;
+	    }
+	    a = (QChar *)sub.unicode();
+	    b = a + sub.length();
+	    while (a<b) {
+		*a = a->mirroredChar();
+		++a;
+	    }
+	}
+	visual += sub;
+    }
+    // replace Unicode newline back with  \n to compare.
+    visual.replace(QChar(0x2028), QChar('\n'));
+    if (l != levels) {
+	delete [] levels;
+	delete [] visualOrder;
+    }
+    return visual;
+}
+
+/*!
+    \reimp
+
+    Since Hebrew (and Arabic) is written from left to right, but
+    iso8859-8 assumes visual ordering (as opposed to the logical
+    ordering of Unicode), we must reverse the order of the input
+    string (the first \a len characters of \a chars) to put it into
+    logical order.
+
+    One problem is that the basic text direction is unknown. So this
+    function uses some heuristics to guess it, and if it can't guess
+    the right one, it assumes, the basic text direction is right to
+    left.
+
+    This behaviour can be overridden, by putting a control character
+    at the beginning of the text to indicate which basic text
+    direction to use. If the basic text direction is left-to-right,
+    the control character should be (uchar) 0xFE. For right-to-left it
+    should be 0xFF. Both characters are undefined in the iso 8859-8
+    charset.
+
+    Example: A visually ordered string "english WERBEH american" would
+    be recognized as having a basic left to right direction. So the
+    logically ordered QString would be "english HEBREW american".
+
+    By prepending a (uchar)0xFF at the start of the string,
+    QHebrewCodec::toUnicode() would use a basic text direction of
+    right to left, and the string would thus become "american HEBREW
+    english".
+*/
+QString QHebrewCodec::toUnicode(const char* chars, int len ) const
+{
+    QString r;
+    const unsigned char * c = (const unsigned char *)chars;
+    QChar::Direction basicDir = QChar::DirON; // neutral, we don't know
+
+    if( len == 0 ) return QString::null;
+
+    // Test, if the user gives us a directionality.
+    // We use 0xFE and 0xFF in ISO8859-8 for that.
+    // These chars are undefined in the charset, and are mapped to
+    // RTL overwrite
+    if( c[0] == 0xfe ) {
+	basicDir = QChar::DirL;
+	c++; // skip directionality hint
+    }
+    if( c[0] == 0xff ) {
+	basicDir = QChar::DirR;
+	c++; // skip directionality hint
+    }
+
+    for( int i=0; i<len; i++ ) {
+	if ( c[i] > 127 )
+	    r[i] = heb_to_unicode[c[i]-128];
+	else
+	    r[i] = c[i];
+    }
+
+    // do transformation from visual byte ordering to logical byte
+    // ordering
+    if( basicDir == QChar::DirON )
+	basicDir = findBasicDirection(r);
+
+    return visualOrder(r, basicDir);
+}
+
+/*!
+    Transforms the logically ordered QString, \a uc, into a visually
+    ordered string in the 8859-8 encoding. Qt's bidi algorithm is used
+    to perform this task. Note that newline characters affect the
+    reordering, since reordering is done on a line by line basis.
+
+    The algorithm is designed to work on whole paragraphs of text, so
+    processing a line at a time may produce incorrect results. This
+    approach is taken because the reordering of the contents of a
+    particular line in a paragraph may depend on the previous line in
+    the same paragraph.
+
+    Some encodings (for example Japanese or UTF-8) are multibyte (so
+    one input character is mapped to two output characters). The \a
+    lenInOut argument specifies the number of QChars that should be
+    converted and is set to the number of characters returned.
+*/
+QCString QHebrewCodec::fromUnicode(const QString& uc, int& lenInOut) const
+{
+    // process only len chars...
+    int l;
+    if( lenInOut > 0 )
+	l = QMIN((int)uc.length(),lenInOut);
+    else
+	l = (int)uc.length();
+
+    QCString rstr;
+    if( l == 1 ) {
+	if( !to8bit( uc[0], &rstr ) )
+	    rstr += (char)unkn;
+    } else {
+	QString tmp = uc;
+	tmp.truncate(l);
+	QString vis = visualOrder(tmp, QChar::DirON);
+
+	for (int i=0; i<l; i++) {
+	    const QChar ch = vis[i];
+
+	    if( !to8bit( ch, &rstr ) )
+		rstr += (char)unkn;
+	}
+	// lenInOut = cursor - result;
+    }
+    if( l > 0 && !rstr.length() )
+	rstr += (char)unkn;
+
+    return rstr;
+}
+
+/*! \reimp
+ */
+int QHebrewCodec::heuristicContentMatch(const char* chars, int len) const
+{
+    const unsigned char * c = (const unsigned char *)chars;
+
+    int score = 0;
+    for (int i=0; i<len; i++) {
+	if(c[i] > 0x80 ) {
+	    if ( heb_to_unicode[c[i] - 0x80] != 0xFFFD)
+		score++;
+	    else
+		return -1;
+	}
+    }
+    return score;
+}
+
+#endif