1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
/*
kmime_charfreq.cpp
KMime, the KDE internet mail/usenet news message library.
Copyright (c) 2001-2002 Marc Mutz <[email protected]>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, US
*/
#include "kmime_charfreq.h"
namespace KMime {
CharFreq::CharFreq( const TQByteArray & buf )
: NUL(0),
CTL(0),
CR(0), LF(0),
CRLF(0),
printable(0),
eightBit(0),
total(0),
lineMin(0xffffffff),
lineMax(0),
mTrailingWS(false),
mLeadingFrom(false)
{
if ( !buf.isEmpty() )
count( buf.data(), buf.size() );
}
CharFreq::CharFreq( const char * buf, size_t len )
: NUL(0),
CTL(0),
CR(0), LF(0),
CRLF(0),
printable(0),
eightBit(0),
total(0),
lineMin(0xffffffff),
lineMax(0),
mTrailingWS(false),
mLeadingFrom(false)
{
if ( buf && len > 0 )
count( buf, len );
}
static inline bool isWS( char ch ) { return ( ch == '\t' || ch == ' ' ); }
void CharFreq::count( const char * it, size_t len ) {
const char * end = it + len;
uint currentLineLength = 0;
// initialize the prevChar with LF so that From_ detection works w/o
// special-casing:
char prevChar = '\n';
char prevPrevChar = 0;
for ( ; it != end ; ++it ) {
++currentLineLength;
switch ( *it ) {
case '\0': ++NUL; break;
case '\r': ++CR; break;
case '\n': ++LF;
if ( prevChar == '\r' ) { --currentLineLength; ++CRLF; }
if ( currentLineLength >= lineMax ) lineMax = currentLineLength-1;
if ( currentLineLength <= lineMin ) lineMin = currentLineLength-1;
if ( !mTrailingWS )
if ( isWS( prevChar ) || ( prevChar == '\r' && isWS( prevPrevChar ) ) )
mTrailingWS = true;
currentLineLength = 0;
break;
case 'F': // check for lines starting with From_ if not found already:
if ( !mLeadingFrom )
if ( prevChar == '\n' && end - it >= 5 && !qstrncmp( "From ", it, 5 ) )
mLeadingFrom = true;
++printable;
break;
default:
{
uchar c = *it;
if ( c == '\t' || c >= ' ' && c <= '~' )
++printable;
else if ( c == 127 || c < ' ' )
++CTL;
else
++eightBit;
}
}
prevPrevChar = prevChar;
prevChar = *it;
}
// consider the length of the last line
if ( currentLineLength >= lineMax ) lineMax = currentLineLength;
if ( currentLineLength <= lineMin ) lineMin = currentLineLength;
// check whether the last character is tab or space
if ( isWS( prevChar ) )
mTrailingWS = true;
total = len;
}
bool CharFreq::isEightBitData() const {
return type() == EightBitData;
}
bool CharFreq::isEightBitText() const {
return type() == EightBitText;
}
bool CharFreq::isSevenBitData() const {
return type() == SevenBitData;
}
bool CharFreq::isSevenBitText() const {
return type() == SevenBitText;
}
bool CharFreq::hasTrailingWhitespace() const {
return mTrailingWS;
}
bool CharFreq::hasLeadingFrom() const {
return mLeadingFrom;
}
CharFreq::Type CharFreq::type() const {
#if 0
qDebug( "Total: %d; NUL: %d; CTL: %d;\n"
"CR: %d; LF: %d; CRLF: %d;\n"
"lineMin: %d; lineMax: %d;\n"
"printable: %d; eightBit: %d;\n"
"trailing whitespace: %s;\n"
"leading 'From ': %s;\n",
total, NUL, CTL, CR, LF, CRLF, lineMin, lineMax,
printable, eightBit,
mTrailingWS ? "yes" : "no" , mLeadingFrom ? "yes" : "no" );
#endif
if ( NUL ) // must be binary
return Binary;
// doesn't contain NUL's:
if ( eightBit ) {
if ( lineMax > 988 ) return EightBitData; // not allowed in 8bit
if ( CR != CRLF || controlCodesRatio() > 0.2 ) return EightBitData;
return EightBitText;
}
// doesn't contain NUL's, nor 8bit chars:
if ( lineMax > 988 ) return SevenBitData;
if ( CR != CRLF || controlCodesRatio() > 0.2 ) return SevenBitData;
// no NUL, no 8bit chars, no excessive CTLs and no lines > 998 chars:
return SevenBitText;
}
float CharFreq::printableRatio() const {
if ( total ) return float(printable) / float(total);
else return 0;
}
float CharFreq::controlCodesRatio() const {
if ( total ) return float(CTL) / float(total);
else return 0;
}
} // namespace KMime
|
