forked from bruderstein/PythonScript
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathUtfConversion.cpp
More file actions
147 lines (133 loc) · 4.46 KB
/
Copy pathUtfConversion.cpp
File metadata and controls
147 lines (133 loc) · 4.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
/**
* Functions for UTF-8, UTF-16 and UTF-32.
* @author Francois-R.Boyer@PolyMtl.ca
* @date March, 2013
* @copyright GPL; see license.txt in Notepad++ source for complete license text.
* @file
*/
#include "stdafx.h"
#include "UtfConversion.h"
namespace UtfConversion {
const unsigned char _firstByteMaskForLength[7] = { 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
const U32 _minimumCodePointForLength[7] = { 0, 0, 0x80, 0x800, 0x10000, 0x200000, 0x4000000 };
#pragma warning( push )
#pragma warning( disable : 4127 )
template <bool generate_utf8>
static unsigned int _toUtf8(const ConstString<U32> u32source, U8* u8destination = 0)
{
static const UCHAR utf8_leading_byte[] = { 0, 0, 0xC0, 0xE0, 0xF0 };
static const UCHAR utf8_continuation_byte = 0x80;
unsigned int u8_length = 0;
for (unsigned int i = 0; i < u32source.length(); ++i)
{
U32 u32c = u32source[i];
if (isInvalidUtf8(u32c)) {
if (generate_utf8)
*(u8destination++) = U8(u32c - invalid_utf8_base);
u8_length += 1;
continue;
}
unsigned int c = unsigned int(u32c);
const unsigned int u8_character_length =
(c < 0x80) ? 1 :
(c < 0x800) ? 2 :
(c < 0x10000) ? 3 :
(c <= maximum_unicode_point) ? 4 : // Should not go higher than maximum_utf8_length.
0;
if (generate_utf8 && u8_character_length != 0) {
*(u8destination++) = U8((c >> (6*(u8_character_length-1))) | utf8_leading_byte[u8_character_length]);
switch (u8_character_length)
{
case 4: *(u8destination++) = U8(((c >> 12) & 0x3F) | utf8_continuation_byte);
case 3: *(u8destination++) = U8(((c >> 6) & 0x3F) | utf8_continuation_byte);
case 2: *(u8destination++) = U8(((c >> 0) & 0x3F) | utf8_continuation_byte);
}
}
u8_length += u8_character_length;
}
return u8_length;
}
#pragma warning( pop )
class Utf16Encoder {
public:
Utf16Encoder(const U32* codepoints, size_t codepointsLeft) :
_codepoints(codepoints), _codepointsLeft(codepointsLeft) { }
void encodeTo(U16* utf16Out) const {
for (size_t i = 0; i < _codepointsLeft; i++) {
U16_char u16c = toUtf16(_codepoints[i]);
u16c.writeTo(utf16Out);
}
}
unsigned int lengthInUtf16Words() const {
unsigned int length = 0;
for (size_t i = 0; i < _codepointsLeft; i++) {
U16_char u16c = toUtf16(_codepoints[i]);
length += u16c.length();
}
return length;
}
private:
const U32* _codepoints;
size_t _codepointsLeft;
};
u32string toUtf32(const ConstString<U8> utf8)
{
Utf8Decoder decoder(utf8.c_str(), utf8.length());
u32string u32;
u32.reserve(decoder.lengthInCharacters()); // Pre reserving requires to decode UTF-8, but reduces allocation and copy operations.
while (!decoder.isEnd()) {
u32 += decoder.decodedChar();
decoder.advanceToNext();
}
return u32;
}
u8string toUtf8(const ConstString<U32> utf32source) {
unsigned int lengthInUtf8Bytes = _toUtf8<false>(utf32source);
u8string utf8Result(lengthInUtf8Bytes, 0);
_toUtf8<true>(utf32source, stringData(utf8Result));
return utf8Result;
}
u16string toUtf16(const ConstString<U32> utf32source) {
Utf16Encoder encoder(utf32source.c_str(), utf32source.length());
u16string utf16Result(encoder.lengthInUtf16Words(), 0);
encoder.encodeTo(stringData(utf16Result));
return utf16Result;
}
unsigned int Utf8Decoder::lengthInCharacters() {
size_t bytesLeftBefore = _bytesLeft;
unsigned int byteIndexBefore = _byteIndex;
unsigned int length_in_characters = 0;
while (!isEnd()) {
length_in_characters++;
advanceToNext();
}
_bytesLeft = bytesLeftBefore;
_byteIndex = byteIndexBefore;
decode();
return length_in_characters;
}
void Utf8Decoder::decodeMultiByte() {
UCHAR first_byte = getByte(0);
if (Utf8::isFirstOfMultibyte(first_byte)) {
unsigned int character_bytes = 1 + Utf8::continuationBytes(first_byte);
const bool enough_bytes = _bytesLeft >= character_bytes;
if (!enough_bytes) {
invalidUtf8(first_byte); return;
}
_decodedChar = first_byte & _firstByteMaskForLength[character_bytes];
for (unsigned int i = 1; i < character_bytes; ++i) {
UCHAR current_byte = getByte(i);
if (!Utf8::isContinuation(current_byte)) {
invalidUtf8(first_byte); return;
}
_decodedChar = (_decodedChar << 6) | (current_byte & 0x3F);
}
if (_decodedChar < _minimumCodePointForLength[character_bytes]) {
invalidUtf8(first_byte); return;
}
_decodedLength = character_bytes;
return;
}
invalidUtf8(first_byte);
}
}