|
@@ -407,32 +407,32 @@ using ByteWriterFunction = std::function<void(uint8_t value)>;
|
|
|
using UTF32WriterFunction = std::function<void(DsrChar character)>;
|
|
using UTF32WriterFunction = std::function<void(DsrChar character)>;
|
|
|
|
|
|
|
|
// Filter out unwanted characters for improved portability
|
|
// Filter out unwanted characters for improved portability
|
|
|
-static void feedCharacter(const UTF32WriterFunction &reciever, DsrChar character) {
|
|
|
|
|
|
|
+static void feedCharacter(const UTF32WriterFunction &receiver, DsrChar character) {
|
|
|
if (character != U'\0' && character != U'\r') {
|
|
if (character != U'\0' && character != U'\r') {
|
|
|
- reciever(character);
|
|
|
|
|
|
|
+ receiver(character);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Appends the content of buffer as a BOM-free Latin-1 file into target
|
|
// Appends the content of buffer as a BOM-free Latin-1 file into target
|
|
|
// fileLength is ignored when nullTerminated is true
|
|
// fileLength is ignored when nullTerminated is true
|
|
|
template <bool nullTerminated>
|
|
template <bool nullTerminated>
|
|
|
-static void feedStringFromFileBuffer_Latin1(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength = 0) {
|
|
|
|
|
|
|
+static void feedStringFromFileBuffer_Latin1(const UTF32WriterFunction &receiver, const uint8_t* buffer, int64_t fileLength = 0) {
|
|
|
for (int64_t i = 0; i < fileLength || nullTerminated; i++) {
|
|
for (int64_t i = 0; i < fileLength || nullTerminated; i++) {
|
|
|
DsrChar character = (DsrChar)(buffer[i]);
|
|
DsrChar character = (DsrChar)(buffer[i]);
|
|
|
if (nullTerminated && character == 0) { return; }
|
|
if (nullTerminated && character == 0) { return; }
|
|
|
- feedCharacter(reciever, character);
|
|
|
|
|
|
|
+ feedCharacter(receiver, character);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
// Appends the content of buffer as a BOM-free UTF-8 file into target
|
|
// Appends the content of buffer as a BOM-free UTF-8 file into target
|
|
|
// fileLength is ignored when nullTerminated is true
|
|
// fileLength is ignored when nullTerminated is true
|
|
|
template <bool nullTerminated>
|
|
template <bool nullTerminated>
|
|
|
-static void feedStringFromFileBuffer_UTF8(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength = 0) {
|
|
|
|
|
|
|
+static void feedStringFromFileBuffer_UTF8(const UTF32WriterFunction &receiver, const uint8_t* buffer, int64_t fileLength = 0) {
|
|
|
for (int64_t i = 0; i < fileLength || nullTerminated; i++) {
|
|
for (int64_t i = 0; i < fileLength || nullTerminated; i++) {
|
|
|
uint8_t byteA = buffer[i];
|
|
uint8_t byteA = buffer[i];
|
|
|
if (byteA < (uint32_t)0b10000000) {
|
|
if (byteA < (uint32_t)0b10000000) {
|
|
|
// Single byte (1xxxxxxx)
|
|
// Single byte (1xxxxxxx)
|
|
|
if (nullTerminated && byteA == 0) { return; }
|
|
if (nullTerminated && byteA == 0) { return; }
|
|
|
- feedCharacter(reciever, (DsrChar)byteA);
|
|
|
|
|
|
|
+ feedCharacter(receiver, (DsrChar)byteA);
|
|
|
} else {
|
|
} else {
|
|
|
uint32_t character = 0;
|
|
uint32_t character = 0;
|
|
|
int extraBytes = 0;
|
|
int extraBytes = 0;
|
|
@@ -459,7 +459,7 @@ static void feedStringFromFileBuffer_UTF8(const UTF32WriterFunction &reciever, c
|
|
|
character = (character << 6) | (nextByte & 0b00111111);
|
|
character = (character << 6) | (nextByte & 0b00111111);
|
|
|
extraBytes--;
|
|
extraBytes--;
|
|
|
}
|
|
}
|
|
|
- feedCharacter(reciever, (DsrChar)character);
|
|
|
|
|
|
|
+ feedCharacter(receiver, (DsrChar)character);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -478,7 +478,7 @@ uint16_t read16bits(const uint8_t* buffer, int64_t startOffset) {
|
|
|
// Appends the content of buffer as a BOM-free UTF-16 file into target as UTF-32
|
|
// Appends the content of buffer as a BOM-free UTF-16 file into target as UTF-32
|
|
|
// fileLength is ignored when nullTerminated is true
|
|
// fileLength is ignored when nullTerminated is true
|
|
|
template <bool LittleEndian, bool nullTerminated>
|
|
template <bool LittleEndian, bool nullTerminated>
|
|
|
-static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength = 0) {
|
|
|
|
|
|
|
+static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &receiver, const uint8_t* buffer, int64_t fileLength = 0) {
|
|
|
for (int64_t i = 0; i < fileLength || nullTerminated; i += 2) {
|
|
for (int64_t i = 0; i < fileLength || nullTerminated; i += 2) {
|
|
|
// Read the first 16-bit word
|
|
// Read the first 16-bit word
|
|
|
uint16_t wordA = read16bits<LittleEndian>(buffer, i);
|
|
uint16_t wordA = read16bits<LittleEndian>(buffer, i);
|
|
@@ -488,7 +488,7 @@ static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever,
|
|
|
if (wordA <= 0xD7FF || wordA >= 0xE000) {
|
|
if (wordA <= 0xD7FF || wordA >= 0xE000) {
|
|
|
// Not in the reserved range, just a single 16-bit character
|
|
// Not in the reserved range, just a single 16-bit character
|
|
|
if (nullTerminated && wordA == 0) { return; }
|
|
if (nullTerminated && wordA == 0) { return; }
|
|
|
- feedCharacter(reciever, (DsrChar)wordA);
|
|
|
|
|
|
|
+ feedCharacter(receiver, (DsrChar)wordA);
|
|
|
} else {
|
|
} else {
|
|
|
// The given range was reserved and therefore using 32 bits
|
|
// The given range was reserved and therefore using 32 bits
|
|
|
i += 2;
|
|
i += 2;
|
|
@@ -496,20 +496,20 @@ static void feedStringFromFileBuffer_UTF16(const UTF32WriterFunction &reciever,
|
|
|
uint32_t higher10Bits = wordA & (uint32_t)0b1111111111;
|
|
uint32_t higher10Bits = wordA & (uint32_t)0b1111111111;
|
|
|
uint32_t lower10Bits = wordB & (uint32_t)0b1111111111;
|
|
uint32_t lower10Bits = wordB & (uint32_t)0b1111111111;
|
|
|
DsrChar finalChar = (DsrChar)(((higher10Bits << 10) | lower10Bits) + (uint32_t)0x10000);
|
|
DsrChar finalChar = (DsrChar)(((higher10Bits << 10) | lower10Bits) + (uint32_t)0x10000);
|
|
|
- feedCharacter(reciever, finalChar);
|
|
|
|
|
|
|
+ feedCharacter(receiver, finalChar);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
// Sends the decoded UTF-32 characters from the encoded buffer into target.
|
|
// Sends the decoded UTF-32 characters from the encoded buffer into target.
|
|
|
// The text encoding should be specified using a BOM at the start of buffer, otherwise Latin-1 is assumed.
|
|
// The text encoding should be specified using a BOM at the start of buffer, otherwise Latin-1 is assumed.
|
|
|
-static void feedStringFromFileBuffer(const UTF32WriterFunction &reciever, const uint8_t* buffer, int64_t fileLength) {
|
|
|
|
|
|
|
+static void feedStringFromFileBuffer(const UTF32WriterFunction &receiver, const uint8_t* buffer, int64_t fileLength) {
|
|
|
// After removing the BOM bytes, the rest can be seen as a BOM-free text file with a known format
|
|
// After removing the BOM bytes, the rest can be seen as a BOM-free text file with a known format
|
|
|
if (fileLength >= 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) { // UTF-8
|
|
if (fileLength >= 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) { // UTF-8
|
|
|
- feedStringFromFileBuffer_UTF8<false>(reciever, buffer + 3, fileLength - 3);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_UTF8<false>(receiver, buffer + 3, fileLength - 3);
|
|
|
} else if (fileLength >= 2 && buffer[0] == 0xFE && buffer[1] == 0xFF) { // UTF-16 BE
|
|
} else if (fileLength >= 2 && buffer[0] == 0xFE && buffer[1] == 0xFF) { // UTF-16 BE
|
|
|
- feedStringFromFileBuffer_UTF16<false, false>(reciever, buffer + 2, fileLength - 2);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_UTF16<false, false>(receiver, buffer + 2, fileLength - 2);
|
|
|
} else if (fileLength >= 2 && buffer[0] == 0xFF && buffer[1] == 0xFE) { // UTF-16 LE
|
|
} else if (fileLength >= 2 && buffer[0] == 0xFF && buffer[1] == 0xFE) { // UTF-16 LE
|
|
|
- feedStringFromFileBuffer_UTF16<true, false>(reciever, buffer + 2, fileLength - 2);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_UTF16<true, false>(receiver, buffer + 2, fileLength - 2);
|
|
|
} else if (fileLength >= 4 && buffer[0] == 0x00 && buffer[1] == 0x00 && buffer[2] == 0xFE && buffer[3] == 0xFF) { // UTF-32 BE
|
|
} else if (fileLength >= 4 && buffer[0] == 0x00 && buffer[1] == 0x00 && buffer[2] == 0xFE && buffer[3] == 0xFF) { // UTF-32 BE
|
|
|
//feedStringFromFileBuffer_UTF32BE(receiver, buffer + 4, fileLength - 4);
|
|
//feedStringFromFileBuffer_UTF32BE(receiver, buffer + 4, fileLength - 4);
|
|
|
throwError(U"UTF-32 BE format is not yet supported!\n");
|
|
throwError(U"UTF-32 BE format is not yet supported!\n");
|
|
@@ -530,21 +530,21 @@ static void feedStringFromFileBuffer(const UTF32WriterFunction &reciever, const
|
|
|
throwError(U"UTF-7 format is not yet supported!\n");
|
|
throwError(U"UTF-7 format is not yet supported!\n");
|
|
|
} else {
|
|
} else {
|
|
|
// No BOM detected, assuming Latin-1 (because it directly corresponds to a unicode sub-set)
|
|
// No BOM detected, assuming Latin-1 (because it directly corresponds to a unicode sub-set)
|
|
|
- feedStringFromFileBuffer_Latin1<false>(reciever, buffer, fileLength);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_Latin1<false>(receiver, buffer, fileLength);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Sends the decoded UTF-32 characters from the encoded null terminated buffer into target.
|
|
// Sends the decoded UTF-32 characters from the encoded null terminated buffer into target.
|
|
|
// buffer may not contain any BOM, and must be null terminated in the specified encoding.
|
|
// buffer may not contain any BOM, and must be null terminated in the specified encoding.
|
|
|
-static void feedStringFromRawData(const UTF32WriterFunction &reciever, const uint8_t* buffer, CharacterEncoding encoding) {
|
|
|
|
|
|
|
+static void feedStringFromRawData(const UTF32WriterFunction &receiver, const uint8_t* buffer, CharacterEncoding encoding) {
|
|
|
if (encoding == CharacterEncoding::Raw_Latin1) {
|
|
if (encoding == CharacterEncoding::Raw_Latin1) {
|
|
|
- feedStringFromFileBuffer_Latin1<true>(reciever, buffer);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_Latin1<true>(receiver, buffer);
|
|
|
} else if (encoding == CharacterEncoding::BOM_UTF8) {
|
|
} else if (encoding == CharacterEncoding::BOM_UTF8) {
|
|
|
- feedStringFromFileBuffer_UTF8<true>(reciever, buffer);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_UTF8<true>(receiver, buffer);
|
|
|
} else if (encoding == CharacterEncoding::BOM_UTF16BE) {
|
|
} else if (encoding == CharacterEncoding::BOM_UTF16BE) {
|
|
|
- feedStringFromFileBuffer_UTF16<false, true>(reciever, buffer);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_UTF16<false, true>(receiver, buffer);
|
|
|
} else if (encoding == CharacterEncoding::BOM_UTF16LE) {
|
|
} else if (encoding == CharacterEncoding::BOM_UTF16LE) {
|
|
|
- feedStringFromFileBuffer_UTF16<true, true>(reciever, buffer);
|
|
|
|
|
|
|
+ feedStringFromFileBuffer_UTF16<true, true>(receiver, buffer);
|
|
|
} else {
|
|
} else {
|
|
|
throwError("Unhandled encoding in feedStringFromRawData!\n");
|
|
throwError("Unhandled encoding in feedStringFromRawData!\n");
|
|
|
}
|
|
}
|
|
@@ -561,10 +561,10 @@ String dsr::string_dangerous_decodeFromData(const void* data, CharacterEncoding
|
|
|
// Pre-allocate the correct amount of memory based on the simulation
|
|
// Pre-allocate the correct amount of memory based on the simulation
|
|
|
string_reserve(result, characterCount);
|
|
string_reserve(result, characterCount);
|
|
|
// Stream output to the result string
|
|
// Stream output to the result string
|
|
|
- UTF32WriterFunction reciever = [&result](DsrChar character) {
|
|
|
|
|
|
|
+ UTF32WriterFunction receiver = [&result](DsrChar character) {
|
|
|
string_appendChar(result, character);
|
|
string_appendChar(result, character);
|
|
|
};
|
|
};
|
|
|
- feedStringFromRawData(reciever, (const uint8_t*)data, encoding);
|
|
|
|
|
|
|
+ feedStringFromRawData(receiver, (const uint8_t*)data, encoding);
|
|
|
return result;
|
|
return result;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -579,10 +579,10 @@ String dsr::string_loadFromMemory(Buffer fileContent) {
|
|
|
// Pre-allocate the correct amount of memory based on the simulation
|
|
// Pre-allocate the correct amount of memory based on the simulation
|
|
|
string_reserve(result, characterCount);
|
|
string_reserve(result, characterCount);
|
|
|
// Stream output to the result string
|
|
// Stream output to the result string
|
|
|
- UTF32WriterFunction reciever = [&result](DsrChar character) {
|
|
|
|
|
|
|
+ UTF32WriterFunction receiver = [&result](DsrChar character) {
|
|
|
string_appendChar(result, character);
|
|
string_appendChar(result, character);
|
|
|
};
|
|
};
|
|
|
- feedStringFromFileBuffer(reciever, buffer_dangerous_getUnsafeData(fileContent), buffer_getSize(fileContent));
|
|
|
|
|
|
|
+ feedStringFromFileBuffer(receiver, buffer_dangerous_getUnsafeData(fileContent), buffer_getSize(fileContent));
|
|
|
return result;
|
|
return result;
|
|
|
}
|
|
}
|
|
|
|
|
|