Merge pull request #124 from tsoding/utf8-support

Initial support for unicode

.github/workflows/ci.yml

@@ -27,8 +27,12 @@ jobs:
           make -B
           ./emote_downloader
           make render
-          cd ./test/pajaWalk/
+          pushd ./test/pajaWalk/
           ./test.sh
+          popd
+          pushd ./test/utf8/
+          ./test.sh
+          popd
         env:
           CC: gcc
           CXX: g++
@@ -61,8 +65,12 @@ jobs:
           make -B
           ./emote_downloader
           make render
-          cd ./test/pajaWalk/
+          pushd ./test/pajaWalk/
+          ./test.sh
+          popd
+          pushd ./test/utf8/
           ./test.sh
+          popd
         env:
           CC: clang
           CXX: clang++

src/aids.hpp

@@ -21,7 +21,7 @@
 //
 // ============================================================
 //
-// aids — 0.12.1 — std replacement for C++. Designed to aid developers
+// aids — 0.13.0 — std replacement for C++. Designed to aid developers
 // to a better programming experience.
 //
 // https://github.com/rexim/aids
@@ -30,6 +30,8 @@
 //
 // ChangeLog (https://semver.org/ is implied)
 //
+//   0.13.0 void print1(FILE *stream, unsigned int x)
+//          Maybe<uint32_t> utf8_get_code(String_View view, size_t *size)
 //   0.12.1 Fix print1 and sprint1 bug for unsigned long long
 //   0.12.0 void print1(FILE *stream, String_Buffer buffer)
 //          void sprint1(String_Buffer *buffer, String_Buffer another_buffer)
@@ -634,6 +636,11 @@ namespace aids
         fprintf(stream, "%lu", x);
     }
 
+    void print1(FILE *stream, unsigned int x)
+    {
+        fprintf(stream, "%u", x);
+    }
+
     void print1(FILE *stream, int x)
     {
         fprintf(stream, "%d", x);
@@ -690,6 +697,67 @@ namespace aids
     {
         print1(stream, buffer.view());
     }
+
+    ////////////////////////////////////////////////////////////
+    // UTF-8
+    ////////////////////////////////////////////////////////////
+
+    Maybe<uint32_t> utf8_get_code(String_View view, size_t *size)
+    {
+#define UTF8_1BYTE_MASK (1 << 7)
+#define UTF8_2BYTES_MASK (1 << 5)
+#define UTF8_3BYTES_MASK (1 << 4)
+#define UTF8_4BYTES_MASK (1 << 3)
+#define UTF8_EXTRA_BYTE_MASK (1 << 6)
+            if (view.count >= 1 &&
+                (*view.data & UTF8_1BYTE_MASK) == 0)
+            {
+                *size = 1;
+                return {true, static_cast<uint32_t>(*view.data)};
+            }
+            if (view.count >= 2 &&
+                (view.data[0] & UTF8_2BYTES_MASK) == 0 &&
+                (view.data[1] & UTF8_EXTRA_BYTE_MASK) == 0)
+            {
+                *size = 2;
+                const auto byte1 = static_cast<uint32_t>((view.data[0] & (UTF8_2BYTES_MASK - 1)) << 6);
+                const auto byte2 = static_cast<uint32_t>(view.data[1] & (UTF8_EXTRA_BYTE_MASK - 1));
+                return {true, byte1 | byte2};
+            }
+            if (view.count >= 3 &&
+                (view.data[0] & UTF8_3BYTES_MASK) == 0 &&
+                (view.data[1] & UTF8_EXTRA_BYTE_MASK) == 0 &&
+                (view.data[2] & UTF8_EXTRA_BYTE_MASK) == 0)
+            {
+                *size = 3;
+                const auto byte1 = static_cast<uint32_t>((view.data[0] & (UTF8_3BYTES_MASK - 1)) << (6 * 2));
+                const auto byte2 = static_cast<uint32_t>((view.data[1] & (UTF8_EXTRA_BYTE_MASK - 1)) << 6);
+                const auto byte3 = static_cast<uint32_t>(view.data[2] & (UTF8_EXTRA_BYTE_MASK - 1));
+                return {true, byte1 | byte2 | byte3};
+            }
+            if (view.count >= 4 &&
+                (view.data[0] & UTF8_4BYTES_MASK) == 0 &&
+                (view.data[1] & UTF8_EXTRA_BYTE_MASK) == 0 &&
+                (view.data[2] & UTF8_EXTRA_BYTE_MASK) == 0 &&
+                (view.data[3] & UTF8_EXTRA_BYTE_MASK) == 0)
+            {
+                *size = 4;
+                const auto byte1 = static_cast<uint32_t>((view.data[0] & (UTF8_3BYTES_MASK - 1)) << (6 * 3));
+                const auto byte2 = static_cast<uint32_t>((view.data[1] & (UTF8_EXTRA_BYTE_MASK - 1)) << (6 * 2));
+                const auto byte3 = static_cast<uint32_t>((view.data[2] & (UTF8_EXTRA_BYTE_MASK - 1)) << 6);
+                const auto byte4 = static_cast<uint32_t>(view.data[3] & (UTF8_EXTRA_BYTE_MASK - 1));
+                return {true, byte1 | byte2 | byte3 | byte4};
+            }
+            else
+            {
+                return {};
+            }
+#undef UTF8_1BYTE_MASK
+#undef UTF8_2BYTES_MASK
+#undef UTF8_3BYTES_MASK
+#undef UTF8_4BYTES_MASK
+#undef UTF8_EXTRA_BYTE_MASK
+    }
 }
 
 #endif  // AIDS_HPP_

src/vodus_image32.cpp

@@ -399,8 +399,17 @@ void slap_text_onto_image32(Image32 surface,
                             Pixel32 color,
                             int *pen_x, int *pen_y)
 {
-    for (size_t i = 0; i < text.count; ++i) {
-        FT_UInt glyph_index = FT_Get_Char_Index(face, text.data[i]);
+    String_View iter = text;
+
+    while (iter.count > 0) {
+        size_t size = 0;
+        auto code = utf8_get_code(iter, &size);
+        if (!code.has_value) {
+            println(stderr, "`", text, "` is not a correct UTF-8 sequence");
+            abort();
+        }
+        FT_UInt glyph_index = FT_Get_Char_Index(face, code.unwrap);
+        iter.chop(size);
 
         auto error = FT_Load_Glyph(face, glyph_index, FT_LOAD_DEFAULT);
         assert(!error);

test/utf8/.gitignore

@@ -0,0 +1 @@
+actual-frames/

test/utf8/test.conf

@@ -0,0 +1,8 @@
+output-type = png
+font = ./fonts-japanese-gothic.ttf
+font_size = 10
+fps = 15
+width = 200
+height = 100
+messages_limit = 20
+bitrate = 6000000

test/utf8/test.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+
+VODUS=../../vodus.release
+DIFFIMG=../../diffimg
+
+set -xe
+
+rm -rf ./actual-frames/
+mkdir -p ./actual-frames/
+$VODUS ./utf-8.txt actual-frames/ --config ./test.conf
+
+EXPECTED_COUNT=`ls ./expected-frames/ | wc -l`
+ACTUAL_COUNT=`ls ./actual-frames/ | wc -l`
+
+if [ "$EXPECTED_COUNT" != "$ACTUAL_COUNT" ]; then
+    echo "UNEXPECTED AMOUNT OF FRAMES!"
+    echo "Expected: $EXPECTED_COUNT"
+    echo "Actual:   $ACTUAL_COUNT"
+    exit 1
+fi
+
+for frame in `ls ./expected-frames/`; do
+    $DIFFIMG -e "./expected-frames/$frame" -a "./actual-frames/$frame" -t 2
+done

test/utf8/utf-8.txt

@@ -0,0 +1,3 @@
+[0:00:00] <Tsoding> Hello, World!
+[0:00:01] <Tsoding> Привет, Мир!
+[0:00:02] <Tsoding> こんにちは世界!