[i18n] Add tests.

core/text/i18n/doc.odin

@@ -0,0 +1,111 @@
+//+ignore
+package i18n
+
+/*
+	The i18n package is flexible and easy to use.
+
+	It has one call to get a translation: `get`, which the user can alias into something like `T`.
+
+	`get`, referred to as `T` here, has a few different signatures.
+	All of them will return the key if the entry can't be found in the active translation catalog.
+
+	- `T(key)`              returns the translation of `key`.
+	- `T(key, n)`           returns a pluralized translation of `key` according to value `n`.
+
+	- `T(section, key)`     returns the translation of `key` in `section`.
+	- `T(section, key, n)`  returns a pluralized translation of `key` in `section` according to value `n`.
+
+	By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use.
+	If you want to override which translation to use, for example in a language preview dialog, you can use the following:
+
+	- `T(key, n, catalog)`           returns the pluralized version of `key` from explictly supplied catalog.
+	- `T(section, key, n, catalog)`  returns the pluralized version of `key` in `section` from explictly supplied catalog.
+
+	If a catalog has translation contexts or sections, then ommitting it in the above calls looks up in section "".
+
+	The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form.
+	Passing n != 1 returns plural form 1.
+
+	Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser.
+	This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used.
+
+	You can also assign it to a loaded catalog after parsing, of course.
+
+	Some code examples follow.
+*/
+
+/*
+```cpp
+import "core:fmt"
+import "core:text/i18n"
+
+T :: i18n.get
+
+mo :: proc() {
+	using fmt
+
+	err: i18n.Error
+
+	/*
+		Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter.
+	*/
+	i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo"))
+	defer i18n.destroy()
+
+	if err != .None { return }
+
+	/*
+		These are in the .MO catalog.
+	*/
+	println("-----")
+	println(T(""))
+	println("-----")
+	println(T("There are 69,105 leaves here."))
+	println("-----")
+	println(T("Hellope, World!"))
+	println("-----")
+	// We pass 1 into `T` to get the singular format string, then 1 again into printf.
+	printf(T("There is %d leaf.\n", 1), 1)
+	// We pass 42 into `T` to get the plural format string, then 42 again into printf.
+	printf(T("There is %d leaf.\n", 42), 42)
+
+	/*
+		This isn't in the translation catalog, so the key is passed back untranslated.
+	*/
+	println("-----")
+	println(T("Come visit us on Discord!"))
+}
+
+qt :: proc() {
+	using fmt
+
+	err: i18n.Error
+
+	/*
+		Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter.
+	*/
+	i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts"))
+	defer i18n.destroy()
+
+	if err != .None {
+		return
+	}
+
+	/*
+		These are in the .TS catalog. As you can see they have sections.
+	*/
+	println("--- Page section ---")
+	println("Page:Text for translation =", T("Page", "Text for translation"))
+	println("-----")
+	println("Page:Also text to translate =", T("Page", "Also text to translate"))
+	println("-----")
+	println("--- installscript section ---")
+	println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall"))
+	println("-----")
+	println("--- apple_count section ---")
+	println("apple_count:%d apple(s) =")
+	println("\t 1  =", T("apple_count", "%d apple(s)", 1))
+	println("\t 42 =", T("apple_count", "%d apple(s)", 42))
+}
+```
+*/

core/text/i18n/example/i18n_example.odin

@@ -1,99 +0,0 @@
-package i18n_example
-
-import "core:mem"
-import "core:fmt"
-import "core:text/i18n"
-
-_T :: i18n.get
-
-mo :: proc() {
-	using fmt
-
-	err: i18n.Error
-
-	/*
-		Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter.
-	*/
-	i18n.ACTIVE, err = i18n.parse_mo(#load("nl_NL.mo"))
-	defer i18n.destroy()
-
-	if err != .None { return }
-
-	/*
-		These are in the .MO catalog.
-	*/
-	println("-----")
-	println(_T(""))
-	println("-----")
-	println(_T("There are 69,105 leaves here."))
-	println("-----")
-	println(_T("Hellope, World!"))
-
-	/*
-		For ease of use, pluralized lookup can use both singular and plural form as key for the same translation.
-		This is a quirk of the GetText format which has separate keys for their different plurals.
-	*/
-	println("-----")
-	printf(_T("There is %d leaf.\n", 1), 1)
-	printf(_T("There is %d leaf.\n", 42), 42)
-
-	printf(_T("There are %d leaves.\n", 1), 1)
-	printf(_T("There are %d leaves.\n", 42), 42)
-
-	/*
-		This isn't.
-	*/
-	println("-----")
-	println(_T("Come visit us on Discord!"))
-}
-
-qt :: proc() {
-	using fmt
-
-	err: i18n.Error
-
-	/*
-		Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter.
-	*/
-	i18n.ACTIVE, err = i18n.parse_qt(#load("../../../../tests/core/assets/XML/nl_NL-qt-ts.ts"))
-	defer i18n.destroy()
-
-	fmt.printf("parse_qt returned %v\n", err)
-	if err != .None {
-		return
-	}
-
-	/*
-		These are in the .TS catalog.
-	*/
-	println("--- Page section ---")
-	println("Page:Text for translation =", _T("Page", "Text for translation"))
-	println("-----")
-	println("Page:Also text to translate =", _T("Page", "Also text to translate"))
-	println("-----")
-	println("--- installscript section ---")
-	println("installscript:99 bottles of beer on the wall =", _T("installscript", "99 bottles of beer on the wall"))
-	println("-----")
-	println("--- apple_count section ---")
-	println("apple_count:%d apple(s) =")
-	println("\t 1  =", _T("apple_count", "%d apple(s)", 1))
-	println("\t 42 =", _T("apple_count", "%d apple(s)", 42))
-}
-
-main :: proc() {
-	using fmt
-
-	track: mem.Tracking_Allocator
-	mem.tracking_allocator_init(&track, context.allocator)
-	context.allocator = mem.tracking_allocator(&track)
-
-	// mo()
-	qt()
-
-	if len(track.allocation_map) > 0 {
-		println()
-		for _, v in track.allocation_map {
-			printf("%v Leaked %v bytes.\n", v.location, v.size)
-		}
-	}
-}

core/text/i18n/gettext.odin

@@ -8,6 +8,9 @@ package i18n
 	A from-scratch implementation based after the specification found here:
 		https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html
 
+	Options are ignored as they're not applicable to this format.
+	They're part of the signature for consistency with other catalog formats.
+
 	List of contributors:
 		Jeroen van Rijn: Initial implementation.
 */
@@ -15,7 +18,7 @@ import "core:os"
 import "core:strings"
 import "core:bytes"
 
-parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+parse_mo_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
 	context.allocator = allocator
 	/*
 		An MO file should have at least a 4-byte magic, 2 x 2 byte version info,
@@ -115,7 +118,7 @@ parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allo
 	return
 }
 
-parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+parse_mo_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
 	context.allocator = allocator
 
 	data, data_ok := os.read_entire_file(filename)
@@ -123,7 +126,7 @@ parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allo
 
 	if !data_ok { return {}, .File_Error }
 
-	return parse_mo_from_slice(data, pluralizer)
+	return parse_mo_from_slice(data, options, pluralizer, allocator)
 }
 
 parse_mo :: proc { parse_mo_file, parse_mo_from_slice }

core/text/i18n/i18n.odin

@@ -74,6 +74,14 @@ Error :: enum {
 
 }
 
+Parse_Options :: struct {
+	merge_sections: bool,
+}
+
+DEFAULT_PARSE_OPTIONS :: Parse_Options{
+	merge_sections = false,
+}
+
 /*
 	Several ways to use:
 	- get(key), which defaults to the singular form and i18n.ACTIVE catalog, or

core/text/i18n/qt_linguist.odin

@@ -27,7 +27,7 @@ TS_XML_Options := xml.Options{
 	expected_doctype = "TS",
 }
 
-parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
 	context.allocator = allocator
 
 	ts, xml_err := xml.parse(data, TS_XML_Options)
@@ -59,7 +59,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int =
 			return translation, .TS_File_Expected_Context_Name,
 		}
 
-		section_name := ts.elements[section_name_id].value
+		section_name := "" if options.merge_sections else ts.elements[section_name_id].value
 
 		if section_name not_in translation.k_v {
 			translation.k_v[section_name] = {}
@@ -139,7 +139,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int =
 	return
 }
 
-parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+parse_qt_linguist_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
 	context.allocator = allocator
 
 	data, data_ok := os.read_entire_file(filename)
@@ -147,7 +147,7 @@ parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int =
 
 	if !data_ok { return {}, .File_Error }
 
-	return parse_qt_linguist_from_slice(data, pluralizer)
+	return parse_qt_linguist_from_slice(data, options, pluralizer, allocator)
 }
 
 parse_qt :: proc { parse_qt_linguist_file, parse_qt_linguist_from_slice }

examples/all/all_main.odin

@@ -56,6 +56,7 @@ import csv            "core:encoding/csv"
 import hxa            "core:encoding/hxa"
 import json           "core:encoding/json"
 import varint         "core:encoding/varint"
+import xml            "core:encoding/xml"
 
 import fmt            "core:fmt"
 import hash           "core:hash"
@@ -100,6 +101,7 @@ import strings        "core:strings"
 import sync           "core:sync"
 import testing        "core:testing"
 import scanner        "core:text/scanner"
+import i18n           "core:text/i18n"
 import thread         "core:thread"
 import time           "core:time"
 
@@ -158,6 +160,7 @@ _ :: csv
 _ :: hxa
 _ :: json
 _ :: varint
+_ :: xml
 _ :: fmt
 _ :: hash
 _ :: image
@@ -192,6 +195,7 @@ _ :: strings
 _ :: sync
 _ :: testing
 _ :: scanner
+_ :: i18n
 _ :: thread
 _ :: time
 _ :: unicode

tests/core/Makefile

@@ -26,9 +26,10 @@ noise_test:
 	$(ODIN) run math/noise -out:test_noise
 
 encoding_test:
-	$(ODIN) run encoding/hxa -collection:tests=.. -out:test_hxa
-	$(ODIN) run encoding/json -out:test_json
+	$(ODIN) run encoding/hxa    -out:test_hxa -collection:tests=..
+	$(ODIN) run encoding/json   -out:test_json
 	$(ODIN) run encoding/varint -out:test_varint
+	$(ODIN) run encoding/xml    -out:test_xml
 
 math_test:
 	$(ODIN) run math/test_core_math.odin -file -collection:tests=.. -out:test_core_math

tests/core/assets/I18N/duplicate-key.ts

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.1" language="nl" sourcelanguage="en">
+<context>
+	<name>Page</name>
+	<message>
+		<source>%d apple(s)</source>
+		<comment>commenting</comment>
+		<translation type="obsolete">Tekst om te vertalen</translation>
+	</message>
+</context>
+<context>
+		<name>apple_count</name>
+		<message numerus="yes">
+			<source>%d apple(s)</source>
+			<translation>
+				<numerusform>%d appel</numerusform>
+				<numerusform>%d appels</numerusform>
+			</translation>
+		</message>
+	</context>
+</TS>

tests/core/build.bat

@@ -64,4 +64,9 @@ echo ---
 echo ---
 echo Running core:reflect tests
 echo ---
-%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe
+%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe
+
+echo ---
+echo Running core:text/i18n tests
+echo ---
+%PATH_TO_ODIN% run text\i18n %COMMON% -out:test_core_i18n.exe

tests/core/text/i18n/test_core_text_i18n.odin

@@ -0,0 +1,165 @@
+package test_core_text_i18n
+
+import "core:mem"
+import "core:fmt"
+import "core:os"
+import "core:testing"
+import "core:text/i18n"
+
+TEST_count := 0
+TEST_fail  := 0
+
+when ODIN_TEST {
+	expect  :: testing.expect
+	log     :: testing.log
+} else {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+		TEST_count += 1
+		if !condition {
+			TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
+			return
+		}
+	}
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("[%v] ", loc)
+		fmt.printf("log: %v\n", v)
+	}
+}
+T :: i18n.get
+
+Test :: struct {
+	section: string,
+	key:     string,
+	val:     string,
+	n:       int,
+}
+
+Test_Suite :: struct {
+	file:    string,
+	loader:  proc(string, i18n.Parse_Options, proc(int) -> int, mem.Allocator) -> (^i18n.Translation, i18n.Error),
+	err:     i18n.Error,
+	options: i18n.Parse_Options,
+	tests:   []Test,
+}
+
+TESTS := []Test_Suite{
+	{
+		file   = "assets/I18N/nl_NL.mo",
+		loader = i18n.parse_mo_file,
+		tests  = {
+			// These are in the catalog.
+			{ "", "There are 69,105 leaves here.", "Er zijn hier 69.105 bladeren.",  1 },
+			{ "", "Hellope, World!",               "Hallo, Wereld!",                 1 },
+			{ "", "There is %d leaf.\n",           "Er is %d blad.\n",               1 },
+			{ "", "There are %d leaves.\n",        "Er is %d blad.\n",               1 },
+			{ "", "There is %d leaf.\n",           "Er zijn %d bladeren.\n",        42 },
+			{ "", "There are %d leaves.\n",        "Er zijn %d bladeren.\n",        42 },
+
+			// This isn't in the catalog, so should ruturn the key.
+			{ "", "Come visit us on Discord!",     "Come visit us on Discord!",      1 },
+		},
+	},
+
+	// QT Linguist with default loader options.
+	{
+		file   = "assets/I18N/nl_NL-qt-ts.ts",
+		loader = i18n.parse_qt_linguist_file,
+		tests  = {
+			// These are in the catalog.
+			{ "Page",          "Text for translation",           "Tekst om te vertalen",        1},
+			{ "Page",          "Also text to translate",         "Ook tekst om te vertalen",    1},
+			{ "installscript", "99 bottles of beer on the wall", "99 flessen bier op de muur",  1},
+			{ "apple_count",   "%d apple(s)",                    "%d appel",                    1},
+			{ "apple_count",   "%d apple(s)",                    "%d appels",                  42},
+
+			// These aren't in the catalog, so should ruturn the key.
+			{ "",              "Come visit us on Discord!",      "Come visit us on Discord!",  1 },
+			{ "Fake_Section",  "Come visit us on Discord!",      "Come visit us on Discord!",  1 },
+		},
+	},
+
+	// QT Linguist, merging sections.
+	{
+		file    = "assets/I18N/nl_NL-qt-ts.ts",
+		loader  = i18n.parse_qt_linguist_file,
+		options = {merge_sections = true},
+		tests   = {
+			// All of them are now in section "", lookup with original section should return the key.
+			{ "",              "Text for translation",           "Tekst om te vertalen",        1},
+			{ "",              "Also text to translate",         "Ook tekst om te vertalen",    1},
+			{ "",              "99 bottles of beer on the wall", "99 flessen bier op de muur",  1},
+			{ "",              "%d apple(s)",                    "%d appel",                    1},
+			{ "",              "%d apple(s)",                    "%d appels",                  42},
+
+			// All of them are now in section "", lookup with original section should return the key.
+			{ "Page",          "Text for translation",           "Text for translation",            1},
+			{ "Page",          "Also text to translate",         "Also text to translate",          1},
+			{ "installscript", "99 bottles of beer on the wall", "99 bottles of beer on the wall",  1},
+			{ "apple_count",   "%d apple(s)",                    "%d apple(s)",                     1},
+			{ "apple_count",   "%d apple(s)",                    "%d apple(s)",                    42},
+		},
+	},
+
+	// QT Linguist, merging sections. Expecting .Duplicate_Key error because same key exists in more than 1 section.
+	{
+		file    = "assets/I18N/duplicate-key.ts",
+		loader  = i18n.parse_qt_linguist_file,
+		options = {merge_sections = true},
+		err     = .Duplicate_Key,
+	},
+
+	// QT Linguist, not merging sections. Shouldn't return error despite same key existing in more than 1 section.
+	{
+		file    = "assets/I18N/duplicate-key.ts",
+		loader  = i18n.parse_qt_linguist_file,
+	},
+}
+
+@test
+tests :: proc(t: ^testing.T) {
+	using fmt
+
+	cat: ^i18n.Translation
+	err: i18n.Error
+
+	for suite in TESTS {
+		cat, err = suite.loader(suite.file, suite.options, nil, context.allocator)
+
+		msg := fmt.tprintf("Expected loading %v to return %v, got %v", suite.file, suite.err, err)
+		expect(t, err == suite.err, msg)
+
+		if err == .None {
+			for test in suite.tests {
+				val := T(test.section, test.key, test.n, cat)
+
+				msg  = fmt.tprintf("Expected key `%v` from section `%v`'s form for value `%v` to equal `%v`, got `%v`", test.key, test.section, test.n, test.val, val)
+				expect(t, val == test.val, msg)
+			}
+		}
+		i18n.destroy(cat)
+	}
+}
+
+main :: proc() {
+	using fmt
+
+	track: mem.Tracking_Allocator
+	mem.tracking_allocator_init(&track, context.allocator)
+	context.allocator = mem.tracking_allocator(&track)
+
+	t := testing.T{}
+	tests(&t)
+
+	fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+	if TEST_fail > 0 {
+		os.exit(1)
+	}
+
+	if len(track.allocation_map) > 0 {
+		println()
+		for _, v in track.allocation_map {
+			printf("%v Leaked %v bytes.\n", v.location, v.size)
+		}
+	}
+}