utils: add unicode utils from Inoussa (part of mantis #0022909)

git-svn-id: trunk@23748 -

.gitattributes

@@ -14520,3 +14520,20 @@ utils/tply/yaccsem.pas svneol=native#text/plain
 utils/tply/yacctabl.pas svneol=native#text/plain
 utils/tply/yylex.cod svneol=native#text/plain
 utils/tply/yyparse.cod svneol=native#text/plain
+utils/unicode/Makefile svneol=native#text/plain
+utils/unicode/Makefile.fpc svneol=native#text/plain
+utils/unicode/cldrhelper.pas svneol=native#text/pascal
+utils/unicode/cldrparser.lpi svneol=native#text/plain
+utils/unicode/cldrparser.lpr svneol=native#text/pascal
+utils/unicode/cldrtest.pas svneol=native#text/pascal
+utils/unicode/cldrxml.pas svneol=native#text/pascal
+utils/unicode/data/readme.txt svneol=native#text/plain
+utils/unicode/grbtree.pas svneol=native#text/pascal
+utils/unicode/helper.pas svneol=native#text/pascal
+utils/unicode/parse-collations.bat svneol=native#text/plain
+utils/unicode/trie.pas svneol=native#text/pascal
+utils/unicode/uca_test.pas svneol=native#text/pascal
+utils/unicode/unicodeset.pas svneol=native#text/pascal
+utils/unicode/unihelper.lpi svneol=native#text/plain
+utils/unicode/unihelper.lpr svneol=native#text/pascal
+utils/unicode/weight_derivation.inc svneol=native#text/pascal

utils/Makefile

@@ -1,5 +1,5 @@
 #
-# Don't edit, this file is generated by FPCMake Version 2.0.0 [2013/01/16]
+# Don't edit, this file is generated by FPCMake Version 2.0.0 [2013/03/08]
 #
 default: all
 MAKEFILETARGETS=i386-linux i386-go32v2 i386-win32 i386-os2 i386-freebsd i386-beos i386-haiku i386-netbsd i386-solaris i386-qnx i386-netware i386-openbsd i386-wdosx i386-darwin i386-emx i386-watcom i386-netwlibc i386-wince i386-embedded i386-symbian i386-nativent i386-iphonesim i386-android m68k-linux m68k-freebsd m68k-netbsd m68k-amiga m68k-atari m68k-openbsd m68k-palmos m68k-embedded powerpc-linux powerpc-netbsd powerpc-amiga powerpc-macos powerpc-darwin powerpc-morphos powerpc-embedded powerpc-wii powerpc-aix sparc-linux sparc-netbsd sparc-solaris sparc-embedded x86_64-linux x86_64-freebsd x86_64-netbsd x86_64-solaris x86_64-openbsd x86_64-darwin x86_64-win64 x86_64-embedded arm-linux arm-palmos arm-darwin arm-wince arm-gba arm-nds arm-embedded arm-symbian arm-android powerpc64-linux powerpc64-darwin powerpc64-embedded powerpc64-aix avr-embedded armeb-linux armeb-embedded mips-linux mipsel-linux jvm-java jvm-android
@@ -268,6 +268,19 @@ ifdef CROSSCOMPILE
 ifndef DARWIN2DARWIN
 ifneq ($(CPU_TARGET),jvm)
 BINUTILSPREFIX=$(CPU_TARGET)-$(OS_TARGET)-
+ifeq ($(OS_TARGET),android)
+ifeq ($(CPU_TARGET),arm)
+BINUTILSPREFIX=arm-linux-androideabi-
+else
+ifeq ($(CPU_TARGET),i386)
+BINUTILSPREFIX=i686-linux-android-
+else
+ifeq ($(CPU_TARGET),mips)
+BINUTILSPREFIX=mipsel-linux-android-
+endif
+endif
+endif
+endif
 endif
 endif
 endif
@@ -311,7 +324,7 @@ ifeq ($(FULL_TARGET),i386-go32v2)
 override TARGET_DIRS+=fppkg fpcm tply h2pas fprcp dxegen fpdoc fpcmkcfg pas2ut pas2fpm  rmwait
 endif
 ifeq ($(FULL_TARGET),i386-win32)
-override TARGET_DIRS+=fppkg fpcm tply h2pas fprcp dxegen fpdoc fpcmkcfg pas2ut pas2fpm  fpmc fpcres rmwait instantfpc importtl
+override TARGET_DIRS+=fppkg fpcm tply h2pas fprcp dxegen fpdoc fpcmkcfg pas2ut pas2fpm  fpmc fpcres rmwait instantfpc importtl unicode
 endif
 ifeq ($(FULL_TARGET),i386-os2)
 override TARGET_DIRS+=fppkg fpcm tply h2pas fprcp dxegen fpdoc fpcmkcfg pas2ut pas2fpm  fpmc fpcres rmwait
@@ -3379,6 +3392,7 @@ TARGET_DIRS_FPCRES=1
 TARGET_DIRS_RMWAIT=1
 TARGET_DIRS_INSTANTFPC=1
 TARGET_DIRS_IMPORTTL=1
+TARGET_DIRS_UNICODE=1
 endif
 ifeq ($(FULL_TARGET),i386-os2)
 TARGET_DIRS_FPPKG=1
@@ -4970,6 +4984,51 @@ importtl:
 	$(MAKE) -C importtl all
 .PHONY: importtl_all importtl_debug importtl_smart importtl_release importtl_units importtl_examples importtl_shared importtl_install importtl_sourceinstall importtl_exampleinstall importtl_distinstall importtl_zipinstall importtl_zipsourceinstall importtl_zipexampleinstall importtl_zipdistinstall importtl_clean importtl_distclean importtl_cleanall importtl_info importtl_makefiles importtl
 endif
+ifdef TARGET_DIRS_UNICODE
+unicode_all:
+	$(MAKE) -C unicode all
+unicode_debug:
+	$(MAKE) -C unicode debug
+unicode_smart:
+	$(MAKE) -C unicode smart
+unicode_release:
+	$(MAKE) -C unicode release
+unicode_units:
+	$(MAKE) -C unicode units
+unicode_examples:
+	$(MAKE) -C unicode examples
+unicode_shared:
+	$(MAKE) -C unicode shared
+unicode_install:
+	$(MAKE) -C unicode install
+unicode_sourceinstall:
+	$(MAKE) -C unicode sourceinstall
+unicode_exampleinstall:
+	$(MAKE) -C unicode exampleinstall
+unicode_distinstall:
+	$(MAKE) -C unicode distinstall
+unicode_zipinstall:
+	$(MAKE) -C unicode zipinstall
+unicode_zipsourceinstall:
+	$(MAKE) -C unicode zipsourceinstall
+unicode_zipexampleinstall:
+	$(MAKE) -C unicode zipexampleinstall
+unicode_zipdistinstall:
+	$(MAKE) -C unicode zipdistinstall
+unicode_clean:
+	$(MAKE) -C unicode clean
+unicode_distclean:
+	$(MAKE) -C unicode distclean
+unicode_cleanall:
+	$(MAKE) -C unicode cleanall
+unicode_info:
+	$(MAKE) -C unicode info
+unicode_makefiles:
+	$(MAKE) -C unicode makefiles
+unicode:
+	$(MAKE) -C unicode all
+.PHONY: unicode_all unicode_debug unicode_smart unicode_release unicode_units unicode_examples unicode_shared unicode_install unicode_sourceinstall unicode_exampleinstall unicode_distinstall unicode_zipinstall unicode_zipsourceinstall unicode_zipexampleinstall unicode_zipdistinstall unicode_clean unicode_distclean unicode_cleanall unicode_info unicode_makefiles unicode
+endif
 ifdef TARGET_DIRS_FPCRESLIPO
 fpcreslipo_all:
 	$(MAKE) -C fpcreslipo all

utils/Makefile.fpc

@@ -10,7 +10,7 @@ version=2.7.1
 dirs=fppkg fpcm tply h2pas fprcp dxegen fpdoc fpcmkcfg pas2ut pas2fpm
 programs=ppdep ptop rstconv data2inc delp bin2obj postw32 rmcvsdir
 programs_linux=grab_vcsa
-dirs_win32=fpmc fpcres rmwait instantfpc importtl
+dirs_win32=fpmc fpcres rmwait instantfpc importtl unicode
 dirs_win64=fpmc fpcres rmwait instantfpc importtl
 dirs_wince=fpcres rmwait instantfpc
 dirs_haiku=fpcres instantfpc

utils/unicode/Makefile

@@ -0,0 +1,2154 @@
+#
+# Don't edit, this file is generated by FPCMake Version 2.0.0 [2013/03/08]
+#
+default: all
+MAKEFILETARGETS=i386-linux i386-go32v2 i386-win32 i386-os2 i386-freebsd i386-beos i386-haiku i386-netbsd i386-solaris i386-qnx i386-netware i386-openbsd i386-wdosx i386-darwin i386-emx i386-watcom i386-netwlibc i386-wince i386-embedded i386-symbian i386-nativent i386-iphonesim i386-android m68k-linux m68k-freebsd m68k-netbsd m68k-amiga m68k-atari m68k-openbsd m68k-palmos m68k-embedded powerpc-linux powerpc-netbsd powerpc-amiga powerpc-macos powerpc-darwin powerpc-morphos powerpc-embedded powerpc-wii powerpc-aix sparc-linux sparc-netbsd sparc-solaris sparc-embedded x86_64-linux x86_64-freebsd x86_64-netbsd x86_64-solaris x86_64-openbsd x86_64-darwin x86_64-win64 x86_64-embedded arm-linux arm-palmos arm-darwin arm-wince arm-gba arm-nds arm-embedded arm-symbian arm-android powerpc64-linux powerpc64-darwin powerpc64-embedded powerpc64-aix avr-embedded armeb-linux armeb-embedded mips-linux mipsel-linux jvm-java jvm-android
+BSDs = freebsd netbsd openbsd darwin
+UNIXs = linux $(BSDs) solaris qnx haiku aix 
+LIMIT83fs = go32v2 os2 emx watcom
+OSNeedsComspecToRunBatch = go32v2 watcom
+FORCE:
+.PHONY: FORCE
+override PATH:=$(patsubst %/,%,$(subst \,/,$(PATH)))
+ifneq ($(findstring darwin,$(OSTYPE)),)
+inUnix=1 #darwin
+SEARCHPATH:=$(filter-out .,$(subst :, ,$(PATH)))
+else
+ifeq ($(findstring ;,$(PATH)),)
+inUnix=1
+SEARCHPATH:=$(filter-out .,$(subst :, ,$(PATH)))
+else
+SEARCHPATH:=$(subst ;, ,$(PATH))
+endif
+endif
+SEARCHPATH+=$(patsubst %/,%,$(subst \,/,$(dir $(MAKE))))
+PWD:=$(strip $(wildcard $(addsuffix /pwd.exe,$(SEARCHPATH))))
+ifeq ($(PWD),)
+PWD:=$(strip $(wildcard $(addsuffix /pwd,$(SEARCHPATH))))
+ifeq ($(PWD),)
+$(error You need the GNU utils package to use this Makefile)
+else
+PWD:=$(firstword $(PWD))
+SRCEXEEXT=
+endif
+else
+PWD:=$(firstword $(PWD))
+SRCEXEEXT=.exe
+endif
+ifndef inUnix
+ifeq ($(OS),Windows_NT)
+inWinNT=1
+else
+ifdef OS2_SHELL
+inOS2=1
+endif
+endif
+else
+ifneq ($(findstring cygdrive,$(PATH)),)
+inCygWin=1
+endif
+endif
+ifdef inUnix
+SRCBATCHEXT=.sh
+else
+ifdef inOS2
+SRCBATCHEXT=.cmd
+else
+SRCBATCHEXT=.bat
+endif
+endif
+ifdef COMSPEC
+ifneq ($(findstring $(OS_SOURCE),$(OSNeedsComspecToRunBatch)),)
+ifndef RUNBATCH
+RUNBATCH=$(COMSPEC) /C
+endif
+endif
+endif
+ifdef inUnix
+PATHSEP=/
+else
+PATHSEP:=$(subst /,\,/)
+ifdef inCygWin
+PATHSEP=/
+endif
+endif
+ifdef PWD
+BASEDIR:=$(subst \,/,$(shell $(PWD)))
+ifdef inCygWin
+ifneq ($(findstring /cygdrive/,$(BASEDIR)),)
+BASENODIR:=$(patsubst /cygdrive%,%,$(BASEDIR))
+BASEDRIVE:=$(firstword $(subst /, ,$(BASENODIR)))
+BASEDIR:=$(subst /cygdrive/$(BASEDRIVE)/,$(BASEDRIVE):/,$(BASEDIR))
+endif
+endif
+else
+BASEDIR=.
+endif
+ifdef inOS2
+ifndef ECHO
+ECHO:=$(strip $(wildcard $(addsuffix /gecho$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(ECHO),)
+ECHO:=$(strip $(wildcard $(addsuffix /echo$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(ECHO),)
+ECHO=echo
+else
+ECHO:=$(firstword $(ECHO))
+endif
+else
+ECHO:=$(firstword $(ECHO))
+endif
+endif
+export ECHO
+endif
+override DEFAULT_FPCDIR=../..
+ifndef FPC
+ifdef PP
+FPC=$(PP)
+endif
+endif
+ifndef FPC
+FPCPROG:=$(strip $(wildcard $(addsuffix /fpc$(SRCEXEEXT),$(SEARCHPATH))))
+ifneq ($(FPCPROG),)
+FPCPROG:=$(firstword $(FPCPROG))
+ifneq ($(CPU_TARGET),)
+FPC:=$(shell $(FPCPROG) -P$(CPU_TARGET) -PB)
+else
+FPC:=$(shell $(FPCPROG) -PB)
+endif
+ifneq ($(findstring Error,$(FPC)),)
+override FPC=$(firstword $(strip $(wildcard $(addsuffix /ppc386$(SRCEXEEXT),$(SEARCHPATH)))))
+else
+ifeq ($(strip $(wildcard $(FPC))),)
+FPC:=$(firstword $(FPCPROG))
+endif
+endif
+else
+override FPC=$(firstword $(strip $(wildcard $(addsuffix /ppc386$(SRCEXEEXT),$(SEARCHPATH)))))
+endif
+endif
+override FPC:=$(subst $(SRCEXEEXT),,$(FPC))
+override FPC:=$(subst \,/,$(FPC))$(SRCEXEEXT)
+FOUNDFPC:=$(strip $(wildcard $(FPC)))
+ifeq ($(FOUNDFPC),)
+FOUNDFPC=$(strip $(wildcard $(addsuffix /$(FPC),$(SEARCHPATH))))
+ifeq ($(FOUNDFPC),)
+$(error Compiler $(FPC) not found)
+endif
+endif
+ifndef FPC_COMPILERINFO
+FPC_COMPILERINFO:=$(shell $(FPC) -iVSPTPSOTO)
+endif
+ifndef FPC_VERSION
+FPC_VERSION:=$(word 1,$(FPC_COMPILERINFO))
+endif
+export FPC FPC_VERSION FPC_COMPILERINFO
+unexport CHECKDEPEND ALLDEPENDENCIES
+ifndef CPU_TARGET
+ifdef CPU_TARGET_DEFAULT
+CPU_TARGET=$(CPU_TARGET_DEFAULT)
+endif
+endif
+ifndef OS_TARGET
+ifdef OS_TARGET_DEFAULT
+OS_TARGET=$(OS_TARGET_DEFAULT)
+endif
+endif
+ifndef CPU_SOURCE
+CPU_SOURCE:=$(word 2,$(FPC_COMPILERINFO))
+endif
+ifndef CPU_TARGET
+CPU_TARGET:=$(word 3,$(FPC_COMPILERINFO))
+endif
+ifndef OS_SOURCE
+OS_SOURCE:=$(word 4,$(FPC_COMPILERINFO))
+endif
+ifndef OS_TARGET
+OS_TARGET:=$(word 5,$(FPC_COMPILERINFO))
+endif
+FULL_TARGET=$(CPU_TARGET)-$(OS_TARGET)
+FULL_SOURCE=$(CPU_SOURCE)-$(OS_SOURCE)
+ifeq ($(CPU_TARGET),armeb)
+ARCH=arm
+override FPCOPT+=-Cb
+else
+ifeq ($(CPU_TARGET),armel)
+ARCH=arm
+override FPCOPT+=-CaEABI
+else
+ARCH=$(CPU_TARGET)
+endif
+endif
+ifeq ($(FULL_TARGET),arm-embedded)
+ifeq ($(SUBARCH),)
+$(error When compiling for arm-embedded, a sub-architecture (e.g. SUBARCH=armv4t or SUBARCH=armv7m) must be defined)
+endif
+override FPCOPT+=-Cp$(SUBARCH)
+endif
+ifneq ($(findstring $(OS_SOURCE),$(LIMIT83fs)),)
+TARGETSUFFIX=$(OS_TARGET)
+SOURCESUFFIX=$(OS_SOURCE)
+else
+ifneq ($(findstring $(OS_TARGET),$(LIMIT83fs)),)
+TARGETSUFFIX=$(OS_TARGET)
+else
+TARGETSUFFIX=$(FULL_TARGET)
+endif
+SOURCESUFFIX=$(FULL_SOURCE)
+endif
+ifneq ($(FULL_TARGET),$(FULL_SOURCE))
+CROSSCOMPILE=1
+endif
+ifeq ($(findstring makefile,$(MAKECMDGOALS)),)
+ifeq ($(findstring $(FULL_TARGET),$(MAKEFILETARGETS)),)
+$(error The Makefile doesn't support target $(FULL_TARGET), please run fpcmake first)
+endif
+endif
+ifneq ($(findstring $(OS_TARGET),$(BSDs)),)
+BSDhier=1
+endif
+ifeq ($(OS_TARGET),linux)
+linuxHier=1
+endif
+ifndef CROSSCOMPILE
+BUILDFULLNATIVE=1
+export BUILDFULLNATIVE
+endif
+ifdef BUILDFULLNATIVE
+BUILDNATIVE=1
+export BUILDNATIVE
+endif
+export OS_TARGET OS_SOURCE ARCH CPU_TARGET CPU_SOURCE FULL_TARGET FULL_SOURCE TARGETSUFFIX SOURCESUFFIX CROSSCOMPILE
+ifdef FPCDIR
+override FPCDIR:=$(subst \,/,$(FPCDIR))
+ifeq ($(wildcard $(addprefix $(FPCDIR)/,rtl units)),)
+override FPCDIR=wrong
+endif
+else
+override FPCDIR=wrong
+endif
+ifdef DEFAULT_FPCDIR
+ifeq ($(FPCDIR),wrong)
+override FPCDIR:=$(subst \,/,$(DEFAULT_FPCDIR))
+ifeq ($(wildcard $(addprefix $(FPCDIR)/,rtl units)),)
+override FPCDIR=wrong
+endif
+endif
+endif
+ifeq ($(FPCDIR),wrong)
+ifdef inUnix
+override FPCDIR=/usr/local/lib/fpc/$(FPC_VERSION)
+ifeq ($(wildcard $(FPCDIR)/units),)
+override FPCDIR=/usr/lib/fpc/$(FPC_VERSION)
+endif
+else
+override FPCDIR:=$(subst /$(FPC),,$(firstword $(strip $(wildcard $(addsuffix /$(FPC),$(SEARCHPATH))))))
+override FPCDIR:=$(FPCDIR)/..
+ifeq ($(wildcard $(addprefix $(FPCDIR)/,rtl units)),)
+override FPCDIR:=$(FPCDIR)/..
+ifeq ($(wildcard $(addprefix $(FPCDIR)/,rtl units)),)
+override FPCDIR:=$(BASEDIR)
+ifeq ($(wildcard $(addprefix $(FPCDIR)/,rtl units)),)
+override FPCDIR=c:/pp
+endif
+endif
+endif
+endif
+endif
+ifndef CROSSBINDIR
+CROSSBINDIR:=$(wildcard $(FPCDIR)/bin/$(TARGETSUFFIX))
+endif
+ifneq ($(findstring $(OS_TARGET),darwin iphonesim),)
+ifeq ($(OS_SOURCE),darwin)
+DARWIN2DARWIN=1
+endif
+endif
+ifndef BINUTILSPREFIX
+ifndef CROSSBINDIR
+ifdef CROSSCOMPILE
+ifndef DARWIN2DARWIN
+ifneq ($(CPU_TARGET),jvm)
+BINUTILSPREFIX=$(CPU_TARGET)-$(OS_TARGET)-
+ifeq ($(OS_TARGET),android)
+ifeq ($(CPU_TARGET),arm)
+BINUTILSPREFIX=arm-linux-androideabi-
+else
+ifeq ($(CPU_TARGET),i386)
+BINUTILSPREFIX=i686-linux-android-
+else
+ifeq ($(CPU_TARGET),mips)
+BINUTILSPREFIX=mipsel-linux-android-
+endif
+endif
+endif
+endif
+endif
+endif
+endif
+endif
+endif
+UNITSDIR:=$(wildcard $(FPCDIR)/units/$(TARGETSUFFIX))
+ifeq ($(UNITSDIR),)
+UNITSDIR:=$(wildcard $(FPCDIR)/units/$(OS_TARGET))
+endif
+PACKAGESDIR:=$(wildcard $(FPCDIR) $(FPCDIR)/packages $(FPCDIR)/packages/base $(FPCDIR)/packages/extra)
+ifndef FPCFPMAKE
+ifdef CROSSCOMPILE
+ifeq ($(strip $(wildcard $(addsuffix /compiler/ppc$(SRCEXEEXT),$(FPCDIR)))),)
+FPCPROG:=$(strip $(wildcard $(addsuffix /fpc$(SRCEXEEXT),$(SEARCHPATH))))
+ifneq ($(FPCPROG),)
+FPCPROG:=$(firstword $(FPCPROG))
+FPCFPMAKE:=$(shell $(FPCPROG) -PB)
+ifeq ($(strip $(wildcard $(FPCFPMAKE))),)
+FPCFPMAKE:=$(firstword $(FPCPROG))
+endif
+else
+override FPCFPMAKE=$(firstword $(strip $(wildcard $(addsuffix /ppc386$(SRCEXEEXT),$(SEARCHPATH)))))
+endif
+else
+FPCFPMAKE=$(strip $(wildcard $(addsuffix /compiler/ppc$(SRCEXEEXT),$(FPCDIR))))
+FPMAKE_SKIP_CONFIG=-n
+export FPCFPMAKE
+export FPMAKE_SKIP_CONFIG
+endif
+else
+FPMAKE_SKIP_CONFIG=-n
+FPCFPMAKE=$(FPC)
+endif
+endif
+ifeq ($(FULL_TARGET),i386-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-go32v2)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-win32)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-os2)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-freebsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-beos)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-haiku)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-netbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-solaris)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-qnx)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-netware)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-openbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-wdosx)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-darwin)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-emx)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-watcom)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-netwlibc)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-wince)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-symbian)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-nativent)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-iphonesim)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-android)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-freebsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-netbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-amiga)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-atari)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-openbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-palmos)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),m68k-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-netbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-amiga)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-macos)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-darwin)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-morphos)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-wii)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc-aix)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),sparc-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),sparc-netbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),sparc-solaris)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),sparc-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-freebsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-netbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-solaris)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-openbsd)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-darwin)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-win64)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),x86_64-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-palmos)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-darwin)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-wince)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-gba)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-nds)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-symbian)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),arm-android)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc64-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc64-darwin)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc64-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),powerpc64-aix)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),avr-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),armeb-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),armeb-embedded)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),mips-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),mipsel-linux)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),jvm-java)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),jvm-android)
+override TARGET_PROGRAMS+=cldrparser unihelper
+endif
+ifeq ($(FULL_TARGET),i386-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-go32v2)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-win32)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-os2)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-freebsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-beos)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-haiku)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-netbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-solaris)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-qnx)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-netware)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-openbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-wdosx)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-darwin)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-emx)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-watcom)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-netwlibc)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-wince)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-symbian)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-nativent)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-iphonesim)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),i386-android)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-freebsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-netbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-amiga)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-atari)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-openbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-palmos)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),m68k-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-netbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-amiga)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-macos)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-darwin)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-morphos)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-wii)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc-aix)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),sparc-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),sparc-netbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),sparc-solaris)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),sparc-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-freebsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-netbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-solaris)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-openbsd)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-darwin)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-win64)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),x86_64-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-palmos)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-darwin)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-wince)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-gba)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-nds)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-symbian)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),arm-android)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc64-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc64-darwin)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc64-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),powerpc64-aix)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),avr-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),armeb-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),armeb-embedded)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),mips-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),mipsel-linux)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),jvm-java)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+ifeq ($(FULL_TARGET),jvm-android)
+override CLEAN_UNITS+=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+endif
+override INSTALL_FPCPACKAGE=y
+ifdef REQUIRE_UNITSDIR
+override UNITSDIR+=$(REQUIRE_UNITSDIR)
+endif
+ifdef REQUIRE_PACKAGESDIR
+override PACKAGESDIR+=$(REQUIRE_PACKAGESDIR)
+endif
+ifdef ZIPINSTALL
+ifneq ($(findstring $(OS_TARGET),$(UNIXs)),)
+UNIXHier=1
+endif
+else
+ifneq ($(findstring $(OS_SOURCE),$(UNIXs)),)
+UNIXHier=1
+endif
+endif
+ifndef INSTALL_PREFIX
+ifdef PREFIX
+INSTALL_PREFIX=$(PREFIX)
+endif
+endif
+ifndef INSTALL_PREFIX
+ifdef UNIXHier
+INSTALL_PREFIX=/usr/local
+else
+ifdef INSTALL_FPCPACKAGE
+INSTALL_BASEDIR:=/pp
+else
+INSTALL_BASEDIR:=/$(PACKAGE_NAME)
+endif
+endif
+endif
+export INSTALL_PREFIX
+ifdef INSTALL_FPCSUBDIR
+export INSTALL_FPCSUBDIR
+endif
+ifndef DIST_DESTDIR
+DIST_DESTDIR:=$(BASEDIR)
+endif
+export DIST_DESTDIR
+ifndef COMPILER_UNITTARGETDIR
+ifdef PACKAGEDIR_MAIN
+COMPILER_UNITTARGETDIR=$(PACKAGEDIR_MAIN)/units/$(TARGETSUFFIX)
+else
+COMPILER_UNITTARGETDIR=units/$(TARGETSUFFIX)
+endif
+endif
+ifndef COMPILER_TARGETDIR
+COMPILER_TARGETDIR=.
+endif
+ifndef INSTALL_BASEDIR
+ifdef UNIXHier
+ifdef INSTALL_FPCPACKAGE
+INSTALL_BASEDIR:=$(INSTALL_PREFIX)/lib/fpc/$(FPC_VERSION)
+else
+INSTALL_BASEDIR:=$(INSTALL_PREFIX)/lib/$(PACKAGE_NAME)
+endif
+else
+INSTALL_BASEDIR:=$(INSTALL_PREFIX)
+endif
+endif
+ifndef INSTALL_BINDIR
+ifdef UNIXHier
+INSTALL_BINDIR:=$(INSTALL_PREFIX)/bin
+else
+INSTALL_BINDIR:=$(INSTALL_BASEDIR)/bin
+ifdef INSTALL_FPCPACKAGE
+ifdef CROSSCOMPILE
+ifdef CROSSINSTALL
+INSTALL_BINDIR:=$(INSTALL_BINDIR)/$(SOURCESUFFIX)
+else
+INSTALL_BINDIR:=$(INSTALL_BINDIR)/$(TARGETSUFFIX)
+endif
+else
+INSTALL_BINDIR:=$(INSTALL_BINDIR)/$(TARGETSUFFIX)
+endif
+endif
+endif
+endif
+ifndef INSTALL_UNITDIR
+INSTALL_UNITDIR:=$(INSTALL_BASEDIR)/units/$(TARGETSUFFIX)
+ifdef INSTALL_FPCPACKAGE
+ifdef PACKAGE_NAME
+INSTALL_UNITDIR:=$(INSTALL_UNITDIR)/$(PACKAGE_NAME)
+endif
+endif
+endif
+ifndef INSTALL_LIBDIR
+ifdef UNIXHier
+INSTALL_LIBDIR:=$(INSTALL_PREFIX)/lib
+else
+INSTALL_LIBDIR:=$(INSTALL_UNITDIR)
+endif
+endif
+ifndef INSTALL_SOURCEDIR
+ifdef UNIXHier
+ifdef BSDhier
+SRCPREFIXDIR=share/src
+else
+ifdef linuxHier
+SRCPREFIXDIR=share/src
+else
+SRCPREFIXDIR=src
+endif
+endif
+ifdef INSTALL_FPCPACKAGE
+ifdef INSTALL_FPCSUBDIR
+INSTALL_SOURCEDIR:=$(INSTALL_PREFIX)/$(SRCPREFIXDIR)/fpc-$(FPC_VERSION)/$(INSTALL_FPCSUBDIR)/$(PACKAGE_NAME)
+else
+INSTALL_SOURCEDIR:=$(INSTALL_PREFIX)/$(SRCPREFIXDIR)/fpc-$(FPC_VERSION)/$(PACKAGE_NAME)
+endif
+else
+INSTALL_SOURCEDIR:=$(INSTALL_PREFIX)/$(SRCPREFIXDIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)
+endif
+else
+ifdef INSTALL_FPCPACKAGE
+ifdef INSTALL_FPCSUBDIR
+INSTALL_SOURCEDIR:=$(INSTALL_BASEDIR)/source/$(INSTALL_FPCSUBDIR)/$(PACKAGE_NAME)
+else
+INSTALL_SOURCEDIR:=$(INSTALL_BASEDIR)/source/$(PACKAGE_NAME)
+endif
+else
+INSTALL_SOURCEDIR:=$(INSTALL_BASEDIR)/source
+endif
+endif
+endif
+ifndef INSTALL_DOCDIR
+ifdef UNIXHier
+ifdef BSDhier
+DOCPREFIXDIR=share/doc
+else
+ifdef linuxHier
+DOCPREFIXDIR=share/doc
+else
+DOCPREFIXDIR=doc
+endif
+endif
+ifdef INSTALL_FPCPACKAGE
+INSTALL_DOCDIR:=$(INSTALL_PREFIX)/$(DOCPREFIXDIR)/fpc-$(FPC_VERSION)/$(PACKAGE_NAME)
+else
+INSTALL_DOCDIR:=$(INSTALL_PREFIX)/$(DOCPREFIXDIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)
+endif
+else
+ifdef INSTALL_FPCPACKAGE
+INSTALL_DOCDIR:=$(INSTALL_BASEDIR)/doc/$(PACKAGE_NAME)
+else
+INSTALL_DOCDIR:=$(INSTALL_BASEDIR)/doc
+endif
+endif
+endif
+ifndef INSTALL_EXAMPLEDIR
+ifdef UNIXHier
+ifdef INSTALL_FPCPACKAGE
+ifdef BSDhier
+INSTALL_EXAMPLEDIR:=$(INSTALL_PREFIX)/share/examples/fpc-$(FPC_VERSION)/$(PACKAGE_NAME)
+else
+ifdef linuxHier
+INSTALL_EXAMPLEDIR:=$(INSTALL_DOCDIR)/examples
+else
+INSTALL_EXAMPLEDIR:=$(INSTALL_PREFIX)/doc/fpc-$(FPC_VERSION)/examples/$(PACKAGE_NAME)
+endif
+endif
+else
+ifdef BSDhier
+INSTALL_EXAMPLEDIR:=$(INSTALL_PREFIX)/share/examples/$(PACKAGE_NAME)-$(PACKAGE_VERSION)
+else
+ifdef linuxHier
+INSTALL_EXAMPLEDIR:=$(INSTALL_DOCDIR)/examples/$(PACKAGE_NAME)-$(PACKAGE_VERSION)
+else
+INSTALL_EXAMPLEDIR:=$(INSTALL_PREFIX)/doc/$(PACKAGE_NAME)-$(PACKAGE_VERSION)
+endif
+endif
+endif
+else
+ifdef INSTALL_FPCPACKAGE
+INSTALL_EXAMPLEDIR:=$(INSTALL_BASEDIR)/examples/$(PACKAGE_NAME)
+else
+INSTALL_EXAMPLEDIR:=$(INSTALL_BASEDIR)/examples
+endif
+endif
+endif
+ifndef INSTALL_DATADIR
+INSTALL_DATADIR=$(INSTALL_BASEDIR)
+endif
+ifndef INSTALL_SHAREDDIR
+INSTALL_SHAREDDIR=$(INSTALL_PREFIX)/lib
+endif
+ifdef CROSSCOMPILE
+ifndef CROSSBINDIR
+CROSSBINDIR:=$(wildcard $(CROSSTARGETDIR)/bin/$(SOURCESUFFIX))
+ifeq ($(CROSSBINDIR),)
+CROSSBINDIR:=$(wildcard $(INSTALL_BASEDIR)/cross/$(TARGETSUFFIX)/bin/$(FULL_SOURCE))
+endif
+endif
+else
+CROSSBINDIR=
+endif
+BATCHEXT=.bat
+LOADEREXT=.as
+EXEEXT=.exe
+PPLEXT=.ppl
+PPUEXT=.ppu
+OEXT=.o
+ASMEXT=.s
+SMARTEXT=.sl
+STATICLIBEXT=.a
+SHAREDLIBEXT=.so
+SHAREDLIBPREFIX=libfp
+STATICLIBPREFIX=libp
+IMPORTLIBPREFIX=libimp
+RSTEXT=.rst
+EXEDBGEXT=.dbg
+ifeq ($(OS_TARGET),go32v1)
+STATICLIBPREFIX=
+SHORTSUFFIX=v1
+endif
+ifeq ($(OS_TARGET),go32v2)
+STATICLIBPREFIX=
+SHORTSUFFIX=dos
+IMPORTLIBPREFIX=
+endif
+ifeq ($(OS_TARGET),watcom)
+STATICLIBPREFIX=
+OEXT=.obj
+ASMEXT=.asm
+SHAREDLIBEXT=.dll
+SHORTSUFFIX=wat
+IMPORTLIBPREFIX=
+endif
+ifneq ($(CPU_TARGET),jvm)
+ifeq ($(OS_TARGET),android)
+BATCHEXT=.sh
+EXEEXT=
+HASSHAREDLIB=1
+SHORTSUFFIX=lnx
+endif
+endif
+ifeq ($(OS_TARGET),linux)
+BATCHEXT=.sh
+EXEEXT=
+HASSHAREDLIB=1
+SHORTSUFFIX=lnx
+endif
+ifeq ($(OS_TARGET),freebsd)
+BATCHEXT=.sh
+EXEEXT=
+HASSHAREDLIB=1
+SHORTSUFFIX=fbs
+endif
+ifeq ($(OS_TARGET),netbsd)
+BATCHEXT=.sh
+EXEEXT=
+HASSHAREDLIB=1
+SHORTSUFFIX=nbs
+endif
+ifeq ($(OS_TARGET),openbsd)
+BATCHEXT=.sh
+EXEEXT=
+HASSHAREDLIB=1
+SHORTSUFFIX=obs
+endif
+ifeq ($(OS_TARGET),win32)
+SHAREDLIBEXT=.dll
+SHORTSUFFIX=w32
+endif
+ifeq ($(OS_TARGET),os2)
+BATCHEXT=.cmd
+AOUTEXT=.out
+STATICLIBPREFIX=
+SHAREDLIBEXT=.dll
+SHORTSUFFIX=os2
+ECHO=echo
+IMPORTLIBPREFIX=
+endif
+ifeq ($(OS_TARGET),emx)
+BATCHEXT=.cmd
+AOUTEXT=.out
+STATICLIBPREFIX=
+SHAREDLIBEXT=.dll
+SHORTSUFFIX=emx
+ECHO=echo
+IMPORTLIBPREFIX=
+endif
+ifeq ($(OS_TARGET),amiga)
+EXEEXT=
+SHAREDLIBEXT=.library
+SHORTSUFFIX=amg
+endif
+ifeq ($(OS_TARGET),morphos)
+EXEEXT=
+SHAREDLIBEXT=.library
+SHORTSUFFIX=mos
+endif
+ifeq ($(OS_TARGET),atari)
+EXEEXT=.ttp
+SHORTSUFFIX=ata
+endif
+ifeq ($(OS_TARGET),beos)
+BATCHEXT=.sh
+EXEEXT=
+SHORTSUFFIX=be
+endif
+ifeq ($(OS_TARGET),haiku)
+BATCHEXT=.sh
+EXEEXT=
+SHORTSUFFIX=hai
+endif
+ifeq ($(OS_TARGET),solaris)
+BATCHEXT=.sh
+EXEEXT=
+SHORTSUFFIX=sun
+endif
+ifeq ($(OS_TARGET),qnx)
+BATCHEXT=.sh
+EXEEXT=
+SHORTSUFFIX=qnx
+endif
+ifeq ($(OS_TARGET),netware)
+EXEEXT=.nlm
+STATICLIBPREFIX=
+SHORTSUFFIX=nw
+IMPORTLIBPREFIX=imp
+endif
+ifeq ($(OS_TARGET),netwlibc)
+EXEEXT=.nlm
+STATICLIBPREFIX=
+SHORTSUFFIX=nwl
+IMPORTLIBPREFIX=imp
+endif
+ifeq ($(OS_TARGET),macos)
+BATCHEXT=
+EXEEXT=
+DEBUGSYMEXT=.xcoff
+SHORTSUFFIX=mac
+IMPORTLIBPREFIX=imp
+endif
+ifneq ($(findstring $(OS_TARGET),darwin iphonesim),)
+BATCHEXT=.sh
+EXEEXT=
+HASSHAREDLIB=1
+SHORTSUFFIX=dwn
+EXEDBGEXT=.dSYM
+endif
+ifeq ($(OS_TARGET),gba)
+EXEEXT=.gba
+SHAREDLIBEXT=.so
+SHORTSUFFIX=gba
+endif
+ifeq ($(OS_TARGET),symbian)
+SHAREDLIBEXT=.dll
+SHORTSUFFIX=symbian
+endif
+ifeq ($(OS_TARGET),NativeNT)
+SHAREDLIBEXT=.dll
+SHORTSUFFIX=nativent
+endif
+ifeq ($(OS_TARGET),wii)
+EXEEXT=.dol
+SHAREDLIBEXT=.so
+SHORTSUFFIX=wii
+endif
+ifeq ($(OS_TARGET),aix)
+BATCHEXT=.sh
+EXEEXT=
+SHORTSUFFIX=aix
+endif
+ifeq ($(OS_TARGET),java)
+OEXT=.class
+ASMEXT=.j
+SHAREDLIBEXT=.jar
+SHORTSUFFIX=java
+endif
+ifeq ($(CPU_TARGET),jvm)
+ifeq ($(OS_TARGET),android)
+OEXT=.class
+ASMEXT=.j
+SHAREDLIBEXT=.jar
+SHORTSUFFIX=android
+endif
+endif
+ifneq ($(findstring $(OS_SOURCE),$(LIMIT83fs)),)
+FPCMADE=fpcmade.$(SHORTSUFFIX)
+ZIPSUFFIX=$(SHORTSUFFIX)
+ZIPCROSSPREFIX=
+ZIPSOURCESUFFIX=src
+ZIPEXAMPLESUFFIX=exm
+else
+FPCMADE=fpcmade.$(TARGETSUFFIX)
+ZIPSOURCESUFFIX=.source
+ZIPEXAMPLESUFFIX=.examples
+ifdef CROSSCOMPILE
+ZIPSUFFIX=.$(SOURCESUFFIX)
+ZIPCROSSPREFIX=$(TARGETSUFFIX)-
+else
+ZIPSUFFIX=.$(TARGETSUFFIX)
+ZIPCROSSPREFIX=
+endif
+endif
+ifndef ECHO
+ECHO:=$(strip $(wildcard $(addsuffix /gecho$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(ECHO),)
+ECHO:=$(strip $(wildcard $(addsuffix /echo$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(ECHO),)
+ECHO= __missing_command_ECHO
+else
+ECHO:=$(firstword $(ECHO))
+endif
+else
+ECHO:=$(firstword $(ECHO))
+endif
+endif
+export ECHO
+ifndef DATE
+DATE:=$(strip $(wildcard $(addsuffix /gdate$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(DATE),)
+DATE:=$(strip $(wildcard $(addsuffix /date$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(DATE),)
+DATE= __missing_command_DATE
+else
+DATE:=$(firstword $(DATE))
+endif
+else
+DATE:=$(firstword $(DATE))
+endif
+endif
+export DATE
+ifndef GINSTALL
+GINSTALL:=$(strip $(wildcard $(addsuffix /ginstall$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(GINSTALL),)
+GINSTALL:=$(strip $(wildcard $(addsuffix /install$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(GINSTALL),)
+GINSTALL= __missing_command_GINSTALL
+else
+GINSTALL:=$(firstword $(GINSTALL))
+endif
+else
+GINSTALL:=$(firstword $(GINSTALL))
+endif
+endif
+export GINSTALL
+ifndef CPPROG
+CPPROG:=$(strip $(wildcard $(addsuffix /cp$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(CPPROG),)
+CPPROG= __missing_command_CPPROG
+else
+CPPROG:=$(firstword $(CPPROG))
+endif
+endif
+export CPPROG
+ifndef RMPROG
+RMPROG:=$(strip $(wildcard $(addsuffix /rm$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(RMPROG),)
+RMPROG= __missing_command_RMPROG
+else
+RMPROG:=$(firstword $(RMPROG))
+endif
+endif
+export RMPROG
+ifndef MVPROG
+MVPROG:=$(strip $(wildcard $(addsuffix /mv$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(MVPROG),)
+MVPROG= __missing_command_MVPROG
+else
+MVPROG:=$(firstword $(MVPROG))
+endif
+endif
+export MVPROG
+ifndef MKDIRPROG
+MKDIRPROG:=$(strip $(wildcard $(addsuffix /gmkdir$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(MKDIRPROG),)
+MKDIRPROG:=$(strip $(wildcard $(addsuffix /mkdir$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(MKDIRPROG),)
+MKDIRPROG= __missing_command_MKDIRPROG
+else
+MKDIRPROG:=$(firstword $(MKDIRPROG))
+endif
+else
+MKDIRPROG:=$(firstword $(MKDIRPROG))
+endif
+endif
+export MKDIRPROG
+ifndef ECHOREDIR
+ifndef inUnix
+ECHOREDIR=echo
+else
+ECHOREDIR=$(ECHO)
+endif
+endif
+ifndef COPY
+COPY:=$(CPPROG) -fp
+endif
+ifndef COPYTREE
+COPYTREE:=$(CPPROG) -Rfp
+endif
+ifndef MKDIRTREE
+MKDIRTREE:=$(MKDIRPROG) -p
+endif
+ifndef MOVE
+MOVE:=$(MVPROG) -f
+endif
+ifndef DEL
+DEL:=$(RMPROG) -f
+endif
+ifndef DELTREE
+DELTREE:=$(RMPROG) -rf
+endif
+ifndef INSTALL
+ifdef inUnix
+INSTALL:=$(GINSTALL) -c -m 644
+else
+INSTALL:=$(COPY)
+endif
+endif
+ifndef INSTALLEXE
+ifdef inUnix
+INSTALLEXE:=$(GINSTALL) -c -m 755
+else
+INSTALLEXE:=$(COPY)
+endif
+endif
+ifndef MKDIR
+MKDIR:=$(GINSTALL) -m 755 -d
+endif
+export ECHOREDIR COPY COPYTREE MOVE DEL DELTREE INSTALL INSTALLEXE MKDIR
+ifndef PPUMOVE
+PPUMOVE:=$(strip $(wildcard $(addsuffix /ppumove$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(PPUMOVE),)
+PPUMOVE= __missing_command_PPUMOVE
+else
+PPUMOVE:=$(firstword $(PPUMOVE))
+endif
+endif
+export PPUMOVE
+ifndef FPCMAKE
+FPCMAKE:=$(strip $(wildcard $(addsuffix /fpcmake$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(FPCMAKE),)
+FPCMAKE= __missing_command_FPCMAKE
+else
+FPCMAKE:=$(firstword $(FPCMAKE))
+endif
+endif
+export FPCMAKE
+ifndef ZIPPROG
+ZIPPROG:=$(strip $(wildcard $(addsuffix /zip$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(ZIPPROG),)
+ZIPPROG= __missing_command_ZIPPROG
+else
+ZIPPROG:=$(firstword $(ZIPPROG))
+endif
+endif
+export ZIPPROG
+ifndef TARPROG
+TARPROG:=$(strip $(wildcard $(addsuffix /gtar$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(TARPROG),)
+TARPROG:=$(strip $(wildcard $(addsuffix /tar$(SRCEXEEXT),$(SEARCHPATH))))
+ifeq ($(TARPROG),)
+TARPROG= __missing_command_TARPROG
+else
+TARPROG:=$(firstword $(TARPROG))
+endif
+else
+TARPROG:=$(firstword $(TARPROG))
+endif
+endif
+export TARPROG
+ASNAME=$(BINUTILSPREFIX)as
+LDNAME=$(BINUTILSPREFIX)ld
+ARNAME=$(BINUTILSPREFIX)ar
+RCNAME=$(BINUTILSPREFIX)rc
+ifndef ASPROG
+ifdef CROSSBINDIR
+ASPROG=$(CROSSBINDIR)/$(ASNAME)$(SRCEXEEXT)
+else
+ASPROG=$(ASNAME)
+endif
+endif
+ifndef LDPROG
+ifdef CROSSBINDIR
+LDPROG=$(CROSSBINDIR)/$(LDNAME)$(SRCEXEEXT)
+else
+LDPROG=$(LDNAME)
+endif
+endif
+ifndef RCPROG
+ifdef CROSSBINDIR
+RCPROG=$(CROSSBINDIR)/$(RCNAME)$(SRCEXEEXT)
+else
+RCPROG=$(RCNAME)
+endif
+endif
+ifndef ARPROG
+ifdef CROSSBINDIR
+ARPROG=$(CROSSBINDIR)/$(ARNAME)$(SRCEXEEXT)
+else
+ARPROG=$(ARNAME)
+endif
+endif
+AS=$(ASPROG)
+LD=$(LDPROG)
+RC=$(RCPROG)
+AR=$(ARPROG)
+PPAS=ppas$(SRCBATCHEXT)
+ifdef inUnix
+LDCONFIG=ldconfig
+else
+LDCONFIG=
+endif
+ifdef DATE
+DATESTR:=$(shell $(DATE) +%Y%m%d)
+else
+DATESTR=
+endif
+ZIPOPT=-9
+ZIPEXT=.zip
+ifeq ($(USETAR),bz2)
+TAROPT=vj
+TAREXT=.tar.bz2
+else
+TAROPT=vz
+TAREXT=.tar.gz
+endif
+override REQUIRE_PACKAGES=rtl rtl
+ifeq ($(FULL_TARGET),i386-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-go32v2)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-win32)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-os2)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-freebsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-beos)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-haiku)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-netbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-solaris)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-qnx)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-netware)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-openbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-wdosx)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-darwin)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-emx)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-watcom)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-netwlibc)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-wince)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-symbian)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-nativent)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-iphonesim)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),i386-android)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-freebsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-netbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-amiga)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-atari)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-openbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-palmos)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),m68k-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-netbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-amiga)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-macos)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-darwin)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-morphos)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-wii)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc-aix)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),sparc-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),sparc-netbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),sparc-solaris)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),sparc-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-freebsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-netbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-solaris)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-openbsd)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-darwin)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-win64)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),x86_64-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-palmos)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-darwin)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-wince)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-gba)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-nds)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-symbian)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),arm-android)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc64-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc64-darwin)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc64-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),powerpc64-aix)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),avr-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),armeb-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),armeb-embedded)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),mips-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),mipsel-linux)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),jvm-java)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifeq ($(FULL_TARGET),jvm-android)
+REQUIRE_PACKAGES_RTL=1
+endif
+ifdef REQUIRE_PACKAGES_RTL
+PACKAGEDIR_RTL:=$(firstword $(subst /Makefile.fpc,,$(strip $(wildcard $(addsuffix /rtl/Makefile.fpc,$(PACKAGESDIR))))))
+ifneq ($(PACKAGEDIR_RTL),)
+ifneq ($(wildcard $(PACKAGEDIR_RTL)/units/$(TARGETSUFFIX)),)
+UNITDIR_RTL=$(PACKAGEDIR_RTL)/units/$(TARGETSUFFIX)
+else
+UNITDIR_RTL=$(PACKAGEDIR_RTL)
+endif
+ifneq ($(wildcard $(PACKAGEDIR_RTL)/units/$(SOURCESUFFIX)),)
+UNITDIR_FPMAKE_RTL=$(PACKAGEDIR_RTL)/units/$(SOURCESUFFIX)
+else
+ifneq ($(wildcard $(PACKAGEDIR_RTL)/units_bs/$(SOURCESUFFIX)),)
+UNITDIR_FPMAKE_RTL=$(PACKAGEDIR_RTL)/units_bs/$(SOURCESUFFIX)
+else
+UNITDIR_FPMAKE_RTL=$(PACKAGEDIR_RTL)
+endif
+endif
+ifdef CHECKDEPEND
+$(PACKAGEDIR_RTL)/$(OS_TARGET)/$(FPCMADE):
+	$(MAKE) -C $(PACKAGEDIR_RTL)/$(OS_TARGET) $(FPCMADE)
+override ALLDEPENDENCIES+=$(PACKAGEDIR_RTL)/$(OS_TARGET)/$(FPCMADE)
+endif
+else
+PACKAGEDIR_RTL=
+UNITDIR_RTL:=$(subst /Package.fpc,,$(strip $(wildcard $(addsuffix /rtl/Package.fpc,$(UNITSDIR)))))
+ifneq ($(UNITDIR_RTL),)
+UNITDIR_RTL:=$(firstword $(UNITDIR_RTL))
+else
+UNITDIR_RTL=
+endif
+endif
+ifdef UNITDIR_RTL
+override COMPILER_UNITDIR+=$(UNITDIR_RTL)
+endif
+ifdef UNITDIR_FPMAKE_RTL
+override COMPILER_FPMAKE_UNITDIR+=$(UNITDIR_FPMAKE_RTL)
+endif
+endif
+ifndef NOCPUDEF
+override FPCOPTDEF=$(ARCH)
+endif
+ifneq ($(OS_TARGET),$(OS_SOURCE))
+override FPCOPT+=-T$(OS_TARGET)
+endif
+ifneq ($(CPU_TARGET),$(CPU_SOURCE))
+override FPCOPT+=-P$(ARCH)
+endif
+ifeq ($(OS_SOURCE),openbsd)
+override FPCOPT+=-FD$(NEW_BINUTILS_PATH)
+override FPCMAKEOPT+=-FD$(NEW_BINUTILS_PATH)
+endif
+ifndef CROSSBOOTSTRAP
+ifneq ($(BINUTILSPREFIX),)
+override FPCOPT+=-XP$(BINUTILSPREFIX)
+endif
+ifneq ($(BINUTILSPREFIX),)
+override FPCOPT+=-Xr$(RLINKPATH)
+endif
+endif
+ifndef CROSSCOMPILE
+ifneq ($(BINUTILSPREFIX),)
+override FPCMAKEOPT+=-XP$(BINUTILSPREFIX)
+endif
+endif
+ifdef UNITDIR
+override FPCOPT+=$(addprefix -Fu,$(UNITDIR))
+endif
+ifdef LIBDIR
+override FPCOPT+=$(addprefix -Fl,$(LIBDIR))
+endif
+ifdef OBJDIR
+override FPCOPT+=$(addprefix -Fo,$(OBJDIR))
+endif
+ifdef INCDIR
+override FPCOPT+=$(addprefix -Fi,$(INCDIR))
+endif
+ifdef LINKSMART
+override FPCOPT+=-XX
+endif
+ifdef CREATESMART
+override FPCOPT+=-CX
+endif
+ifdef DEBUG
+override FPCOPT+=-gl
+override FPCOPTDEF+=DEBUG
+endif
+ifdef RELEASE
+ifneq ($(findstring 2.0.,$(FPC_VERSION)),)
+ifeq ($(CPU_TARGET),i386)
+FPCCPUOPT:=-OG2p3
+endif
+ifeq ($(CPU_TARGET),powerpc)
+FPCCPUOPT:=-O1r
+endif
+else
+FPCCPUOPT:=-O2
+endif
+override FPCOPT+=-Ur -Xs $(FPCCPUOPT) -n
+override FPCOPTDEF+=RELEASE
+endif
+ifdef STRIP
+override FPCOPT+=-Xs
+endif
+ifdef OPTIMIZE
+override FPCOPT+=-O2
+endif
+ifdef VERBOSE
+override FPCOPT+=-vwni
+endif
+ifdef COMPILER_OPTIONS
+override FPCOPT+=$(COMPILER_OPTIONS)
+endif
+ifdef COMPILER_UNITDIR
+override FPCOPT+=$(addprefix -Fu,$(COMPILER_UNITDIR))
+endif
+ifdef COMPILER_LIBRARYDIR
+override FPCOPT+=$(addprefix -Fl,$(COMPILER_LIBRARYDIR))
+endif
+ifdef COMPILER_OBJECTDIR
+override FPCOPT+=$(addprefix -Fo,$(COMPILER_OBJECTDIR))
+endif
+ifdef COMPILER_INCLUDEDIR
+override FPCOPT+=$(addprefix -Fi,$(COMPILER_INCLUDEDIR))
+endif
+ifdef CROSSBINDIR
+override FPCOPT+=-FD$(CROSSBINDIR)
+endif
+ifdef COMPILER_TARGETDIR
+override FPCOPT+=-FE$(COMPILER_TARGETDIR)
+ifeq ($(COMPILER_TARGETDIR),.)
+override TARGETDIRPREFIX=
+else
+override TARGETDIRPREFIX=$(COMPILER_TARGETDIR)/
+endif
+endif
+ifdef COMPILER_UNITTARGETDIR
+override FPCOPT+=-FU$(COMPILER_UNITTARGETDIR)
+ifeq ($(COMPILER_UNITTARGETDIR),.)
+override UNITTARGETDIRPREFIX=
+else
+override UNITTARGETDIRPREFIX=$(COMPILER_UNITTARGETDIR)/
+endif
+else
+ifdef COMPILER_TARGETDIR
+override COMPILER_UNITTARGETDIR=$(COMPILER_TARGETDIR)
+override UNITTARGETDIRPREFIX=$(TARGETDIRPREFIX)
+endif
+endif
+ifdef CREATESHARED
+override FPCOPT+=-Cg
+endif
+ifneq ($(findstring $(OS_TARGET),freebsd openbsd netbsd linux solaris),)
+ifeq ($(CPU_TARGET),x86_64)
+override FPCOPT+=-Cg
+endif
+endif
+ifdef LINKSHARED
+endif
+ifdef OPT
+override FPCOPT+=$(OPT)
+endif
+ifdef FPCOPTDEF
+override FPCOPT+=$(addprefix -d,$(FPCOPTDEF))
+endif
+ifdef CFGFILE
+override FPCOPT+=@$(CFGFILE)
+endif
+ifdef USEENV
+override FPCEXTCMD:=$(FPCOPT)
+override FPCOPT:=!FPCEXTCMD
+export FPCEXTCMD
+endif
+override AFULL_TARGET=$(CPU_TARGET)-$(OS_TARGET)
+override AFULL_SOURCE=$(CPU_SOURCE)-$(OS_SOURCE)
+ifneq ($(AFULL_TARGET),$(AFULL_SOURCE))
+override ACROSSCOMPILE=1
+endif
+ifdef ACROSSCOMPILE
+override FPCOPT+=$(CROSSOPT)
+endif
+override COMPILER:=$(FPC) $(FPCOPT)
+ifeq (,$(findstring -s ,$(COMPILER)))
+EXECPPAS=
+else
+ifeq ($(FULL_SOURCE),$(FULL_TARGET))
+ifdef RUNBATCH
+EXECPPAS:=@$(RUNBATCH) $(PPAS)
+else
+EXECPPAS:=@$(PPAS)
+endif
+endif
+endif
+.PHONY: fpc_exes
+ifndef CROSSINSTALL
+ifneq ($(TARGET_PROGRAMS),)
+override EXEFILES=$(addsuffix $(EXEEXT),$(TARGET_PROGRAMS))
+override EXEOFILES:=$(addsuffix $(OEXT),$(TARGET_PROGRAMS)) $(addprefix $(STATICLIBPREFIX),$(addsuffix $(STATICLIBEXT),$(TARGET_PROGRAMS))) $(addprefix $(IMPORTLIBPREFIX),$(addsuffix $(STATICLIBEXT),$(TARGET_PROGRAMS)))
+override EXEDBGFILES:=$(addsuffix $(EXEDBGEXT),$(TARGET_PROGRAMS))
+override ALLTARGET+=fpc_exes
+override INSTALLEXEFILES+=$(EXEFILES)
+override CLEANEXEFILES+=$(EXEFILES) $(EXEOFILES)
+override CLEANEXEDBGFILES+=$(EXEDBGFILES)
+ifeq ($(OS_TARGET),os2)
+override CLEANEXEFILES+=$(addsuffix $(AOUTEXT),$(TARGET_PROGRAMS))
+endif
+ifeq ($(OS_TARGET),emx)
+override CLEANEXEFILES+=$(addsuffix $(AOUTEXT),$(TARGET_PROGRAMS))
+endif
+endif
+endif
+fpc_exes: $(COMPILER_TARGETDIR) $(COMPILER_UNITTARGETDIR) $(EXEFILES)
+ifdef TARGET_RSTS
+override RSTFILES=$(addsuffix $(RSTEXT),$(TARGET_RSTS))
+override CLEANRSTFILES+=$(RSTFILES)
+endif
+.PHONY: fpc_all fpc_smart fpc_debug fpc_release fpc_shared
+$(FPCMADE): $(ALLDEPENDENCIES) $(ALLTARGET)
+	@$(ECHOREDIR) Compiled > $(FPCMADE)
+fpc_all: $(FPCMADE)
+fpc_smart:
+	$(MAKE) all LINKSMART=1 CREATESMART=1
+fpc_debug:
+	$(MAKE) all DEBUG=1
+fpc_release:
+	$(MAKE) all RELEASE=1
+.SUFFIXES: $(EXEEXT) $(PPUEXT) $(OEXT) .pas .lpr .dpr .pp .rc .res
+$(COMPILER_UNITTARGETDIR):
+	$(MKDIRTREE) $(COMPILER_UNITTARGETDIR)
+$(COMPILER_TARGETDIR):
+	$(MKDIRTREE) $(COMPILER_TARGETDIR)
+%$(PPUEXT): %.pp
+	$(COMPILER) $<
+	$(EXECPPAS)
+%$(PPUEXT): %.pas
+	$(COMPILER) $<
+	$(EXECPPAS)
+%$(EXEEXT): %.pp
+	$(COMPILER) $<
+	$(EXECPPAS)
+%$(EXEEXT): %.pas
+	$(COMPILER) $<
+	$(EXECPPAS)
+%$(EXEEXT): %.lpr
+	$(COMPILER) $<
+	$(EXECPPAS)
+%$(EXEEXT): %.dpr
+	$(COMPILER) $<
+	$(EXECPPAS)
+%.res: %.rc
+	windres -i $< -o $@
+vpath %.pp $(COMPILER_SOURCEDIR) $(COMPILER_INCLUDEDIR)
+vpath %.pas $(COMPILER_SOURCEDIR) $(COMPILER_INCLUDEDIR)
+vpath %.lpr $(COMPILER_SOURCEDIR) $(COMPILER_INCLUDEDIR)
+vpath %.dpr $(COMPILER_SOURCEDIR) $(COMPILER_INCLUDEDIR)
+vpath %.inc $(COMPILER_INCLUDEDIR)
+vpath %$(OEXT) $(COMPILER_UNITTARGETDIR)
+vpath %$(PPUEXT) $(COMPILER_UNITTARGETDIR)
+.PHONY: fpc_shared
+override INSTALLTARGET+=fpc_shared_install
+ifndef SHARED_LIBVERSION
+SHARED_LIBVERSION=$(FPC_VERSION)
+endif
+ifndef SHARED_LIBNAME
+SHARED_LIBNAME=$(PACKAGE_NAME)
+endif
+ifndef SHARED_FULLNAME
+SHARED_FULLNAME=$(SHAREDLIBPREFIX)$(SHARED_LIBNAME)-$(SHARED_LIBVERSION)$(SHAREDLIBEXT)
+endif
+ifndef SHARED_LIBUNITS
+SHARED_LIBUNITS:=$(TARGET_UNITS) $(TARGET_IMPLICITUNITS)
+override SHARED_LIBUNITS:=$(filter-out $(INSTALL_BUILDUNIT),$(SHARED_LIBUNITS))
+endif
+fpc_shared:
+ifdef HASSHAREDLIB
+	$(MAKE) all CREATESHARED=1 LINKSHARED=1 CREATESMART=1
+ifneq ($(SHARED_BUILD),n)
+	$(PPUMOVE) -q $(SHARED_LIBUNITS) -i$(COMPILER_UNITTARGETDIR) -o$(SHARED_FULLNAME) -d$(COMPILER_UNITTARGETDIR)
+endif
+else
+	@$(ECHO) Shared Libraries not supported
+endif
+fpc_shared_install:
+ifneq ($(SHARED_BUILD),n)
+ifneq ($(SHARED_LIBUNITS),)
+ifneq ($(wildcard $(COMPILER_UNITTARGETDIR)/$(SHARED_FULLNAME)),)
+	$(INSTALL) $(COMPILER_UNITTARGETDIR)/$(SHARED_FULLNAME) $(INSTALL_SHAREDDIR)
+endif
+endif
+endif
+.PHONY: fpc_install fpc_sourceinstall fpc_exampleinstall
+ifdef INSTALL_UNITS
+override INSTALLPPUFILES+=$(addsuffix $(PPUEXT),$(INSTALL_UNITS))
+endif
+ifdef INSTALL_BUILDUNIT
+override INSTALLPPUFILES:=$(filter-out $(INSTALL_BUILDUNIT)$(PPUEXT),$(INSTALLPPUFILES))
+endif
+ifdef INSTALLPPUFILES
+override INSTALLPPULINKFILES:=$(subst $(PPUEXT),$(OEXT),$(INSTALLPPUFILES)) $(addprefix $(STATICLIBPREFIX),$(subst $(PPUEXT),$(STATICLIBEXT),$(INSTALLPPUFILES))) $(addprefix $(IMPORTLIBPREFIX),$(subst $(PPUEXT),$(STATICLIBEXT),$(INSTALLPPUFILES)))
+ifneq ($(UNITTARGETDIRPREFIX),)
+override INSTALLPPUFILES:=$(addprefix $(UNITTARGETDIRPREFIX),$(notdir $(INSTALLPPUFILES)))
+override INSTALLPPULINKFILES:=$(wildcard $(addprefix $(UNITTARGETDIRPREFIX),$(notdir $(INSTALLPPULINKFILES))))
+endif
+override INSTALL_CREATEPACKAGEFPC=1
+endif
+ifdef INSTALLEXEFILES
+ifneq ($(TARGETDIRPREFIX),)
+override INSTALLEXEFILES:=$(addprefix $(TARGETDIRPREFIX),$(notdir $(INSTALLEXEFILES)))
+endif
+endif
+fpc_install: all $(INSTALLTARGET)
+ifdef INSTALLEXEFILES
+	$(MKDIR) $(INSTALL_BINDIR)
+	$(INSTALLEXE) $(INSTALLEXEFILES) $(INSTALL_BINDIR)
+endif
+ifdef INSTALL_CREATEPACKAGEFPC
+ifdef FPCMAKE
+ifdef PACKAGE_VERSION
+ifneq ($(wildcard Makefile.fpc),)
+	$(FPCMAKE) -p -T$(CPU_TARGET)-$(OS_TARGET) Makefile.fpc
+	$(MKDIR) $(INSTALL_UNITDIR)
+	$(INSTALL) Package.fpc $(INSTALL_UNITDIR)
+endif
+endif
+endif
+endif
+ifdef INSTALLPPUFILES
+	$(MKDIR) $(INSTALL_UNITDIR)
+	$(INSTALL) $(INSTALLPPUFILES) $(INSTALL_UNITDIR)
+ifneq ($(INSTALLPPULINKFILES),)
+	$(INSTALL) $(INSTALLPPULINKFILES) $(INSTALL_UNITDIR)
+endif
+ifneq ($(wildcard $(LIB_FULLNAME)),)
+	$(MKDIR) $(INSTALL_LIBDIR)
+	$(INSTALL) $(LIB_FULLNAME) $(INSTALL_LIBDIR)
+ifdef inUnix
+	ln -sf $(LIB_FULLNAME) $(INSTALL_LIBDIR)/$(LIB_NAME)
+endif
+endif
+endif
+ifdef INSTALL_FILES
+	$(MKDIR) $(INSTALL_DATADIR)
+	$(INSTALL) $(INSTALL_FILES) $(INSTALL_DATADIR)
+endif
+fpc_sourceinstall: distclean
+	$(MKDIR) $(INSTALL_SOURCEDIR)
+	$(COPYTREE) $(BASEDIR)/* $(INSTALL_SOURCEDIR)
+fpc_exampleinstall: $(addsuffix _distclean,$(TARGET_EXAMPLEDIRS))
+ifdef HASEXAMPLES
+	$(MKDIR) $(INSTALL_EXAMPLEDIR)
+endif
+ifdef EXAMPLESOURCEFILES
+	$(COPY) $(EXAMPLESOURCEFILES) $(INSTALL_EXAMPLEDIR)
+endif
+ifdef TARGET_EXAMPLEDIRS
+	$(COPYTREE) $(addsuffix /*,$(TARGET_EXAMPLEDIRS)) $(INSTALL_EXAMPLEDIR)
+endif
+.PHONY: fpc_clean fpc_cleanall fpc_distclean
+ifdef EXEFILES
+override CLEANEXEFILES:=$(addprefix $(TARGETDIRPREFIX),$(CLEANEXEFILES))
+override CLEANEXEDBGFILES:=$(addprefix $(TARGETDIRPREFIX),$(CLEANEXEDBGFILES))
+endif
+ifdef CLEAN_PROGRAMS
+override CLEANEXEFILES+=$(addprefix $(TARGETDIRPREFIX),$(addsuffix $(EXEEXT), $(CLEAN_PROGRAMS)))
+override CLEANEXEDBGFILES+=$(addprefix $(TARGETDIRPREFIX),$(addsuffix $(EXEDBGEXT), $(CLEAN_PROGRAMS)))
+endif
+ifdef CLEAN_UNITS
+override CLEANPPUFILES+=$(addsuffix $(PPUEXT),$(CLEAN_UNITS))
+endif
+ifdef CLEANPPUFILES
+override CLEANPPULINKFILES:=$(subst $(PPUEXT),$(OEXT),$(CLEANPPUFILES)) $(addprefix $(STATICLIBPREFIX),$(subst $(PPUEXT),$(STATICLIBEXT),$(CLEANPPUFILES))) $(addprefix $(IMPORTLIBPREFIX),$(subst $(PPUEXT),$(STATICLIBEXT),$(CLEANPPUFILES)))
+ifdef DEBUGSYMEXT
+override CLEANPPULINKFILES+=$(subst $(PPUEXT),$(DEBUGSYMEXT),$(CLEANPPUFILES))
+endif
+override CLEANPPUFILES:=$(addprefix $(UNITTARGETDIRPREFIX),$(CLEANPPUFILES))
+override CLEANPPULINKFILES:=$(wildcard $(addprefix $(UNITTARGETDIRPREFIX),$(CLEANPPULINKFILES)))
+endif
+fpc_clean: $(CLEANTARGET)
+ifdef CLEANEXEFILES
+	-$(DEL) $(CLEANEXEFILES)
+endif
+ifdef CLEANEXEDBGFILES
+	-$(DELTREE) $(CLEANEXEDBGFILES)
+endif
+ifdef CLEANPPUFILES
+	-$(DEL) $(CLEANPPUFILES)
+endif
+ifneq ($(CLEANPPULINKFILES),)
+	-$(DEL) $(CLEANPPULINKFILES)
+endif
+ifdef CLEANRSTFILES
+	-$(DEL) $(addprefix $(UNITTARGETDIRPREFIX),$(CLEANRSTFILES))
+endif
+ifdef CLEAN_FILES
+	-$(DEL) $(CLEAN_FILES)
+endif
+ifdef LIB_NAME
+	-$(DEL) $(LIB_NAME) $(LIB_FULLNAME)
+endif
+	-$(DEL) $(FPCMADE) Package.fpc $(PPAS) script.res link.res $(FPCEXTFILE) $(REDIRFILE)
+	-$(DEL) *$(ASMEXT) *_ppas$(BATCHEXT)
+fpc_cleanall: $(CLEANTARGET)
+ifdef CLEANEXEFILES
+	-$(DEL) $(CLEANEXEFILES)
+endif
+ifdef COMPILER_UNITTARGETDIR
+ifdef CLEANPPUFILES
+	-$(DEL) $(CLEANPPUFILES)
+endif
+ifneq ($(CLEANPPULINKFILES),)
+	-$(DEL) $(CLEANPPULINKFILES)
+endif
+ifdef CLEANRSTFILES
+	-$(DEL) $(addprefix $(UNITTARGETDIRPREFIX),$(CLEANRSTFILES))
+endif
+endif
+ifdef CLEAN_FILES
+	-$(DEL) $(CLEAN_FILES)
+endif
+	-$(DELTREE) units
+	-$(DEL) *$(OEXT) *$(PPUEXT) *$(RSTEXT) *$(ASMEXT) *$(STATICLIBEXT) *$(SHAREDLIBEXT) *$(PPLEXT)
+ifneq ($(PPUEXT),.ppu)
+	-$(DEL) *.o *.ppu *.a
+endif
+	-$(DELTREE) *$(SMARTEXT)
+	-$(DEL) fpcmade.* Package.fpc $(PPAS) script.res link.res $(FPCEXTFILE) $(REDIRFILE)
+	-$(DEL) *_ppas$(BATCHEXT)
+ifdef AOUTEXT
+	-$(DEL) *$(AOUTEXT)
+endif
+ifdef DEBUGSYMEXT
+	-$(DEL) *$(DEBUGSYMEXT)
+endif
+fpc_distclean: cleanall
+.PHONY: fpc_baseinfo
+override INFORULES+=fpc_baseinfo
+fpc_baseinfo:
+	@$(ECHO)
+	@$(ECHO)  == Package info ==
+	@$(ECHO)  Package Name..... $(PACKAGE_NAME)
+	@$(ECHO)  Package Version.. $(PACKAGE_VERSION)
+	@$(ECHO)
+	@$(ECHO)  == Configuration info ==
+	@$(ECHO)
+	@$(ECHO)  FPC.......... $(FPC)
+	@$(ECHO)  FPC Version.. $(FPC_VERSION)
+	@$(ECHO)  Source CPU... $(CPU_SOURCE)
+	@$(ECHO)  Target CPU... $(CPU_TARGET)
+	@$(ECHO)  Source OS.... $(OS_SOURCE)
+	@$(ECHO)  Target OS.... $(OS_TARGET)
+	@$(ECHO)  Full Source.. $(FULL_SOURCE)
+	@$(ECHO)  Full Target.. $(FULL_TARGET)
+	@$(ECHO)  SourceSuffix. $(SOURCESUFFIX)
+	@$(ECHO)  TargetSuffix. $(TARGETSUFFIX)
+	@$(ECHO)  FPC fpmake... $(FPCFPMAKE)
+	@$(ECHO)
+	@$(ECHO)  == Directory info ==
+	@$(ECHO)
+	@$(ECHO)  Required pkgs... $(REQUIRE_PACKAGES)
+	@$(ECHO)
+	@$(ECHO)  Basedir......... $(BASEDIR)
+	@$(ECHO)  FPCDir.......... $(FPCDIR)
+	@$(ECHO)  CrossBinDir..... $(CROSSBINDIR)
+	@$(ECHO)  UnitsDir........ $(UNITSDIR)
+	@$(ECHO)  PackagesDir..... $(PACKAGESDIR)
+	@$(ECHO)
+	@$(ECHO)  GCC library..... $(GCCLIBDIR)
+	@$(ECHO)  Other library... $(OTHERLIBDIR)
+	@$(ECHO)
+	@$(ECHO)  == Tools info ==
+	@$(ECHO)
+	@$(ECHO)  As........ $(AS)
+	@$(ECHO)  Ld........ $(LD)
+	@$(ECHO)  Ar........ $(AR)
+	@$(ECHO)  Rc........ $(RC)
+	@$(ECHO)
+	@$(ECHO)  Mv........ $(MVPROG)
+	@$(ECHO)  Cp........ $(CPPROG)
+	@$(ECHO)  Rm........ $(RMPROG)
+	@$(ECHO)  GInstall.. $(GINSTALL)
+	@$(ECHO)  Echo...... $(ECHO)
+	@$(ECHO)  Shell..... $(SHELL)
+	@$(ECHO)  Date...... $(DATE)
+	@$(ECHO)  FPCMake... $(FPCMAKE)
+	@$(ECHO)  PPUMove... $(PPUMOVE)
+	@$(ECHO)  Zip....... $(ZIPPROG)
+	@$(ECHO)
+	@$(ECHO)  == Object info ==
+	@$(ECHO)
+	@$(ECHO)  Target Loaders........ $(TARGET_LOADERS)
+	@$(ECHO)  Target Units.......... $(TARGET_UNITS)
+	@$(ECHO)  Target Implicit Units. $(TARGET_IMPLICITUNITS)
+	@$(ECHO)  Target Programs....... $(TARGET_PROGRAMS)
+	@$(ECHO)  Target Dirs........... $(TARGET_DIRS)
+	@$(ECHO)  Target Examples....... $(TARGET_EXAMPLES)
+	@$(ECHO)  Target ExampleDirs.... $(TARGET_EXAMPLEDIRS)
+	@$(ECHO)
+	@$(ECHO)  Clean Units......... $(CLEAN_UNITS)
+	@$(ECHO)  Clean Files......... $(CLEAN_FILES)
+	@$(ECHO)
+	@$(ECHO)  Install Units....... $(INSTALL_UNITS)
+	@$(ECHO)  Install Files....... $(INSTALL_FILES)
+	@$(ECHO)
+	@$(ECHO)  == Install info ==
+	@$(ECHO)
+	@$(ECHO)  DateStr.............. $(DATESTR)
+	@$(ECHO)  ZipName.............. $(ZIPNAME)
+	@$(ECHO)  ZipPrefix............ $(ZIPPREFIX)
+	@$(ECHO)  ZipCrossPrefix....... $(ZIPCROSSPREFIX)
+	@$(ECHO)  ZipSuffix............ $(ZIPSUFFIX)
+	@$(ECHO)  FullZipName.......... $(FULLZIPNAME)
+	@$(ECHO)  Install FPC Package.. $(INSTALL_FPCPACKAGE)
+	@$(ECHO)
+	@$(ECHO)  Install base dir..... $(INSTALL_BASEDIR)
+	@$(ECHO)  Install binary dir... $(INSTALL_BINDIR)
+	@$(ECHO)  Install library dir.. $(INSTALL_LIBDIR)
+	@$(ECHO)  Install units dir.... $(INSTALL_UNITDIR)
+	@$(ECHO)  Install source dir... $(INSTALL_SOURCEDIR)
+	@$(ECHO)  Install doc dir...... $(INSTALL_DOCDIR)
+	@$(ECHO)  Install example dir.. $(INSTALL_EXAMPLEDIR)
+	@$(ECHO)  Install data dir..... $(INSTALL_DATADIR)
+	@$(ECHO)
+	@$(ECHO)  Dist destination dir. $(DIST_DESTDIR)
+	@$(ECHO)  Dist zip name........ $(DIST_ZIPNAME)
+	@$(ECHO)
+.PHONY: fpc_info
+fpc_info: $(INFORULES)
+.PHONY: fpc_makefile fpc_makefiles fpc_makefile_sub1 fpc_makefile_sub2 \
+	fpc_makefile_dirs
+fpc_makefile:
+	$(FPCMAKE) -w -T$(OS_TARGET) Makefile.fpc
+fpc_makefile_sub1:
+ifdef TARGET_DIRS
+	$(FPCMAKE) -w -T$(OS_TARGET) $(addsuffix /Makefile.fpc,$(TARGET_DIRS))
+endif
+ifdef TARGET_EXAMPLEDIRS
+	$(FPCMAKE) -w -T$(OS_TARGET) $(addsuffix /Makefile.fpc,$(TARGET_EXAMPLEDIRS))
+endif
+fpc_makefile_sub2: $(addsuffix _makefile_dirs,$(TARGET_DIRS) $(TARGET_EXAMPLEDIRS))
+fpc_makefile_dirs: fpc_makefile_sub1 fpc_makefile_sub2
+fpc_makefiles: fpc_makefile fpc_makefile_dirs
+all: fpc_all
+debug: fpc_debug
+smart: fpc_smart
+release: fpc_release
+units: fpc_units
+examples:
+shared: fpc_shared
+install: fpc_install
+sourceinstall: fpc_sourceinstall
+exampleinstall: fpc_exampleinstall
+distinstall:
+zipinstall:
+zipsourceinstall:
+zipexampleinstall:
+zipdistinstall:
+clean: fpc_clean
+distclean: fpc_distclean
+cleanall: fpc_cleanall
+info: fpc_info
+makefiles: fpc_makefiles
+.PHONY: all debug smart release units examples shared install sourceinstall exampleinstall distinstall zipinstall zipsourceinstall zipexampleinstall zipdistinstall clean distclean cleanall info makefiles
+ifneq ($(wildcard fpcmake.loc),)
+include fpcmake.loc
+endif
+.NOTPARALLEL:
+cldrparser$(EXEEXT): cldrparser.lpr cldrhelper.pas helper.pas cldrtest.pas cldrxml.pas unicodeset.pas
+unihelper$(EXEEXT): unihelper.lpr helper.pas uca_test.pas

utils/unicode/Makefile.fpc

@@ -0,0 +1,23 @@
+#
+#   Makefile.fpc for Unicode Utils
+#
+
+[target]
+programs=cldrparser unihelper
+
+[clean]
+units=cldrhelper cldrtest cldrxml grbtree helper trie uca_test unicodeset
+
+[require]
+packages=rtl
+
+[install]
+fpcpackage=y
+
+[default]
+fpcdir=../..
+
+[rules]
+.NOTPARALLEL:
+cldrparser$(EXEEXT): cldrparser.lpr cldrhelper.pas helper.pas cldrtest.pas cldrxml.pas unicodeset.pas
+unihelper$(EXEEXT): unihelper.lpr helper.pas uca_test.pas

utils/unicode/cldrhelper.pas

@@ -0,0 +1,1634 @@
+{   CLDR collation helper unit.
+
+    Copyright (c) 2013 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+}
+
+unit cldrhelper;
+
+{$mode objfpc}
+{$H+}
+{$PACKENUM 1}
+{$modeswitch advancedrecords}
+{$scopedenums on}
+
+interface
+
+uses
+  SysUtils, Classes, helper;
+
+const
+  COLLATION_FILE_PREFIX = 'collation_';
+
+type
+
+  TUCA_LineRecArray = array of TUCA_LineRec;
+
+
+//----------------------------------------------------
+
+  ECldrException = class(Exception)
+  end;
+
+  TReorderWeigthKind = (
+    Primary, Secondary, Tertiary, Identity, Deletion
+  );
+  TReorderWeigthKinds = set of TReorderWeigthKind;
+  TReorderLogicalReset = (
+    None,// FirstVariable, LastVariable,
+    FirstTertiaryIgnorable, LastTertiaryIgnorable,
+    FirstSecondaryIgnorable, LastSecondaryIgnorable,
+    FirstPrimaryIgnorable, LastPrimaryIgnorable,
+    LastRegular,
+    FirstNonIgnorable, LastNonIgnorable,
+    FirstTrailing, LastTrailing
+  );
+  TCollationField = (BackWard, VariableLowLimit, VariableHighLimit);
+  TCollationFields = set of TCollationField;
+
+  { TReorderUnit }
+
+  TReorderUnit = packed record
+  public
+    Context         : TUnicodeCodePointArray;
+    ExpansionChars  : TUnicodeCodePointArray;
+    Characters      : TUnicodeCodePointArray;
+    WeigthKind      : TReorderWeigthKind;
+    InitialPosition : Integer;
+    Changed         : Boolean;
+  public
+    class function From(
+      const AChars,
+            AContext         : array of TUnicodeCodePoint;
+      const AWeigthKind      : TReorderWeigthKind;
+      const AInitialPosition : Integer
+    ) : TReorderUnit;static;overload;
+    class function From(
+      const AChars           : array of TUnicodeCodePoint;
+      const AWeigthKind      : TReorderWeigthKind;
+      const AInitialPosition : Integer
+    ) : TReorderUnit;static;overload;
+    class function From(
+      const AChar            : TUnicodeCodePoint;
+      const AWeigthKind      : TReorderWeigthKind;
+      const AInitialPosition : Integer
+    ) : TReorderUnit;static;overload;
+    class function From(
+      const AChar            : TUnicodeCodePoint;
+      const AContext         : array of TUnicodeCodePoint;
+      const AWeigthKind      : TReorderWeigthKind;
+      const AInitialPosition : Integer
+    ) : TReorderUnit;static;overload;
+    procedure SetExpansion(const AChars : array of TUnicodeCodePoint);
+    procedure SetExpansion(const AChar : TUnicodeCodePoint);
+    procedure Clear();
+    procedure Assign(const AItem : TReorderUnit);
+    function HasContext() : Boolean;
+    function IsExpansion() : Boolean;
+  end;
+
+  PReorderUnit = ^TReorderUnit;
+
+  { TReorderSequence }
+
+  TReorderSequence = packed record
+  public
+    Reset           : array of TUnicodeCodePoint;
+    Elements        : array of TReorderUnit;
+    LogicalPosition : TReorderLogicalReset;
+    Before          : Boolean;
+  public
+    procedure Clear();
+  end;
+  PReorderSequence = ^TReorderSequence;
+  TReorderSequenceArray = array of TReorderSequence;
+
+  { TOrderedCharacters }
+
+  TOrderedCharacters = record
+  private
+    FActualLength : Integer;
+  private
+    procedure EnsureSize(const AMinSize : Integer);
+  public
+    Data : array of TReorderUnit;
+    property ActualLength : Integer read FActualLength;
+
+  public
+    class function Create(const ACapacity : Integer) : TOrderedCharacters;static;overload;
+    class function Create() : TOrderedCharacters;static;overload;
+    procedure Clear();
+    function Clone() : TOrderedCharacters;
+    function Insert(const AItem : TReorderUnit; const ADestPos : Integer) : Integer;
+    function Append(const AItem : TReorderUnit) : Integer;
+    procedure Delete(const AIndex : Integer);
+
+    procedure ApplyStatement(const AStatement : PReorderSequence);
+  end;
+  POrderedCharacters = ^TOrderedCharacters;
+
+  TCldrCollation = class;
+
+  TCldrCollationItem = class
+  private
+    FBackwards: Boolean;
+    FBase: string;
+    FChangedFields: TCollationFields;
+    FParent: TCldrCollation;
+    FRules: TReorderSequenceArray;
+    FTypeName: string;
+  public
+    property Parent : TCldrCollation read FParent;
+    property TypeName : string read FTypeName write FTypeName;
+    property Base : string read FBase write FBase;
+    property Backwards : Boolean read FBackwards write FBackwards;
+    property Rules : TReorderSequenceArray read FRules write FRules;
+    property ChangedFields : TCollationFields read FChangedFields write FChangedFields;
+  end;
+
+  { TCldrCollation }
+
+  TCldrCollation = class
+  private
+    FItems : array of TCldrCollationItem;
+    FLocalID: string;
+    FDefaultType: string;
+    FVersion: string;
+    FLanguage: string;
+  private
+    function GetItem(Index : Integer): TCldrCollationItem;
+    function GetItemCount: Integer;
+  public
+    destructor Destroy();override;
+    procedure Clear();
+    function IndexOf(const AItemName : string) : Integer;
+    function Find(const AItemName : string) : TCldrCollationItem;
+    function Add(AItem : TCldrCollationItem) : Integer;
+    property Language : string read FLanguage write FLanguage;
+    property LocalID : string read FLocalID write FLocalID;
+    property Version : string read FVersion write FVersion;
+    property DefaultType : string read FDefaultType write FDefaultType;
+    property ItemCount : Integer read GetItemCount;
+    property Items[Index : Integer] : TCldrCollationItem read GetItem;
+  end;
+
+  function ComputeWeigths(
+    const AData        : PReorderUnit;
+    const ADataLen     : Integer;
+    const ADataWeigths : TUCA_LineRecArray;
+    out   AResult      : TUCA_LineRecArray
+  ) : Integer;
+  function FindCollationDefaultItemName(ACollation : TCldrCollation) : string;
+  procedure GenerateCdlrCollation(
+    ACollation    : TCldrCollation;
+    AItemName     : string;
+    AStoreName    : string;
+    AStream,
+    AEndianStream : TStream;
+    ARootChars    : TOrderedCharacters;
+    ARootWeigths  : TUCA_LineRecArray
+  );
+
+  procedure GenerateUCA_CLDR_Head(
+    ADest  : TStream;
+    ABook  : PUCA_DataBook;
+    AProps : PUCA_PropBook;
+    ACollation : TCldrCollationItem
+  );
+
+  function FillInitialPositions(
+          AData        : PReorderUnit;
+    const ADataLen     : Integer;
+    const ADataWeigths : TUCA_LineRecArray
+  ) : Integer;
+
+  function IndexOf(
+    const APattern        : array of TUnicodeCodePoint;
+    const APatternContext : array of TUnicodeCodePoint;
+    const ASequence       : PReorderUnit;
+    const ASequenceLength : Integer
+  ) : Integer;
+
+implementation
+uses
+  RtlConsts, typinfo;
+
+function ToStr(const ACharacters : array of TUnicodeCodePoint): string;
+var
+  i : Integer;
+begin
+  Result := '';
+  for i := Low(ACharacters) to High(ACharacters) do begin
+    if (ACharacters[i] > $FFFF) then
+      Result := Result + ' ' + IntToHex(ACharacters[i],5)
+    else
+      Result := Result + ' ' + IntToHex(ACharacters[i],4);
+  end;
+  Result := Trim(Result);
+end;
+
+function IndexOf(
+  const APattern        : array of TUnicodeCodePoint;
+  const APatternContext : array of TUnicodeCodePoint;
+  const ASequence       : PReorderUnit;
+  const ASequenceLength : Integer
+) : Integer;
+var
+  i, lp, sizep, lengthContext, sizeContext : Integer;
+  p : PReorderUnit;
+begin
+  Result := -1;
+  if (ASequenceLength = 0) then
+    exit;
+  lp := Length(APattern);
+  if (lp = 0) then
+    exit;
+  sizep := lp*SizeOf(TUnicodeCodePoint);
+  lengthContext := Length(APatternContext);
+  sizeContext := lengthContext*SizeOf(TUnicodeCodePoint);
+  p := ASequence;
+  for i := 0 to ASequenceLength - 1 do begin
+    if (Length(p^.Characters) = lp) then begin
+      if CompareMem(@APattern[0],@p^.Characters[0],sizep) then begin
+        if (Length(p^.Context) = lengthContext) and
+           ( (lengthContext = 0) or
+             CompareMem(@p^.Context[0],@APatternContext[0],sizeContext)
+           )
+        then begin
+          Result := i;
+          Break;
+        end;
+      end;
+    end;
+    Inc(p);
+  end;
+end;
+
+{procedure ApplyStatementToSequence(
+  var   ASequence  : TOrderedCharacters;
+  const AStatement : PReorderSequence;
+  const AStatementCount : Integer
+);
+var
+  pse, pd : PReorderUnit;
+  kr : Integer;
+
+  function GetNextInsertPos() : Integer;
+  var
+    kk : Integer;
+  begin
+    if (pse^.WeigthKind = rwkDeletion) then
+      exit(0);
+    if (pse^.WeigthKind = rwkIdentity) then
+      exit(kr + 1);
+    kk := kr + 1;
+    pd := @ASequence.Data[kk];
+    for kk := kk to ASequence.ActualLength - 1 do begin
+      if (pd^.WeigthKind <= pse^.WeigthKind) then
+        exit(kk);
+      Inc(pd);
+    end;
+    Result := ASequence.ActualLength;
+  end;
+
+var
+  locResetPos, i, k, h : Integer;
+  pst : PReorderSequence;
+begin
+  pst := AStatement;
+  for h := 0 to AStatementCount - 1 do begin
+    locResetPos := -1;
+    if (Length(pst^.Reset) > 0) then begin
+      locResetPos := IndexOf(pst^.Reset,[],@ASequence.Data[0],ASequence.ActualLength);
+      if (locResetPos = -1) then
+        raise ECldrException.CreateFmt('Character(s) not found in sequence : "%s".',[ToStr(pst^.Reset)]);
+    end;
+    pse := @pst^.Elements[0];
+    kr := locResetPos;
+    k := GetNextInsertPos();
+    for i := Low(pst^.Elements) to High(pst^.Elements) do begin
+      k := ASequence.Insert(pse^,k)+1;
+      Inc(pse);
+    end;
+    Inc(pst);
+  end;
+end;}
+function FindLogicalPos(
+  const ASequence  : POrderedCharacters;
+  const APosition  : TReorderLogicalReset
+) : Integer;
+var
+  i, c : Integer;
+  p : PReorderUnit;
+  firstPos, lastPos : Integer;
+begin
+  Result := 0;
+  if (ASequence^.ActualLength = 0) then
+    exit;
+  p := @ASequence^.Data[0];
+  c := ASequence^.ActualLength;
+  if (APosition in [TReorderLogicalReset.FirstTertiaryIgnorable, TReorderLogicalReset.LastTertiaryIgnorable])
+  then begin
+    firstPos := -1;
+    for i := 0 to c - 1 do begin
+      if (p^.WeigthKind <= TReorderWeigthKind.Tertiary) then begin
+        firstPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (firstPos = -1) then
+      exit(0);
+    if (APosition = TReorderLogicalReset.FirstTertiaryIgnorable) then
+      exit(firstPos);
+    if (p^.WeigthKind < TReorderWeigthKind.Tertiary) then
+      exit(firstPos);
+    lastPos := -1;
+    for i := firstPos + 1 to c - 1 do begin
+      if (p^.WeigthKind <> TReorderWeigthKind.Identity) then begin
+        lastPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (lastPos = -1) then
+      exit(c);
+    exit(lastPos);
+  end;
+  if (APosition in [TReorderLogicalReset.FirstSecondaryIgnorable, TReorderLogicalReset.LastSecondaryIgnorable])
+  then begin
+    firstPos := -1;
+    for i := 0 to c - 1 do begin
+      if (p^.WeigthKind <= TReorderWeigthKind.Secondary) then begin
+        firstPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (firstPos = -1) then
+      exit(0);
+    if (APosition = TReorderLogicalReset.FirstSecondaryIgnorable) then
+      exit(firstPos);
+    if (p^.WeigthKind < TReorderWeigthKind.Secondary) then
+      exit(firstPos);
+    lastPos := -1;
+    for i := firstPos + 1 to c - 1 do begin
+      if (p^.WeigthKind <> TReorderWeigthKind.Identity) then begin
+        lastPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (lastPos = -1) then
+      exit(c);
+    exit(lastPos);
+  end;
+  if (APosition in [TReorderLogicalReset.FirstPrimaryIgnorable, TReorderLogicalReset.LastPrimaryIgnorable])
+  then begin
+    firstPos := -1;
+    for i := 0 to c - 1 do begin
+      if (p^.WeigthKind <= TReorderWeigthKind.Primary) then begin
+        firstPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (firstPos = -1) then
+      exit(0);
+    if (APosition = TReorderLogicalReset.FirstPrimaryIgnorable) then
+      exit(firstPos);
+    if (p^.WeigthKind < TReorderWeigthKind.Primary) then
+      exit(firstPos);
+    lastPos := -1;
+    for i := firstPos + 1 to c - 1 do begin
+      if (p^.WeigthKind <> TReorderWeigthKind.Identity) then begin
+        lastPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (lastPos = -1) then
+      exit(c);
+    exit(lastPos);
+  end;
+  if (APosition = TReorderLogicalReset.FirstNonIgnorable) then begin
+    firstPos := -1;
+    for i := 0 to c - 1 do begin
+      if (p^.WeigthKind <= TReorderWeigthKind.Primary) then begin
+        firstPos := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+    if (firstPos = -1) then
+      exit(0);
+    exit(firstPos);
+  end;
+  if (APosition = TReorderLogicalReset.LastNonIgnorable) then
+    exit(c);
+end;
+
+procedure ApplyStatementToSequence(
+  var   ASequence  : TOrderedCharacters;
+  const AStatement : PReorderSequence;
+  const AStatementCount : Integer
+);
+var
+  pse, pd : PReorderUnit;
+  kr : Integer;
+  pst : PReorderSequence;
+
+  function GetNextInsertPos() : Integer;
+  var
+    kk : Integer;
+  begin
+    if (pse^.WeigthKind = TReorderWeigthKind.Deletion) then
+      exit(0);
+    if (pse^.WeigthKind = TReorderWeigthKind.Identity) then
+      exit(kr + 1);
+    if not pst^.Before then begin
+      kk := kr + 1;
+      pd := @ASequence.Data[kk];
+      for kk := kk to ASequence.ActualLength - 1 do begin
+        if (pd^.WeigthKind <= pse^.WeigthKind) then
+          exit(kk);
+        Inc(pd);
+      end;
+      Result := ASequence.ActualLength;
+    end else begin
+      if (kr = 0) then
+        exit(0);
+      kk := kr;
+      pd := @ASequence.Data[kk];
+      if (pd^.WeigthKind = TReorderWeigthKind.Primary) then begin
+        pd^.WeigthKind := pse^.WeigthKind;
+        pse^.WeigthKind := TReorderWeigthKind.Primary;
+        exit(kk);
+      end;
+      for kk := kk downto 0 do begin
+        if (pd^.WeigthKind = TReorderWeigthKind.Deletion) or (pd^.WeigthKind <= pse^.WeigthKind) then begin
+          if (pd^.WeigthKind > pse^.WeigthKind) then
+            pd^.WeigthKind := pse^.WeigthKind;
+          exit(kk);
+        end;
+        Dec(pd);
+      end;
+      Result := 0;
+    end;
+  end;
+
+var
+  locResetPos, i, k, h : Integer;
+begin
+  if (Length(AStatement^.Elements) = 0) then
+    exit;
+  pst := AStatement;
+  for h := 0 to AStatementCount - 1 do begin
+    locResetPos := -1;
+    if (AStatement^.LogicalPosition > TReorderLogicalReset.None) then
+      locResetPos := FindLogicalPos(@ASequence,AStatement^.LogicalPosition)
+    else if (Length(pst^.Reset) > 0) then begin
+      locResetPos := IndexOf(pst^.Reset,[],@ASequence.Data[0],ASequence.ActualLength);
+      {if (locResetPos = -1) then
+        raise ECldrException.CreateFmt('Character(s) not found in sequence : "%s".',[ToStr(pst^.Reset)]);}
+      if (locResetPos = -1) then
+        locResetPos := ASequence.ActualLength;
+    end;
+    pse := @pst^.Elements[0];
+    kr := locResetPos;
+    k := GetNextInsertPos();
+    for i := Low(pst^.Elements) to High(pst^.Elements) do begin
+      k := ASequence.Insert(pse^,k)+1;
+      Inc(pse);
+    end;
+    Inc(pst);
+  end;
+end;
+
+type
+  PUCA_WeightRecArray = ^TUCA_WeightRecArray;
+  TUCASortKey = array of Word;
+
+function SimpleFormKey(const ACEList : TUCA_WeightRecArray) : TUCASortKey;
+var
+  r : TUCASortKey;
+  i, c, k, ral, levelCount : Integer;
+  pce : PUCA_PropWeights;
+begin
+  c := Length(ACEList);
+  if (c = 0) then
+    exit(nil);
+  SetLength(r,((3+1{Level Separator})*c));
+  ral := 0;
+  levelCount := Length(ACEList[0].Weights);
+  for i := 0 to levelCount - 1 do begin
+    for k := 0 to c - 1 do begin
+      pce := @ACEList[k];
+      if (pce^.Weights[i] <> 0) then begin
+        r[ral] := pce^.Weights[i];
+        ral := ral + 1;
+      end;
+      //pce := pce + 1;
+    end;
+    r[ral] := 0;
+    ral := ral + 1;
+  end;
+  ral := ral - 1;
+  SetLength(r,ral);
+  Result := r;
+end;
+
+function CompareSortKey(const A, B : TUCASortKey) : Integer;
+var
+  i, hb : Integer;
+begin
+  if (Pointer(A) = Pointer(B)) then
+    exit(0);
+  Result := 1;
+  hb := Length(B) - 1;
+  for i := 0 to Length(A) - 1 do begin
+    if (i > hb) then
+      exit;
+    if (A[i] < B[i]) then
+      exit(-1);
+    if (A[i] > B[i]) then
+      exit(1);
+  end;
+  if (Length(A) = Length(B)) then
+    exit(0);
+  exit(-1);
+end;
+
+{function ComputeWeigths(
+  const AData        : PReorderUnit;
+  const ADataLen     : Integer;
+  const ADataWeigths : TUCA_LineRecArray;
+  out   AResult      : TUCA_LineRecArray
+) : Integer;
+
+  function GetWeigth(AItem : PReorderUnit) : PUCA_WeightRecArray;
+  begin
+    Result := nil;
+    if (AItem^.InitialPosition < 1) or (AItem^.InitialPosition > Length(ADataWeigths)) then
+      raise ECldrException.CreateFmt('Invalid "InitialPosition" value : %d.',[AItem^.InitialPosition]);
+    Result := @ADataWeigths[(AItem^.InitialPosition-1)].Weights;
+  end;
+
+var
+  c, i, ral : Integer;
+  p, q : PReorderUnit;
+  r : TUCA_LineRecArray;
+  pr : PUCA_LineRec;
+  pbase : PReorderUnit;
+  pw, pwb : PUCA_WeightRecArray;
+  cw, ki : Integer;
+begin
+  Result := 0;
+  if (ADataLen < 1) then
+    exit;
+  c := ADataLen;
+  ral := 0;
+  SetLength(r,c);
+  FillByte(r[0],(Length(r)*SizeOf(r[0])),0);
+  q := nil;
+  pbase := nil;
+  p := AData+1;
+  pr := @r[0];
+  i := 1;
+  while (i < c) do begin
+    if p^.Changed then begin
+      if (pbase = nil) then begin
+        pbase := p - 1;
+        pwb := GetWeigth(pbase);
+      end;
+      if (p^.WeigthKind = rwkIdentity) then begin
+        pr^.CodePoints := Copy(p^.Characters);
+        q := p - 1;
+        if (q = pbase) then
+          pw := pwb
+        else
+          pw := @((pr-1)^.Weights);
+        pr^.Weights := Copy(pw^);
+        Inc(pr);
+        Inc(ral);
+      end else begin
+        pr^.CodePoints := Copy(p^.Characters);
+        q := p - 1;
+        if (q = pbase) then begin
+          pw := pwb;
+          cw := (Length(pw^)+1);
+          SetLength(pr^.Weights,cw);
+          Move(pw^[0],pr^.Weights[0],((cw-1)*SizeOf(pw^[0])));
+          FillByte(pr^.Weights[(cw-1)],SizeOf(pr^.Weights[0]),0);
+          ki := Ord(p^.WeigthKind);
+          pr^.Weights[(cw-1)].Weights[ki] := pr^.Weights[(cw-2)].Weights[ki]+1;
+        end else begin
+          pw := @((pr-1)^.Weights);
+          pr^.Weights := Copy(pw^);
+          cw := Length(pr^.Weights);
+          ki := Ord(p^.WeigthKind);
+          for ki := Ord(rwkPrimary) to Ord(rwkTertiary) do begin
+            if (ki < Ord(p^.WeigthKind)) then
+              pr^.Weights[(cw-1)].Weights[ki] := pw^[(cw-1)].Weights[ki]
+            else if (ki = Ord(p^.WeigthKind)) then begin
+              if (pw^[(cw-1)].Weights[ki] = 0) then
+                pr^.Weights[(cw-1)].Weights[ki] := pwb^[(Length(pwb^)-1)].Weights[ki]+1
+              else
+                pr^.Weights[(cw-1)].Weights[ki] := pw^[(cw-1)].Weights[ki]+1;
+            end else begin
+              pr^.Weights[(cw-1)].Weights[ki] := 0;
+            end;
+          end;
+        end;
+        Inc(pr);
+        Inc(ral);
+      end;
+    end else begin
+      pbase := nil;
+      pwb := nil;
+    end;
+    Inc(p);
+    Inc(i);
+  end;
+  SetLength(r,ral);
+  AResult := r;
+  Result := Length(AResult);
+end;}
+function IndexOf(
+  const APattern : array of TUnicodeCodePoint;
+  const AList    : PUCA_LineRec;
+  const AListLen : Integer
+) : Integer;
+var
+  i, lengthPattern, sizePattern : Integer;
+  pl : PUCA_LineRec;
+begin
+  Result := -1;
+  if (Length(APattern) = 0) then
+    exit;
+  if (AListLen = 0) then
+    exit;
+  lengthPattern := Length(APattern);
+  sizePattern := lengthPattern*SizeOf(TUnicodeCodePoint);
+  pl := AList;
+  for i := 0 to AListLen - 1 do begin
+    if (Length(pl^.CodePoints) = lengthPattern) and
+       CompareMem(@pl^.CodePoints[0],@APattern[0],sizePattern)
+    then begin
+      Result := i;
+      Break;
+    end;
+    Inc(pl);
+  end;
+end;
+
+function Compress(
+  const AData   : TUCA_LineRecArray;
+  out   AResult : TUCA_LineRecArray
+) : Boolean;
+var
+  r : TUCA_LineRecArray;
+  pr, p : PUCA_LineRec;
+  ral : Integer;
+
+  function FindOutSlot() : Boolean;
+  var
+    k : Integer;
+  begin
+    k := IndexOf(p^.CodePoints,@r[0],ral);
+    Result := (k >= 0);
+    if (k = -1) then begin
+      k := ral;
+      ral := ral + 1;
+    end;
+    pr := @r[k];
+  end;
+
+  procedure AddContextData();
+  var
+    k : Integer;
+  begin
+    if not p^.HasContext() then
+      exit;
+    k := Length(pr^.Context.Data);
+    SetLength(pr^.Context.Data,(k+1));
+    pr^.Context.Data[k].CodePoints := Copy(p^.Context.Data[0].CodePoints);
+    pr^.Context.Data[k].Weights := Copy(p^.Weights);
+  end;
+
+  procedure AddItem();
+  begin
+    pr^.Assign(p^);
+    if p^.HasContext() then begin
+      SetLength(pr^.Context.Data,0);
+      pr^.Weights := nil;
+      AddContextData();
+    end;
+  end;
+
+var
+  c, i : Integer;
+begin
+  c := Length(AData);
+  if (c = 0) then
+    exit;
+  SetLength(r,c);
+  FillByte(r[0],(Length(r)*SizeOf(r[0])),0);
+  pr := @r[0];
+  p := @AData[0];
+  ral := 0;
+  i := 0;
+  AddItem();
+  ral := 1;
+  i := 1;
+  Inc(p);
+  while (i < c) do begin
+    if FindOutSlot() then
+      AddContextData()
+    else
+      AddItem();
+    Inc(p);
+    Inc(i);
+  end;
+  SetLength(r,ral);
+  AResult := r;
+  Result := (ral < Length(AData));
+end;
+
+function MarkSuffixAsChanged(
+  const AData : PReorderUnit;
+  const ADataLen : Integer
+) : Integer;
+var
+  i, k : Integer;
+  p, q : PReorderUnit;
+  suffixChar : TUnicodeCodePoint;
+begin
+  Result := 0;
+  if (ADataLen <= 1) then
+    exit;
+  q := AData;
+  p := AData;
+  for i := 0 to ADataLen - 1 do begin
+    if p^.Changed then begin
+      suffixChar := p^.Characters[0];
+      for k := 0 to ADataLen - 1 do begin
+        if not(q[k].Changed) and (q[k].Characters[0] = suffixChar) then begin
+          q[k].Changed := True;
+          Result := Result + 1;
+        end;
+      end;
+    end;
+    Inc(p);
+  end;
+end;
+
+{$include weight_derivation.inc}
+
+function ComputeWeigths(
+  const AData        : PReorderUnit;
+  const ADataLen     : Integer;
+  const ADataWeigths : TUCA_LineRecArray;
+  out   AResult      : TUCA_LineRecArray
+) : Integer;
+
+  function GetWeigth(AItem : PReorderUnit) : PUCA_WeightRecArray;
+  begin
+    Result := nil;
+    if (AItem^.InitialPosition < 1) or (AItem^.InitialPosition > Length(ADataWeigths)) then
+      raise ECldrException.CreateFmt('Invalid "InitialPosition" value : %d.',[AItem^.InitialPosition]);
+    Result := @ADataWeigths[(AItem^.InitialPosition-1)].Weights;
+  end;
+
+var
+  r : TUCA_LineRecArray;
+  pr : PUCA_LineRec;
+
+  procedure AddContext(const ACodePointPattern : TUnicodeCodePointArray);
+  var
+    k : Integer;
+  begin
+    k := Length(pr^.Context.Data);
+    SetLength(pr^.Context.Data,(k+1));
+    pr^.Context.Data[k].CodePoints := Copy(ACodePointPattern);
+    SetLength(pr^.Context.Data[k].Weights,0);
+  end;
+
+var
+  ral : Integer;
+  i : Integer;
+  p : PReorderUnit;
+  pbase : PReorderUnit;
+  pwb : PUCA_WeightRecArray;
+  actualBegin : Boolean;
+  loopIndex : Integer;
+
+  procedure SkipDeletion();
+  begin
+    pr^.CodePoints := Copy(p^.Characters);
+    pr^.Deleted := True;
+    SetLength(pr^.Weights,0);
+    if p^.HasContext() then
+      AddContext(p^.Context);
+    Inc(pr);
+    Inc(ral);
+    Inc(p);
+    Inc(i);
+  end;
+
+  procedure FindBaseItem();
+  begin
+    if (pbase = nil) or (pwb^ = nil) then begin
+      if actualBegin then begin
+        pwb := @ADataWeigths[0].Weights;
+      end else begin
+        pbase := p - 1;
+        if pbase^.Changed then
+          pwb := @((pr-1)^.Weights)
+        else
+          pwb := GetWeigth(pbase);
+        if (pwb^ = nil) and (pbase = AData) then
+          pwb := @ADataWeigths[0].Weights;
+      end;
+    end;
+  end;
+
+  function InternalComputeWeights(const AList : array of TUnicodeCodePointArray) : TUCA_WeightRecArray;
+  var
+    kral : Integer;
+    kres : TUCA_WeightRecArray;
+
+    procedure EnsureResultLength(const APlus : Integer);inline;
+    begin
+      if ((kral+APlus) > Length(kres)) then
+        SetLength(kres,(2*(kral+APlus)));
+    end;
+
+    procedure AddToResult(const AValue : TUCA_WeightRecArray);inline;
+    begin
+      EnsureResultLength(Length(AValue));
+      Move(AValue[0],kres[kral],(Length(AValue)*SizeOf(kres[0])));
+      kral := kral + Length(AValue);
+    end;
+
+  var
+    kc, k, ktempIndex, ki : Integer;
+  begin
+    kc := Length(AList);
+    kral := 0;
+    SetLength(kres,(10*kc));
+    FillChar(kres[0],(Length(kres)*SizeOf(kres[0])),0);
+    for k := 0 to kc - 1 do begin
+      ktempIndex := IndexOf(AList[k],@r[0],ral);
+      if (ktempIndex <> -1) then begin
+        AddToResult(r[ktempIndex].Weights);
+        Continue;
+      end;
+      ktempIndex := IndexOf(AList[k],[],AData,ADataLen);
+      if (ktempIndex <> -1) then begin
+        if not AData[ktempIndex].Changed then begin
+          AddToResult(ADataWeigths[AData[ktempIndex].InitialPosition-1].Weights);
+          Continue;
+        end;
+      end;
+      if (Length(AList[k]) > 1) then begin
+        for ki := 0 to Length(AList[k]) - 1 do begin
+          ktempIndex := IndexOf([AList[k][ki]],@r[0],ral);
+          if (ktempIndex <> -1) then begin
+            AddToResult(r[ktempIndex].Weights);
+            Continue;
+          end;
+          ktempIndex := IndexOf([AList[k][ki]],[],AData,ADataLen);
+          if (ktempIndex <> -1) then begin
+            if not AData[ktempIndex].Changed then begin
+              AddToResult(ADataWeigths[AData[ktempIndex].InitialPosition-1].Weights);
+              Continue;
+            end;
+          end;
+          EnsureResultLength(2);
+          DeriveWeight(AList[k][ki],@kres[kral]);
+          kral := kral + 2;
+        end
+      end;
+      EnsureResultLength(2);
+      DeriveWeight(AList[k][0],@kres[kral]);
+      kral := kral + 2;
+    end;
+    SetLength(kres,kral);
+    Result := kres;
+  end;
+
+  procedure Handle_Expansion();
+  var
+    expChars : array[0..1] of TUnicodeCodePointArray;
+    kres : TUCA_WeightRecArray;
+  begin
+    expChars[0] := (p-1)^.Characters;
+    expChars[1] := p^.ExpansionChars;
+    kres := InternalComputeWeights(expChars);
+    if (p^.WeigthKind <= TReorderWeigthKind.Tertiary) then
+      Inc(kres[Length(kres)-1].Weights[Ord(p^.WeigthKind)]);
+    pr^.Weights := Copy(kres);
+  end;
+
+var
+  c, ti : Integer;
+  q : PReorderUnit;
+  pw : PUCA_WeightRecArray;
+begin
+  Result := 0;
+  if (ADataLen < 1) then
+    exit;
+  while True do begin
+    for loopIndex := 0 to 1 do begin
+      c := ADataLen;
+      ral := 0;
+      SetLength(r,c);
+      FillByte(r[0],(Length(r)*SizeOf(r[0])),0);
+      q := nil;
+      pbase := nil;
+      pr := @r[0];
+      p := AData;
+      i := 0;
+      while (i < c) do begin
+        if (p^.WeigthKind = TReorderWeigthKind.Deletion) then begin
+          SkipDeletion();
+          Continue;
+        end;
+        if p^.Changed then begin
+          actualBegin := (i = 0) or (((p-1)^.WeigthKind = TReorderWeigthKind.Deletion));
+          FindBaseItem();
+          if p^.IsExpansion() then begin
+            if (loopIndex = 0) then begin
+              Inc(p);
+              Inc(i);
+              while (i < c) do begin
+                if (p^.WeigthKind = TReorderWeigthKind.Primary) then
+                  Break;
+                Inc(p);
+                Inc(i);
+              end;
+              Continue;
+            end;
+            pr^.CodePoints := Copy(p^.Characters);
+            Handle_Expansion();
+            if p^.HasContext() then
+              AddContext(p^.Context);
+            Inc(pr);
+            Inc(ral);
+          end else if actualBegin then begin
+            pr^.CodePoints := Copy(p^.Characters);
+            pw := pwb;
+            pr^.Weights := Copy(pw^);
+            if p^.HasContext() then
+              AddContext(p^.Context);
+            Inc(pr);
+            Inc(ral);
+          end else if (p^.WeigthKind = TReorderWeigthKind.Identity) then begin
+            pr^.CodePoints := Copy(p^.Characters);
+            q := p - 1;
+            if (q = pbase) then
+              pw := pwb
+            else
+              pw := @((pr-1)^.Weights);
+            pr^.Weights := Copy(pw^);
+            if p^.HasContext() then
+              AddContext(p^.Context);
+            Inc(pr);
+            Inc(ral);
+          end else begin
+            pr^.CodePoints := Copy(p^.Characters);
+            if ((p - 1) = pbase) then begin
+              if (p^.WeigthKind = TReorderWeigthKind.Primary) then begin
+                SetLength(pr^.Weights,2);
+                FillByte(pr^.Weights[0],SizeOf(pr^.Weights[0]),0);
+                pr^.Weights[0].Weights[0] := (pwb^[0].Weights[0] + 1);
+                pr^.Weights[0].Variable := pwb^[0].Variable;
+                pr^.Weights[1] := pr^.Weights[0];
+              end else if (p^.WeigthKind = TReorderWeigthKind.Secondary) then begin
+                SetLength(pr^.Weights,2);
+                FillByte(pr^.Weights[0],SizeOf(pr^.Weights[0]),0);
+                pr^.Weights[0].Weights[0] := pwb^[0].Weights[0];
+                pr^.Weights[0].Weights[1] := (pwb^[0].Weights[1] + 1);
+                pr^.Weights[0].Variable := pwb^[0].Variable;
+                pr^.Weights[1].Weights[0] := pr^.Weights[0].Weights[0];
+                pr^.Weights[1].Variable := pr^.Weights[0].Variable;
+              end else if (p^.WeigthKind = TReorderWeigthKind.Tertiary) then begin
+                SetLength(pr^.Weights,2);
+                FillByte(pr^.Weights[0],SizeOf(pr^.Weights[0]),0);
+                pr^.Weights[0].Weights[0] := pwb^[0].Weights[0];
+                pr^.Weights[0].Weights[1] := pwb^[0].Weights[1];
+                pr^.Weights[0].Weights[2] := (pwb^[0].Weights[2] + 1);
+                pr^.Weights[0].Variable := pwb^[0].Variable;
+                pr^.Weights[1].Weights[0] := pr^.Weights[0].Weights[0];
+                pr^.Weights[1].Variable := pr^.Weights[0].Variable;
+              end;
+            end else begin
+              pr^.Weights := Copy((pr-1)^.Weights);
+              if (p^.WeigthKind = TReorderWeigthKind.Primary) then
+                Inc(pr^.Weights[1].Weights[Ord(p^.WeigthKind)])
+              else
+                Inc(pr^.Weights[0].Weights[Ord(p^.WeigthKind)]);
+            end;
+            if p^.HasContext() then
+              AddContext(p^.Context);
+            Inc(pr);
+            Inc(ral);
+          end;
+        end else begin
+          if (i > 0) and ((p-1)^.WeigthKind <> TReorderWeigthKind.Deletion) and (p-1)^.Changed and
+             (ral > 0)
+          then begin
+            pw := GetWeigth(p);
+            ti := CompareSortKey(SimpleFormKey((pr-1)^.Weights),SimpleFormKey(pw^));
+            if ( (p^.WeigthKind = TReorderWeigthKind.Identity) and (ti > 0) ) or
+               ( (p^.WeigthKind >= TReorderWeigthKind.Primary) and (ti >= 0) )
+            then begin
+              p^.Changed := True;
+              Continue;
+            end;
+          end;
+          pbase := nil;
+          pwb := nil;
+        end;
+        Inc(p);
+        Inc(i);
+      end;
+    end;
+    SetLength(r,ral);
+    if (MarkSuffixAsChanged(AData,ADataLen) = 0) then
+      Break;
+  end;
+  Compress(r,AResult);
+  Result := Length(AResult);
+end;
+
+function FillInitialPositions(
+        AData        : PReorderUnit;
+  const ADataLen     : Integer;
+  const ADataWeigths : TUCA_LineRecArray
+) : Integer;
+var
+  locNotFound, i, cw : Integer;
+  p : PReorderUnit;
+  pw : PUCA_LineRec;
+begin
+  locNotFound := 0;
+  cw := Length(ADataWeigths);
+  if (cw > 0) then
+    pw := @ADataWeigths[0]
+  else
+    pw := nil;
+  p := AData;
+  for i := 0 to ADataLen - 1 do begin
+    p^.InitialPosition := IndexOf(p^.Characters,pw,cw) + 1;
+    if (p^.InitialPosition = 0) then
+      Inc(locNotFound);
+    Inc(p);
+  end;
+  Result := locNotFound;
+end;
+
+{ TCldrCollation }
+
+function TCldrCollation.GetItem(Index : Integer): TCldrCollationItem;
+begin
+  if (Index < 0) or (Index >= Length(FItems)) then
+    raise ERangeError.CreateFmt(SListIndexError,[Index]);
+  Result := FItems[Index];
+end;
+
+function TCldrCollation.GetItemCount: Integer;
+begin
+  Result := Length(FItems);
+end;
+
+destructor TCldrCollation.Destroy;
+begin
+  Clear();
+  inherited Destroy;
+end;
+
+procedure TCldrCollation.Clear();
+var
+  i : Integer;
+begin
+  for i := 0 to Length(FItems) - 1 do
+    FreeAndNil(FItems[i]);
+  SetLength(FItems,0);
+  FLocalID := '';
+  FDefaultType := '';
+end;
+
+function TCldrCollation.IndexOf(const AItemName: string): Integer;
+var
+  i : Integer;
+begin
+  for i := 0 to ItemCount - 1 do begin
+    if SameText(AItemName,Items[i].TypeName) then
+      exit(i);
+  end;
+  Result := -1;
+end;
+
+function TCldrCollation.Find(const AItemName: string): TCldrCollationItem;
+var
+  i : Integer;
+begin
+  i := IndexOf(AItemName);
+  if (i = - 1) then
+    Result := nil
+  else
+    Result := Items[i];
+end;
+
+function TCldrCollation.Add(AItem: TCldrCollationItem): Integer;
+begin
+  Result := Length(FItems);
+  SetLength(FItems,(Result+1));
+  FItems[Result] := AItem;
+  AItem.FParent := Self;
+end;
+
+{ TReorderSequence }
+
+procedure TReorderSequence.Clear();
+begin
+  Reset    := nil;
+  Elements := nil;
+  LogicalPosition := TReorderLogicalReset(0);
+  Before   := False;
+end;
+
+{ TReorderUnit }
+
+class function TReorderUnit.From(
+  const AChars,
+        AContext         : array of TUnicodeCodePoint;
+  const AWeigthKind      : TReorderWeigthKind;
+  const AInitialPosition : Integer
+) : TReorderUnit;
+var
+  c : Integer;
+begin
+  c := Length(AChars);
+  SetLength(Result.Characters,c);
+  if (c > 0) then
+    Move(AChars[0],Result.Characters[0],(c*SizeOf(Result.Characters[0])));
+  Result.WeigthKind := AWeigthKind;
+  Result.InitialPosition := AInitialPosition;
+  Result.Changed := False;
+  c := Length(AContext);
+  SetLength(Result.Context,c);
+  if (c > 0) then
+    Move(AContext[0],Result.Context[0],(c*SizeOf(Result.Context[0])));
+end;
+
+class function TReorderUnit.From(
+  const AChars           : array of TUnicodeCodePoint;
+  const AWeigthKind      : TReorderWeigthKind;
+  const AInitialPosition : Integer
+) : TReorderUnit;
+begin
+  Result := From(AChars,[],AWeigthKind,AInitialPosition);
+end;
+
+class function TReorderUnit.From(
+  const AChar            : TUnicodeCodePoint;
+  const AWeigthKind      : TReorderWeigthKind;
+  const AInitialPosition : Integer
+) : TReorderUnit;
+begin
+  Result := From([AChar],AWeigthKind,AInitialPosition);
+end;
+
+class function TReorderUnit.From(
+  const AChar            : TUnicodeCodePoint;
+  const AContext         : array of TUnicodeCodePoint;
+  const AWeigthKind      : TReorderWeigthKind;
+  const AInitialPosition : Integer
+) : TReorderUnit;
+begin
+  Result := From([AChar],AContext,AWeigthKind,AInitialPosition);
+end;
+
+procedure TReorderUnit.SetExpansion(const AChars: array of TUnicodeCodePoint);
+var
+  c : Integer;
+begin
+  c := Length(AChars);
+  SetLength(ExpansionChars,c);
+  if (c > 0) then
+    Move(AChars[0],ExpansionChars[0],(c*SizeOf(AChars[0])));
+end;
+
+procedure TReorderUnit.SetExpansion(const AChar: TUnicodeCodePoint);
+begin
+  SetExpansion([AChar]);
+end;
+
+procedure TReorderUnit.Clear();
+begin
+  Self.Characters := nil;
+  Self.Context := nil;
+  Self.ExpansionChars := nil;
+  Self.InitialPosition := 0;
+  Self.WeigthKind := TReorderWeigthKind(0);
+  Self.Changed := False;
+end;
+
+procedure TReorderUnit.Assign(const AItem : TReorderUnit);
+begin
+  Clear();
+  Self.Characters := Copy(AItem.Characters);
+  //SetLength(Self.Context,Length(AItem.Context));
+  Self.Context := Copy(AItem.Context);
+  Self.ExpansionChars := Copy(AItem.ExpansionChars);
+  Self.WeigthKind := AItem.WeigthKind;
+  Self.InitialPosition := AItem.InitialPosition;
+  Self.Changed := AItem.Changed;
+end;
+
+function TReorderUnit.HasContext() : Boolean;
+begin
+  Result := (Length(Context) > 0);
+end;
+
+function TReorderUnit.IsExpansion() : Boolean;
+begin
+  Result := (Length(ExpansionChars) > 0);
+end;
+
+{ TOrderedCharacters }
+
+procedure TOrderedCharacters.EnsureSize(const AMinSize : Integer);
+var
+  c : Integer;
+begin
+  if (AMinSize > Length(Data)) then begin
+    if (AMinSize > 1000) then
+      c := AMinSize + 100
+    else
+      c := (3*AMinSize) div 2 ;
+    SetLength(Data,c);
+  end;
+  FActualLength := AMinSize;
+end;
+
+class function TOrderedCharacters.Create(const ACapacity : Integer) : TOrderedCharacters;
+begin
+  if (ACapacity < 0) then
+    raise ERangeError.Create(SRangeError);
+  Result.FActualLength := 0;
+  SetLength(Result.Data,ACapacity);
+end;
+
+class function TOrderedCharacters.Create() : TOrderedCharacters;
+begin
+  Result := Create(0);
+end;
+
+procedure TOrderedCharacters.Clear;
+begin
+  Data := nil;
+  FActualLength := 0;
+end;
+
+function TOrderedCharacters.Clone() : TOrderedCharacters;
+var
+  i : Integer;
+begin
+  Result.Clear();
+  SetLength(Result.Data,Self.ActualLength);
+  for i := 0 to Length(Result.Data) - 1 do
+    Result.Data[i].Assign(Self.Data[i]);
+  Result.FActualLength := Self.FActualLength;
+end;
+
+function TOrderedCharacters.Insert(
+  const AItem    : TReorderUnit;
+  const ADestPos : Integer
+) : Integer;
+var
+  k, finalPos : Integer;
+  p : PReorderUnit;
+  i, c : Integer;
+begin
+  k := IndexOf(AItem.Characters,AItem.Context,@Data[0],ActualLength);
+  if (k = ADestPos) then begin
+    Data[ADestPos].Assign(AItem);
+    Data[ADestPos].Changed := True;
+    exit(k);
+  end;
+  finalPos := ADestPos;
+  if (finalPos > ActualLength) then
+    finalPos := ActualLength;
+  c := ActualLength;
+  EnsureSize(ActualLength + 1);
+  Data[c].Clear();
+  p := @Data[finalPos];
+  if (finalPos = ActualLength) then begin
+    p^.Assign(AItem);
+    p^.Changed := True;
+  end else begin
+    p := @Data[c-1];
+    for i := finalPos to c - 1 do begin
+      Move(p^,(p+1)^,SizeOf(p^));
+      Dec(p);
+    end;
+    p := @Data[finalPos];
+
+    {Move(
+      Pointer(p)^,Pointer(@p[1])^,
+      (ActualLength-(finalPos+1))*SizeOf(TReorderUnit)
+    );}
+    FillChar(Pointer(p)^,SizeOf(TReorderUnit),0);
+    p^.Assign(AItem);
+    p^.Changed := True;
+  end;
+  if (k >= 0) then begin
+    if (k > finalPos) then
+      Inc(k);
+    Delete(k);
+  end;
+  Result := finalPos;
+end;
+
+function TOrderedCharacters.Append(const AItem : TReorderUnit) : Integer;
+begin
+  Result := Insert(AItem,ActualLength);
+end;
+
+procedure TOrderedCharacters.Delete(const AIndex : Integer);
+var
+  i : Integer;
+  p : PReorderUnit;
+begin
+  if (AIndex < 0) or (AIndex >= ActualLength) then
+    raise ERangeError.CreateFmt(SListIndexError,[AIndex]);
+  if (AIndex = (ActualLength-1)) then begin
+    Data[AIndex].Clear();
+  end else begin
+    //Data[AIndex].Clear();
+    p := @Data[AIndex];
+    p^.Clear();
+    for i := AIndex to ActualLength-2 do begin
+      Move((p+1)^,p^,SizeOf(p^));
+      Inc(p);
+    end;
+    {Move(
+      Pointer(@Data[(AIndex+1)])^,Pointer(@Data[AIndex])^,
+      (ActualLength-(AIndex+1))*SizeOf(TReorderUnit)
+    );}
+    FillChar(Pointer(@Data[(FActualLength-1)])^,SizeOf(TReorderUnit),0);
+  end;
+  FActualLength := FActualLength - 1;
+end;
+
+procedure TOrderedCharacters.ApplyStatement(const AStatement : PReorderSequence);
+begin
+  ApplyStatementToSequence(Self,AStatement,1);
+end;
+
+function FindCollationDefaultItemName(ACollation : TCldrCollation) : string;
+begin
+  if (ACollation.ItemCount = 0) then
+    exit('');
+  if (ACollation.IndexOf(ACollation.DefaultType) <> -1) then
+    exit(ACollation.DefaultType);
+  Result := 'standard';
+  if (ACollation.IndexOf(Result) <> -1) then
+    exit;
+  Result := 'search';
+  if (ACollation.IndexOf(Result) <> -1) then
+    exit;
+  if (ACollation.ItemCount > 0) then
+    Result := ACollation.Items[0].TypeName;
+end;
+
+procedure GenerateUCA_CLDR_Head(
+  ADest  : TStream;
+  ABook  : PUCA_DataBook;
+  AProps : PUCA_PropBook;
+  ACollation : TCldrCollationItem
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+  procedure AddFields();
+  var
+    kc : Integer;
+    e : TCollationField;
+    ks : string;
+    ti : PTypeInfo;
+  begin
+    ti := TypeInfo(TCollationField);
+    ks := '';
+    kc := 0;
+    for e := Low(TCollationField) to High(TCollationField) do begin
+      if (e in ACollation.ChangedFields) then begin
+        ks := ks + ti^.Name + '.' +
+              GetEnumName(ti,Ord(e)) + ', ';
+        kc := kc + 1;
+      end
+    end;
+    if (AProps <> nil) then begin
+      if (AProps^.VariableLowLimit < High(Word)) then begin
+        ks := ks + ti^.Name + '.' +
+              GetEnumName(ti,Ord(TCollationField.VariableLowLimit)) + ', ';
+        kc := kc + 1;
+      end;
+      if (AProps^.VariableHighLimit > 0) then begin
+        ks := ks + ti^.Name + '.' +
+              GetEnumName(ti,Ord(TCollationField.VariableHighLimit)) + ', ';
+        kc := kc + 1;
+      end;
+    end;
+    if (kc > 0) then
+      ks := Copy(ks,1,(Length(ks)-2));
+    AddLine('  UPDATED_FIELDS = [ ' + ks + ' ];');
+  end;
+
+begin
+  AddLine('{$mode objfpc}{$H+}');
+  AddLine('unit ' + COLLATION_FILE_PREFIX + LowerCase(ACollation.Parent.LocalID)+ ';'+sLineBreak);
+  AddLine('interface'+sLineBreak);
+  AddLine('implementation');
+  AddLine('uses');
+  AddLine('  unicodedata, unicodeducet;'+sLineBreak);
+  AddLine('const');
+  AddFields();
+  AddLine('  COLLATION_NAME = ' + QuotedStr(ACollation.Parent.Language) + ';');
+  AddLine('  BASE_COLLATION = ' + QuotedStr(ACollation.Base) + ';');
+  AddLine('  VERSION_STRING = ' + QuotedStr(ABook^.Version) + ';');
+  if (AProps <> nil) then begin
+    AddLine('  VARIABLE_LOW_LIMIT = ' + IntToStr(AProps^.VariableLowLimit) + ';');
+    AddLine('  VARIABLE_HIGH_LIMIT = ' + IntToStr(AProps^.VariableHighLimit) + ';');
+    AddLine('  VARIABLE_WEIGHT = ' + IntToStr(Ord(ABook^.VariableWeight)) + ';');
+  end else begin
+    AddLine('  VARIABLE_LOW_LIMIT = ' + IntToStr(High(Word)) + ';');
+    AddLine('  VARIABLE_HIGH_LIMIT = ' + IntToStr(0) + ';');
+    AddLine('  VARIABLE_WEIGHT = ' + IntToStr(0) + ';');
+  end;
+  AddLine('  BACKWARDS_0 = ' + BoolToStr(ABook^.Backwards[0],'True','False') + ';');
+  AddLine('  BACKWARDS_1 = ' + BoolToStr(ABook^.Backwards[1],'True','False') + ';');
+  AddLine('  BACKWARDS_2 = ' + BoolToStr(ABook^.Backwards[2],'True','False') + ';');
+  AddLine('  BACKWARDS_3 = ' + BoolToStr(ABook^.Backwards[3],'True','False') + ';');
+  if (AProps <> nil) then
+    AddLine('  PROP_COUNT  = ' + IntToStr(Ord(AProps^.ItemSize)) + ';');
+
+  AddLine('');
+end;
+
+procedure GenerateUCA_CLDR_Registration(
+  ADest  : TStream;
+  ABook  : PUCA_DataBook;
+  AProps : PUCA_PropBook;
+  ACollation : TCldrCollationItem
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+begin
+  AddLine('var');
+  AddLine('  CLDR_Collation : TUCA_DataBook = (');
+  AddLine('    Base               : nil;');
+  AddLine('    Version            : VERSION_STRING;');
+  AddLine('    CollationName      : COLLATION_NAME;');
+  AddLine('    VariableWeight     : TUCA_VariableKind(VARIABLE_WEIGHT);');
+  AddLine('    Backwards          : (BACKWARDS_0,BACKWARDS_1,BACKWARDS_2,BACKWARDS_3);');
+  if (Length(ABook^.Lines) > 0) then begin
+    AddLine('    BMP_Table1         : @UCA_TABLE_1[0];');
+    AddLine('    BMP_Table2         : @UCA_TABLE_2[0];');
+    AddLine('    OBMP_Table1        : @UCAO_TABLE_1[0];');
+    AddLine('    OBMP_Table2        : @UCAO_TABLE_2[0];');
+    AddLine('    PropCount          : PROP_COUNT;');
+    AddLine('    Props              : PUCA_PropItemRec(@UCA_PROPS[0]);');
+  end else begin
+    AddLine('    BMP_Table1         : nil;');
+    AddLine('    BMP_Table2         : nil;');
+    AddLine('    OBMP_Table1        : nil;');
+    AddLine('    OBMP_Table2        : nil;');
+    AddLine('    PropCount          : 0;');
+    AddLine('    Props              : nil;');
+  end;
+  AddLine('    VariableLowLimit   : VARIABLE_LOW_LIMIT;');
+  AddLine('    VariableHighLimit  : VARIABLE_HIGH_LIMIT;');
+  AddLine('  );');
+  AddLine('');
+
+  AddLine('procedure Register();');
+  AddLine('begin');
+  AddLine('  PrepareCollation(@CLDR_Collation,BASE_COLLATION,UPDATED_FIELDS);');
+  AddLine('  RegisterCollation(@CLDR_Collation);');
+  AddLine('end;');
+  AddLine('');
+
+  AddLine('initialization');
+  AddLine('  Register();');
+  AddLine('');
+
+  AddLine('finalization');
+  AddLine('  UnregisterCollation(COLLATION_NAME);');
+  AddLine('');
+  AddLine('end.');
+end;
+
+procedure GenerateCdlrCollation(
+  ACollation    : TCldrCollation;
+  AItemName     : string;
+  AStoreName    : string;
+  AStream,
+  AEndianStream : TStream;
+  ARootChars    : TOrderedCharacters;
+  ARootWeigths  : TUCA_LineRecArray
+);
+
+  procedure AddLine(const ALine : ansistring; ADestStream : TStream);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADestStream.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  locUcaBook : TUCA_DataBook;
+  locSequence : TOrderedCharacters;
+  locItem : TCldrCollationItem;
+  i : Integer;
+  locUcaProps : PUCA_PropBook;
+  ucaFirstTable   : TucaBmpFirstTable;
+  ucaSecondTable  : TucaBmpSecondTable;
+  ucaoFirstTable   : TucaoBmpFirstTable;
+  ucaoSecondTable  : TucaOBmpSecondTable;
+  locHasProps : Boolean;
+  s : string;
+begin
+  locItem := ACollation.Find(AItemName);
+  if (locItem = nil) then
+    raise Exception.CreateFmt('Collation Item not found : "%s".',[AItemName]);
+  locSequence := ARootChars.Clone();
+  for i := 0 to Length(locItem.Rules) - 1 do
+    locSequence.ApplyStatement(@locItem.Rules[i]);
+  FillChar(locUcaBook,SizeOf(locUcaBook),0);
+  locUcaBook.Version := ACollation.Version;
+  locUcaBook.Backwards[1] := locItem.Backwards;
+  ComputeWeigths(@locSequence.Data[0],locSequence.ActualLength,ARootWeigths,locUcaBook.Lines);
+  for i := 0 to Length(locUcaBook.Lines) - 1 do
+    locUcaBook.Lines[i].Stored := True;
+  locHasProps := (Length(locUcaBook.Lines) > 0);
+  if not locHasProps then
+    locUcaProps := nil
+  else
+    MakeUCA_Props(@locUcaBook,locUcaProps);
+  try
+    if locHasProps then begin
+      MakeUCA_BmpTables(ucaFirstTable,ucaSecondTable,locUcaProps);
+      SetLength(ucaoSecondTable,100);
+      MakeUCA_OBmpTables(ucaoFirstTable,ucaoSecondTable,locUcaProps);
+    end;
+    GenerateLicenceText(AStream);
+    GenerateUCA_CLDR_Head(AStream,@locUcaBook,locUcaProps,locItem);
+    if locHasProps then begin
+      GenerateUCA_BmpTables(AStream,AEndianStream,ucaFirstTable,ucaSecondTable,THIS_ENDIAN);
+      GenerateUCA_OBmpTables(AStream,AEndianStream,ucaoFirstTable,ucaoSecondTable,THIS_ENDIAN);
+      GenerateUCA_PropTable(AEndianStream,locUcaProps);
+      s := GenerateEndianIncludeFileName(AStoreName);
+      AddLine(Format('{$include %s}'+sLineBreak,[ExtractFileName(s)]),AStream);
+    end;
+    GenerateUCA_CLDR_Registration(AStream,@locUcaBook,locUcaProps,locItem);
+  finally
+    locSequence.Clear();
+    FreeUcaBook(locUcaProps);
+  end;
+end;
+
+end.

utils/unicode/cldrparser.lpi

@@ -0,0 +1,105 @@
+<?xml version="1.0"?>
+<CONFIG>
+  <ProjectOptions>
+    <Version Value="9"/>
+    <PathDelim Value="\"/>
+    <General>
+      <Flags>
+        <MainUnitHasCreateFormStatements Value="False"/>
+        <MainUnitHasTitleStatement Value="False"/>
+      </Flags>
+      <SessionStorage Value="InProjectDir"/>
+      <MainUnit Value="0"/>
+      <Title Value="cldrparser"/>
+      <UseAppBundle Value="False"/>
+      <ResourceType Value="res"/>
+    </General>
+    <i18n>
+      <EnableI18N LFM="False"/>
+    </i18n>
+    <VersionInfo>
+      <StringTable ProductVersion=""/>
+    </VersionInfo>
+    <BuildModes Count="1">
+      <Item1 Name="Default" Default="True"/>
+    </BuildModes>
+    <PublishOptions>
+      <Version Value="2"/>
+      <IncludeFileFilter Value="*.(pas|pp|inc|lfm|lpr|lrs|lpi|lpk|sh|xml)"/>
+      <ExcludeFileFilter Value="*.(bak|ppu|o|so);*~;backup"/>
+    </PublishOptions>
+    <RunParams>
+      <local>
+        <FormatVersion Value="1"/>
+        <CommandLineParams Value="de.xml search"/>
+      </local>
+    </RunParams>
+    <Units Count="7">
+      <Unit0>
+        <Filename Value="cldrparser.lpr"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="cldrparser"/>
+      </Unit0>
+      <Unit1>
+        <Filename Value="cldrhelper.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="cldrhelper"/>
+      </Unit1>
+      <Unit2>
+        <Filename Value="..\helper.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="helper"/>
+      </Unit2>
+      <Unit3>
+        <Filename Value="cldrtest.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="cldrtest"/>
+      </Unit3>
+      <Unit4>
+        <Filename Value="cldrxml.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="cldrxml"/>
+      </Unit4>
+      <Unit5>
+        <Filename Value="..\tests\weight_derivation.inc"/>
+        <IsPartOfProject Value="True"/>
+      </Unit5>
+      <Unit6>
+        <Filename Value="unicodeset.pas"/>
+        <IsPartOfProject Value="True"/>
+        <UnitName Value="unicodeset"/>
+      </Unit6>
+    </Units>
+  </ProjectOptions>
+  <CompilerOptions>
+    <Version Value="11"/>
+    <PathDelim Value="\"/>
+    <Target>
+      <Filename Value="cldrparser"/>
+    </Target>
+    <SearchPaths>
+      <IncludeFiles Value="$(ProjOutDir);..\src"/>
+      <OtherUnitFiles Value="..\src;..\rbtree\src;..\trie"/>
+      <UnitOutputDirectory Value="lib\$(TargetCPU)-$(TargetOS)"/>
+    </SearchPaths>
+    <Other>
+      <CompilerMessages>
+        <MsgFileName Value=""/>
+      </CompilerMessages>
+      <CompilerPath Value="$(CompPath)"/>
+    </Other>
+  </CompilerOptions>
+  <Debugging>
+    <Exceptions Count="3">
+      <Item1>
+        <Name Value="EAbort"/>
+      </Item1>
+      <Item2>
+        <Name Value="ECodetoolError"/>
+      </Item2>
+      <Item3>
+        <Name Value="EFOpenError"/>
+      </Item3>
+    </Exceptions>
+  </Debugging>
+</CONFIG>

utils/unicode/cldrparser.lpr

@@ -0,0 +1,205 @@
+{   Unicode CLDR's collation parser.
+    Copyright (c) 2013 by Inoussa OUEDRAOGO
+
+    It creates units from CLDR's collation files.
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+}
+program cldrparser;
+
+{$mode objfpc}{$H+}
+
+{ $define test_suite} // Define this to execute the parser test suite.
+{$define actual_parsing}
+
+uses
+  SysUtils, classes, getopts,
+  cldrhelper, helper, cldrtest, cldrxml, unicodeset;
+
+const
+  SUsageText =
+    'This program creates pascal units from CLDR''s collation files for usage ' + sLineBreak +
+    'with the FreePascal Native Unicode Manager.' + sLineBreak + sLineBreak +
+    'Usage : cldrparser <collationFileName> [<typeName>] [-d<dataDir>] [-o<outputDir>]' + sLineBreak + sLineBreak +
+    '  where :' + sLineBreak +
+    ' ' + sLineBreak +
+    '   - collationFileName : specify the target file.' + sLineBreak +
+    '   - typeName : optional, specify the collation'' type-name to be parse;' + sLineBreak +
+    '     If this argument is not supplied, a default type-name' + sLineBreak +
+    '     is chosen from the type-name list in the following order  : ' + sLineBreak +
+    '          * the "default" specified type-name indicated by the collation' + sLineBreak +
+    '          * the type named "standard" ' + sLineBreak +
+    '          * the type named "search" ' + sLineBreak +
+    '          * the first type.' + sLineBreak +
+    '   - dataDir : specify the directory that contains the collation files.' + sLineBreak +
+    '               The default value is the program''s directory.' + sLineBreak +
+    '   - outputDir : specify the directory where the generated files will be stored.' + sLineBreak +
+    '                 The default value is the program''s directory.' + sLineBreak +
+    ' ' + sLineBreak +
+    '  The program expects some files to be present in the <dataDir> folder : ' + sLineBreak +
+    '     - UCA_Rules_SHORT.xml found in the CollationAuxiliary.zip available on unicode.org' + sLineBreak +
+    '     - allkeys.txt this is the file allkeys_CLDR.txt contained in CollationAuxiliary.zip renamed to allkeys.txt' + sLineBreak +
+    '  The CollationAuxiliary.zip archive is provided by unicode in the "unicode collation algorithm data files" section.';
+
+
+function ParseOptions(
+  var ADataDir, AOuputDir, ACollationFileName, ACollationTypeName : string
+) : Boolean;
+var
+  c : Char;
+  idx, k : Integer;
+  s : string;
+begin
+  if (ParamCount() = 0) then
+    exit(False);
+  Result := True;
+  repeat
+    c := GetOpt('d:o:h');
+    case c of
+      'd' : ADataDir := ExpandFileName(Trim(OptArg));
+      'o' : AOuputDir := ExpandFileName(Trim(OptArg));
+      'h', '?' :
+        begin
+          WriteLn(SUsageText);
+          Result := False;
+        end;
+    end;
+  until (c = EndOfOptions);
+  idx := 0;
+  for k := 1 to ParamCount() do begin
+    s := Trim(ParamStr(k));
+    if (s <> '') and (s[1] <> '-') then begin
+      if (idx = 0) then
+        ACollationFileName := s
+      else if (idx = 1) then
+        ACollationTypeName := s;
+      Inc(idx);
+      if (idx >= 2) then
+        Break;
+    end;
+  end;
+end;
+
+var
+  orderedChars : TOrderedCharacters;
+  ucaBook : TUCA_DataBook;
+  stream, endianStream : TMemoryStream;
+  s, collationFileName, collationTypeName : string;
+  i , c: Integer;
+  collation : TCldrCollation;
+  dataPath, outputPath : string;
+begin
+{$ifdef test_suite}
+  exec_tests();
+{$endif test_suite}
+
+{$ifdef actual_parsing}
+  dataPath := '';
+  outputPath := '';
+  collationFileName := '';
+  collationTypeName := '';
+  if not ParseOptions(dataPath,outputPath,collationFileName,collationTypeName) then
+    Halt(1);
+  if (dataPath <> '') and not(DirectoryExists(dataPath)) then begin
+    WriteLn('This directory does not exist : ',dataPath);
+    Halt(1);
+  end;
+  if (dataPath = '') then
+    dataPath := ExtractFilePath(ParamStr(0))
+  else
+    dataPath := IncludeTrailingPathDelimiter(dataPath);
+  if (outputPath = '') then
+    outputPath := dataPath
+  else
+    outputPath := IncludeTrailingPathDelimiter(outputPath);
+  if (ParamCount() = 0) then begin
+    WriteLn(SUsageText);
+    Halt(1);
+  end;
+  if not(
+       FileExists(dataPath+'UCA_Rules_SHORT.xml') and
+       FileExists(dataPath+'allkeys.txt')
+     )
+  then begin
+    WriteLn(Format('File not found : %s or %s.',[dataPath+'UCA_Rules_SHORT.xml',dataPath+'allkeys.txt']));
+    Halt(1);
+  end;
+
+  collationFileName := dataPath + collationFileName;
+  if not FileExists(collationFileName) then begin
+    WriteLn('File not found : "',collationFileName,'"');
+    Halt(1);
+  end;
+
+  WriteLn(sLineBreak,'Collation Parsing ',QuotedStr(collationFileName),'  ...');
+  stream := nil;
+  endianStream := nil;
+  collation := TCldrCollation.Create();
+  try
+    ParseCollationDocument(collationFileName,collation);
+    WriteLn(Format('  Collation Count = %d',[collation.ItemCount]));
+    if (collation.ItemCount = 0) then begin
+      WriteLn('No collation in this file.');
+    end else begin
+      for i := 0 to collation.ItemCount - 1 do
+        WriteLn(Format('  Item[%d] = %d "resets"; Type = %s',[i, Length(collation.Items[i].Rules),collation.Items[i].TypeName]));
+      if (collation.Find(collationTypeName) = nil) then
+        collationTypeName := FindCollationDefaultItemName(collation);
+      WriteLn('Collation Item Name : ',collationTypeName);
+
+
+      s := dataPath + 'UCA_Rules_SHORT.xml';
+      WriteLn;
+      WriteLn('Parsing ',QuotedStr(s),'  ...');
+      FillByte(orderedChars,SizeOf(orderedChars),0);
+      orderedChars.Clear();
+      ParseInitialDocument(@orderedChars,s);
+      WriteLn('File parsed, ',orderedChars.ActualLength,' characters.');
+
+      WriteLn('Loading CLDR root''s key table ...');
+      stream := TMemoryStream.Create();
+      s := dataPath + 'allkeys.txt';
+      stream.LoadFromFile(s);
+      ParseUCAFile(stream,ucaBook);
+      c := FillInitialPositions(@orderedChars.Data[0],orderedChars.ActualLength,ucaBook.Lines);
+      if (c > 0) then
+        WriteLn('  Missed Initial Positions = ',c);
+      WriteLn('   Loaded.');
+
+      WriteLn('Start generation ...');
+      stream.Clear();
+      endianStream := TMemoryStream.Create();
+      s := COLLATION_FILE_PREFIX + ChangeFileExt(LowerCase(ExtractFileName(collationFileName)),'.pas');
+      GenerateCdlrCollation(
+        collation,collationTypeName,s,stream,endianStream,
+        orderedChars,ucaBook.Lines
+      );
+      stream.SaveToFile(ExtractFilePath(collationFileName)+s);
+      if (endianStream.Size > 0) then
+        endianStream.SaveToFile(ExtractFilePath(collationFileName)+GenerateEndianIncludeFileName(s));
+    end;
+  finally
+    endianStream.Free();
+    stream.Free();
+    collation.Free();
+  end;
+
+{$endif actual_parsing}
+
+  WriteLn(sLineBreak,'Finished.');
+end.
+

utils/unicode/cldrtest.pas

@@ -0,0 +1,1920 @@
+{   CLDR collation Algorithm test routines.
+
+    Copyright (c) 2013 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+}
+
+unit cldrtest;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  Classes, SysUtils,
+  helper, cldrhelper, unicodedata;
+
+  function ToAnsiChars(const AValue : array of TUnicodeCodePoint) : string;
+  function DumpSequenceAnsi(const ASequence : TOrderedCharacters) : string;
+  function DumpWeigth(const AItem : TUCA_WeightRec) : string;
+  function DumpLine(ALine : TUCA_LineRec) : string;
+  function DumpLines(ALines : TUCA_LineRecArray) : string;
+  function CodePointToArray(const ACodePoint : TUnicodeCodePoint) : TUnicodeCodePointArray;overload;
+  function CodePointToArray(const ACodePoints : array of TUnicodeCodePoint) : TUnicodeCodePointArray;overload;
+  function ToWeight(const APrimary, ASecondary, ATertiary : Cardinal) : TUCA_WeightRecArray;overload;
+  function ToWeight(const AWeigths : array of Cardinal) : TUCA_WeightRecArray;overload;
+
+  procedure exec_tests();
+
+  procedure test1();
+  procedure test2();
+  procedure test3();
+  procedure test4();
+  procedure test5();
+  procedure test6();
+  procedure test7();
+  procedure test8();
+  procedure test9();
+  procedure test10();
+  procedure test11();
+
+implementation
+
+procedure exec_tests();
+begin
+  WriteLn('***************************** TEST 1 ******************');
+  test1();
+  WriteLn('***************************** TEST 2 ******************');
+  test2();
+  WriteLn('***************************** TEST 3 ******************');
+  test3();
+  WriteLn('***************************** TEST 4 ******************');
+  test4();
+  WriteLn('***************************** TEST 5 ******************');
+  test5();
+  WriteLn('***************************** TEST 6 ******************');
+  test6();
+  WriteLn('***************************** TEST 7 ******************');
+  test7();
+  WriteLn('***************************** TEST 8 ******************');
+  test8();
+  WriteLn('***************************** TEST 9 ******************');
+  test9();
+  WriteLn('***************************** TEST 10 ******************');
+  test10();
+  WriteLn('***************************** TEST 11 ******************');
+  test11();
+end;
+
+function ToAnsiChars(const AValue : array of TUnicodeCodePoint) : string;
+var
+  i : Integer;
+  s : string;
+begin
+  Result := '';
+  for i := Low(AValue) to High(AValue) do begin
+    if (AValue[i] <= 127) then
+      Result := Result + AnsiChar(AValue[i])
+    else
+      begin
+        s := Format('%x',[AValue[i]]);
+        if (Length(s) < 4) then
+          s := StringOfChar('0',4-Length(s)) + s;
+        Result := Result + '$' + s;
+      end;
+  end;
+end;
+
+function DumpSequenceAnsi(const ASequence : TOrderedCharacters) : string;
+var
+  i : Integer;
+  s : string;
+  p : PReorderUnit;
+begin
+  s := '';
+  if (ASequence.ActualLength < 1) then
+    exit;
+  p := @ASequence.Data[0];
+  i := 0;
+  while (i < ASequence.ActualLength) do begin
+    if (p^.WeigthKind <> TReorderWeigthKind.Deletion) then
+      Break;
+    WriteStr(s,s,  ' ',ToAnsiChars(p^.Characters),'- ');
+    Inc(p);
+    Inc(i);
+  end;
+
+  if (i < ASequence.ActualLength) then begin
+    s := s + '   ' + ToAnsiChars(p^.Characters) + ' ';
+    Inc(i);
+    Inc(p);
+    for i := i to ASequence.ActualLength - 1 do begin
+      //WriteStr(s,s,AnsiChar(p^.Characters[0]),' <',(1+Ord(p^.WeigthKind)),' ');
+      WriteStr(s,s,'<',(1+Ord(p^.WeigthKind)),' ',ToAnsiChars(p^.Characters),' ');
+      Inc(p);
+    end;
+  end;
+  Result := s;
+end;
+
+function DumpWeigth(const AItem : TUCA_WeightRec) : string;
+var
+  r : string;
+begin
+  r := '[';
+  if AItem.Variable then
+    r := r + '*'
+  else
+    r := r + '.';
+  r := r + Format('%x.%x.%x',[AItem.Weights[0],AItem.Weights[1],AItem.Weights[2]]);
+  r := r + ']';
+  Result := r;
+end;
+
+function DumpKey(const AItem : TUCASortKey) : string;
+var
+  i : Integer;
+  r : string;
+begin
+  r := '';
+  for i := Low(AItem) to High(AItem) do
+    r := Trim(r) + ' ' + Format('%4x',[AItem[i]]);
+  Result := r;
+end;
+
+function DumpLine(ALine : TUCA_LineRec) : string;
+var
+  i : Integer;
+  r : string;
+begin
+  r := '';
+  if (Length(ALine.Weights) = 0) then begin
+    r := '[]';
+  end else begin
+    for i := Low(ALine.Weights) to High(ALine.Weights) do
+      r := r + DumpWeigth(ALine.Weights[i]);
+  end;
+  Result := Format('%s %s',[ToAnsiChars(ALine.CodePoints),r]);
+end;
+
+function DumpLines(ALines : TUCA_LineRecArray) : string;
+var
+  i : Integer;
+  r : string;
+begin
+  r := '';
+  for i := Low(ALines) to High(ALines) do
+    r := r + '    ' +  DumpLine(ALines[i]) + sLineBreak;
+  Result := r;
+end;
+
+function CodePointToArray(const ACodePoint : TUnicodeCodePoint) : TUnicodeCodePointArray;overload;
+begin
+  SetLength(Result,1);
+  Result[0] := ACodePoint;
+end;
+
+function CodePointToArray(const ACodePoints : array of TUnicodeCodePoint) : TUnicodeCodePointArray;overload;
+var
+  i : Integer;
+begin
+  SetLength(Result,Length(ACodePoints));
+  for i := 0 to length(ACodePoints) - 1 do
+    Result[i] := ACodePoints[i];
+end;
+
+function ToWeight(const APrimary, ASecondary, ATertiary : Cardinal) : TUCA_WeightRecArray;overload;
+begin
+  SetLength(Result,1);
+  Result[0].Weights[0] := APrimary;
+  Result[0].Weights[1] := ASecondary;
+  Result[0].Weights[2] := ATertiary;
+  Result[0].Weights[3] := 0;
+end;
+
+function ToWeight(const AWeigths : array of Cardinal) : TUCA_WeightRecArray;overload;
+var
+  i, k, c : Integer;
+begin
+  c := Length(AWeigths);
+  SetLength(Result,(c div 3));
+  k := 0;
+  for i := 0 to (c div 3) - 1 do begin
+    Result[i].Weights[0] := AWeigths[k+0];
+    Result[i].Weights[1] := AWeigths[k+1];
+    Result[i].Weights[2] := AWeigths[k+2];
+    Result[i].Weights[3] := 0;
+    k := k + 3;
+  end;
+end;
+
+procedure constructPropBook(
+  var ABook : unicodedata.TUCA_DataBook;
+  const AFirstTable  : TucaBmpFirstTable;
+  const ASecondTable  : TucaBmpSecondTable;
+  const AOFirstTable   : TucaOBmpFirstTable;
+  const AOSecondTable  : TucaOBmpSecondTable;
+  const AInitDataBook : helper.TUCA_DataBook;
+  const AInitPropBook : helper.PUCA_PropBook
+);
+var
+  c, i, k, ci : Integer;
+begin
+  c := Length(AFirstTable);
+  if (c > 0) then begin
+    ABook.BMP_Table1 := AllocMem(c);
+    Move(AFirstTable[0],ABook.BMP_Table1^,c);
+  end;
+  c := Length(ASecondTable);
+  if (c > 0) then begin
+    ABook.BMP_Table2 := AllocMem(c*SizeOf(UInt24)*256);
+    for i := 0 to c - 1 do begin
+      for k := 0 to 255 do
+        ABook.BMP_Table2[(i*256)+k] := ASecondTable[i][k];
+    end;
+  end;
+
+  c := Length(AOFirstTable);
+  if (c > 0) then begin
+    ABook.OBMP_Table1 := AllocMem(c*SizeOf(Word));
+    Move(AOFirstTable[0],ABook.OBMP_Table1^,(c*SizeOf(Word)));
+  end;
+  c := Length(AOSecondTable);
+  if (c > 0) then begin
+    ci := Length(AOSecondTable[0]);
+    ABook.OBMP_Table2 := AllocMem(c*SizeOf(UInt24)*ci);
+    for i := 0 to c - 1 do begin
+      for k := 0 to ci - 1 do
+        ABook.OBMP_Table2[(i*ci)+k] := AOSecondTable[i][k];
+    end;
+  end;
+
+  ABook.Version := AInitDataBook.Version;
+  ABook.VariableWeight := unicodedata.TUCA_VariableKind(Ord(AInitDataBook.VariableWeight));
+  ABook.Backwards := AInitDataBook.Backwards;
+  ABook.PropCount := AInitPropBook^.ItemSize;
+  ABook.Props := Pointer(AInitPropBook^.Items);
+  ABook.VariableLowLimit := AInitPropBook^.VariableLowLimit;
+  ABook.VariableHighLimit := AInitPropBook^.VariableHighLimit;
+end;
+
+procedure ConstructUnicodeBook(
+  const AWeitghs       : TUCA_LineRecArray;
+  const AVersion       : string;
+  const ACollationName : string;
+  const ABase          : unicodedata.PUCA_DataBook;
+  var   AUnicodeBook   : unicodedata.TUCA_DataBook
+);
+var
+  dataBook : helper.TUCA_DataBook;
+  propBook : helper.PUCA_PropBook;
+  firstTable   : TucaBmpFirstTable;
+  secondTable  : TucaBmpSecondTable;
+  ofirstTable   : TucaOBmpFirstTable;
+  osecondTable  : TucaOBmpSecondTable;
+  i : Integer;
+begin
+  FillByte(dataBook,SizeOf(dataBook),0);
+  dataBook.Version := AVersion;
+  SetLength(dataBook.Lines,Length(AWeitghs));
+  for i := 0 to Length(AWeitghs)-1 do begin
+    dataBook.Lines[i] := AWeitghs[i];
+    dataBook.Lines[i].Stored := True;
+  end;
+  MakeUCA_Props(@dataBook,propBook);
+  MakeUCA_BmpTables(firstTable,secondTable,propBook);
+  MakeUCA_OBmpTables(ofirstTable,osecondTable,propBook);
+  FillByte(AUnicodeBook,SizeOf(AUnicodeBook),0);
+  constructPropBook(
+    AUnicodeBook,firstTable,secondTable,ofirstTable,osecondTable,
+    dataBook,propBook
+  );
+  AUnicodeBook.CollationName := ACollationName;
+  AUnicodeBook.Base := ABase;
+end;
+
+procedure CheckEqual(A,B : UnicodeString; ACollation : unicodedata.PUCA_DataBook);
+var
+  keyA, keyB : TUCASortKey;
+  s : string;
+begin
+  keyA := ComputeSortKey(A,ACollation);
+  keyB := ComputeSortKey(B,ACollation);
+  if (CompareSortKey(keyA,keyB) <> 0) then begin
+    s := Format('  KeyA=%s%s  KeyB=%s',[DumpKey(keyA),sLineBreak,DumpKey(keyB)]);
+    s := Format('"%s" <>= "%s" %s%s',[A,B,sLineBreak,s]);
+    raise Exception.Create(s);
+  end;
+end;
+
+procedure CheckNotEqual(A,B : UnicodeString; ACollation : unicodedata.PUCA_DataBook);
+var
+  keyA, keyB : TUCASortKey;
+  s : string;
+begin
+  keyA := ComputeSortKey(A,ACollation);
+  keyB := ComputeSortKey(B,ACollation);
+  if (CompareSortKey(keyA,keyB) = 0) then begin
+    s := Format('  KeyA=%s%s  KeyB=%s',[DumpKey(keyA),sLineBreak,DumpKey(keyB)]);
+    s := Format('"%s" = "%s" %s%s',[A,B,sLineBreak,s]);
+    raise Exception.Create(s);
+  end;
+end;
+
+procedure CheckInf(A,B : UnicodeString; ACollation : unicodedata.PUCA_DataBook);
+var
+  keyA, keyB : TUCASortKey;
+begin
+  keyA := ComputeSortKey(A,ACollation);
+  keyB := ComputeSortKey(B,ACollation);
+  if (CompareSortKey(keyA,keyB) >= 0) then
+    raise Exception.CreateFmt('"%s" >= "%s" !',[A,B]);
+end;
+
+procedure CheckInf(AStrings : array of UnicodeString; ACollation : unicodedata.PUCA_DataBook);
+var
+  c, i : Integer;
+  keyA, keyB : TUCASortKey;
+  s : string;
+begin
+  c := Length(AStrings);
+  if (c < 2) then
+    exit;
+  keyA := ComputeSortKey(AStrings[0],ACollation);
+  for i := 1 to c - 1 do begin
+    keyB := ComputeSortKey(AStrings[i],ACollation);
+    if (CompareSortKey(keyA,keyB) >= 0) then begin
+      s := Format('  KeyA=%s%s  KeyB=%s',[DumpKey(keyA),sLineBreak,DumpKey(keyB)]);
+      s := Format('"%s" >= "%s" %s%s',[AStrings[i-1],AStrings[i],sLineBreak,s]);
+      raise Exception.Create(s);
+    end;
+    keyA := keyB;
+  end;
+end;
+
+procedure test1_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,12);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('f'));
+    p^.Weights := ToWeight($1684,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('g'));
+    p^.Weights := ToWeight($1691,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('h'));
+    p^.Weights := ToWeight($16B4,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('i'));
+    p^.Weights := ToWeight($16CD,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('j'));
+    p^.Weights := ToWeight($16E6,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('l'));
+    p^.Weights := ToWeight($1711,$0020,$0002);
+end;
+
+procedure test1();
+var
+  sequence : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  i : Integer;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+begin
+  statement.Clear();
+  test1_prepareWeigth(wfirst);
+  sequence := TOrderedCharacters.Create();
+  sequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  sequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,2));
+  sequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,3));
+  sequence.Append(TReorderUnit.From(Ord('d'),TReorderWeigthKind.Primary,4));
+  sequence.Append(TReorderUnit.From(Ord('e'),TReorderWeigthKind.Primary,5));
+  sequence.Append(TReorderUnit.From(Ord('f'),TReorderWeigthKind.Primary,6));
+  sequence.Append(TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,7));
+  sequence.Append(TReorderUnit.From(Ord('h'),TReorderWeigthKind.Primary,8));
+  sequence.Append(TReorderUnit.From(Ord('i'),TReorderWeigthKind.Primary,9));
+  sequence.Append(TReorderUnit.From(Ord('j'),TReorderWeigthKind.Primary,10));
+  sequence.Append(TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,11));
+  sequence.Append(TReorderUnit.From(Ord('l'),TReorderWeigthKind.Primary,12));
+  for i := 0 to sequence.ActualLength - 1 do
+    sequence.Data[i].Changed := False;
+  WriteLn('Initial = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wfirst,'test1','first',nil,unicodeBook1);
+  CheckInf(['a','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook1);
+
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a','g'{*},'b','c','d','e','f','h','i','j','k','l'],@unicodeBook2);
+
+
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,2);
+  statement.Elements[0] := TReorderUnit.From(Ord('h'),TReorderWeigthKind.Primary,0);
+  statement.Elements[1] := TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a','h'{*},'k'{*},'g'{*},'b','c','d','e','f','i','j','l'],@unicodeBook2);
+
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('h');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Secondary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #3 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence));
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a','h'{*},'g'{*},'k'{*},'b','c','d','e','f','i','j','l'],@unicodeBook2);
+end;
+
+procedure test2_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,11);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('('),Ord('a'),Ord(')')]);
+    p^.Weights := ToWeight($15EF,$0020,$0006); //15EF.0020.0006.24D0
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('A'));
+    p^.Weights := ToWeight($15EF,$0020,$0008); //15EF.0020.0008.0041
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('('),Ord('A'),Ord(')')]);
+    p^.Weights := ToWeight($15EF,$0020,$000C);  //15EF.0020.000C
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('`')]);
+    p^.Weights := ToWeight([$15EF,$0020,$0002,  $0000,$0035,$0002]); //[.15EF.0020.0002.0061][.0000.0035.0002.0300]
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('A'),Ord('`')]);
+    p^.Weights := ToWeight([$15EF,$0020,$0008,  $0000,$0035,$0002]);  //[.15EF.0020.0008.0041][.0000.0035.0002.0300]
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('e')]);
+    p^.Weights := ToWeight([$15F0,$0020,$0002]);  //[.15EF.0020.0004.00E6][.0000.0139.0004.00E6][.164C.0020.0004.00E6]
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord(UpCase('a')),Ord(UpCase('e'))]);
+    p^.Weights := ToWeight([$15F0,$0020,$0006]);//[.15EF.0020.000A.00C6][.0000.0139.0004.00C6][.164C.0020.000A.00C6]
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('('),Ord('b'),Ord(')')]);
+    p^.Weights := ToWeight($1605,$0020,$0006);  //.1605.0020.0006.24D1
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('B'));
+    p^.Weights := ToWeight($1605,$0020,$0008); //1605.0020.0008.0042
+end;
+
+procedure test2();
+var
+  sequenceClean, sequence : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  i : Integer;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+begin
+  statement.Clear();
+  test2_prepareWeigth(wfirst);
+  sequenceClean := TOrderedCharacters.Create();
+  sequenceClean.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  sequenceClean.Append(TReorderUnit.From([Ord('('),Ord('a'),Ord(')')],TReorderWeigthKind.Tertiary,2));
+  sequenceClean.Append(TReorderUnit.From(Ord('A'),TReorderWeigthKind.Tertiary,3));
+  sequenceClean.Append(TReorderUnit.From([Ord('('),Ord('A'),Ord(')')],TReorderWeigthKind.Tertiary,4));
+
+  //sequenceClean.Append(TReorderUnit.From(Ord('à'),TReorderWeigthKind.Secondary,0));
+  sequenceClean.Append(TReorderUnit.From([Ord('a'),Ord('`')],TReorderWeigthKind.Secondary,5));
+  //sequenceClean.Append(TReorderUnit.From(Ord(UpCase('à')),TReorderWeigthKind.Tertiary,0));
+  sequenceClean.Append(TReorderUnit.From([Ord('A'),Ord('`')],TReorderWeigthKind.Tertiary,6));
+
+  sequenceClean.Append(TReorderUnit.From([Ord('a'),Ord('e')],TReorderWeigthKind.Primary,7));
+  sequenceClean.Append(TReorderUnit.From([Ord(UpCase('a')),Ord(UpCase('e'))],TReorderWeigthKind.Tertiary,8));
+
+  sequenceClean.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,9));
+  sequenceClean.Append(TReorderUnit.From([Ord('('),Ord('b'),Ord(')')],TReorderWeigthKind.Tertiary,10));
+  sequenceClean.Append(TReorderUnit.From(Ord('B'),TReorderWeigthKind.Tertiary,11));
+  for i := 0 to sequenceClean.ActualLength - 1 do
+    sequenceClean.Data[i].Changed := False;
+
+  WriteLn('Initial = ',sLineBreak,'  ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wfirst,'test1','first',nil,unicodeBook1);
+  CheckInf(['a','(a)','A','(A)',  'a`','A`',  'ae','AE',  'b','(b)','B'],@unicodeBook1);
+
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a','(a)','A','(A)',  'a`','A`', 'g'{*},  'ae','AE',  'b','(b)','B'],@unicodeBook2);
+  CheckInf(['gg','ae'],@unicodeBook2);
+  CheckInf(['gb','ae'],@unicodeBook2);
+  //CheckInf(['aeae','AE'],@unicodeBook2);
+
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Secondary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a','(a)','A','(A)', 'g'{*},  'a`','A`',  'ae','AE',  'b','(b)','B'],@unicodeBook2);
+  CheckInf(['(A)a','ga'],@unicodeBook2);
+  CheckInf(['g','ae'],@unicodeBook2);
+
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Tertiary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #3 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a', 'g'{*},'(a)','A','(A)',  'a`','A`',  'ae','AE',  'b','(b)','B'],@unicodeBook2);
+  CheckInf(['aa','ga'],@unicodeBook2);
+  CheckInf(['ga','(a)a'],@unicodeBook2);
+
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,2);
+  statement.Reset[0] := Ord('a');
+  statement.Reset[1] := Ord('`');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Tertiary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #4 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+  CheckInf(['a','(a)','A','(A)',  'a`', 'g'{*},'A`',  'ae','AE',  'b','(b)','B'],@unicodeBook2);
+  CheckInf(['a`a','ga'],@unicodeBook2);
+  CheckInf(['ga','ae'],@unicodeBook2);
+end;
+
+//------------------------------------------------------
+
+procedure test3_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,12);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('f'));
+    p^.Weights := ToWeight($1684,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('g'));
+    p^.Weights := ToWeight($1691,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('h'));
+    p^.Weights := ToWeight($16B4,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('i'));
+    p^.Weights := ToWeight($16CD,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('j'));
+    p^.Weights := ToWeight($16E6,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('l'));
+    p^.Weights := ToWeight($1711,$0020,$0002);
+end;
+
+procedure PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  ASequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,2));
+  ASequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From(Ord('d'),TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From(Ord('e'),TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From(Ord('f'),TReorderWeigthKind.Primary,6));
+  ASequence.Append(TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,7));
+  ASequence.Append(TReorderUnit.From(Ord('h'),TReorderWeigthKind.Primary,8));
+  ASequence.Append(TReorderUnit.From(Ord('i'),TReorderWeigthKind.Primary,9));
+  ASequence.Append(TReorderUnit.From(Ord('j'),TReorderWeigthKind.Primary,10));
+  ASequence.Append(TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,11));
+  ASequence.Append(TReorderUnit.From(Ord('l'),TReorderWeigthKind.Primary,12));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test3();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  i : Integer;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+  keyA, keyB : TUCASortKey;
+  us : UnicodeString;
+begin //'a','b','c','d','e','f','g','h','i','j','k','l'
+  statement.Clear();
+  test3_prepareWeigth(wfirst);
+  PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test3','first',nil,unicodeBook1);
+
+  us := 'a';
+  keyA := ComputeSortKey(us,@unicodeBook1);
+  for i := Ord('b') to Ord('l') do begin
+    us := unicodeChar(i);
+    keyB := ComputeSortKey(us,@unicodeBook1);
+    if (CompareSortKey(keyA,keyB) >= 0) then
+      raise Exception.CreateFmt('"%s" >= "%s" !',[AnsiChar(i-1),AnsiChar(i)]);
+    keyA := keyB;
+  end;
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('b');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test3','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','g'{*}, 'c','d','e','f','h','i','j','k','l'],@unicodeBook2);
+    CheckInf(['bb','g'{*}],@unicodeBook2);
+    CheckInf(['bc','g'{*}],@unicodeBook2);
+    CheckInf(['bc','gg'{*}],@unicodeBook2);
+    CheckInf(['bg','bc'{*}],@unicodeBook2);
+    WriteLn('    -- test 1 - ok');
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('c'),Ord('h')],TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test3','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','ch'{*},'d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckInf(['ca','ch'{*}],@unicodeBook2);
+    CheckInf(['cc','ch'{*}],@unicodeBook2);
+    CheckInf(['cd','ch'{*}],@unicodeBook2);
+    CheckInf(['ce','ch'{*}],@unicodeBook2);
+    CheckInf(['cf','ch'{*}],@unicodeBook2);
+    CheckInf(['ci','ch'{*}],@unicodeBook2);
+    CheckInf(['cj','ch'{*}],@unicodeBook2);
+    CheckInf(['ck','ch'{*}],@unicodeBook2);
+    CheckInf(['cl','ch'{*}],@unicodeBook2);
+    CheckInf(['ac','ach'{*}],@unicodeBook2);
+    CheckInf(['aci','achat'{*}],@unicodeBook2);
+    WriteLn('    -- test 2 - ok');
+
+  // --- test 3
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('k'),TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test3','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','k'{*},'d','e','f','g','h','i','j','l'],@unicodeBook2);
+    CheckInf(['a','b','c'{*},'d','e','f','g','h','i','j','l'],@unicodeBook2);
+    CheckEqual('c','k',@unicodeBook2);
+    CheckEqual('cc','kk',@unicodeBook2);
+    CheckEqual('ck','kc',@unicodeBook2);
+    CheckEqual('kc','kk',@unicodeBook2);
+    CheckEqual('cckkc','kckcc',@unicodeBook2);
+
+    CheckInf(['acb','akc'{*}],@unicodeBook2);
+    WriteLn('    -- test 3 - ok');
+
+  // --- test 4
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('c')],TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('c'),Ord('h')],TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test3','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','c'{*},'ch'{*},'b','d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckInf(['ca','ch'{*}],@unicodeBook2);
+    CheckInf(['cc','ch'{*}],@unicodeBook2);
+    CheckInf(['cd','ch'{*}],@unicodeBook2);
+    CheckInf(['ce','ch'{*}],@unicodeBook2);
+    CheckInf(['cf','ch'{*}],@unicodeBook2);
+    CheckInf(['ci','ch'{*}],@unicodeBook2);
+    CheckInf(['cj','ch'{*}],@unicodeBook2);
+    CheckInf(['ck','ch'{*}],@unicodeBook2);
+    CheckInf(['cl','ch'{*}],@unicodeBook2);
+    CheckInf(['ac','ach'{*}],@unicodeBook2);
+    CheckInf(['aci','achat'{*}],@unicodeBook2);
+    WriteLn('    -- test 4 - ok');
+end;
+
+
+//------------------------------------------------------
+
+procedure test4_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,12);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c')); {*}
+    p^.Weights := ToWeight($1606,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('f'));
+    p^.Weights := ToWeight($1684,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('g'));
+    p^.Weights := ToWeight($1691,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('h'));
+    p^.Weights := ToWeight($16B4,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('i'));
+    p^.Weights := ToWeight($16CD,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('j'));
+    p^.Weights := ToWeight($16E6,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('l'));
+    p^.Weights := ToWeight($1711,$0020,$0002);
+end;
+
+procedure test4();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  i : Integer;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+  keyA, keyB : TUCASortKey;
+  us : UnicodeString;
+begin
+  statement.Clear();
+  test4_prepareWeigth(wfirst);
+  PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test4','first',nil,unicodeBook1);
+
+  us := 'a';
+  keyA := ComputeSortKey(us,@unicodeBook1);
+  for i := Ord('b') to Ord('l') do begin
+    us := unicodeChar(i);
+    keyB := ComputeSortKey(us,@unicodeBook1);
+    if (CompareSortKey(keyA,keyB) >= 0) then
+      raise Exception.CreateFmt('"%s" >= "%s" !',[AnsiChar(i-1),AnsiChar(i)]);
+    keyA := keyB;
+  end;
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('b');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test4','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf(['a','b','g'{*}, 'c','d','e','f','h','i','j','k','l'],@unicodeBook2);
+    CheckInf(['g'{*}, 'c'],@unicodeBook2);
+    CheckInf(['gg'{*}, 'c'],@unicodeBook2);
+    CheckInf(['gg'{*}, 'cc'],@unicodeBook2);
+    CheckInf(['g'{*}, 'ca'],@unicodeBook2);
+    CheckInf(['gg'{*}, 'ca'],@unicodeBook2);
+    CheckInf(['bb','g'{*}],@unicodeBook2);
+    CheckInf(['bc','g'{*}],@unicodeBook2);
+    CheckInf(['bc','gg'{*}],@unicodeBook2);
+    CheckInf(['bg','bc'{*}],@unicodeBook2);
+    WriteLn('    -- test 1 - ok',sLineBreak);
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('b');
+  SetLength(statement.Elements,2);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  statement.Elements[1] := TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test4','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf(['a','b','g'{*},'k'{*}, 'c','d','e','f','h','i','j','l'],@unicodeBook2);
+    CheckInf(['g'{*}, 'c'],@unicodeBook2);
+    CheckInf(['k'{*}, 'c'],@unicodeBook2);
+    CheckInf(['b','kk'{*}],@unicodeBook2);
+    CheckInf(['bb','kk'{*}],@unicodeBook2);
+    CheckInf(['b','kkk'{*}],@unicodeBook2);
+    CheckInf(['gk','kk'{*}],@unicodeBook2);
+    CheckInf(['gk','k'{*}],@unicodeBook2);
+    CheckInf(['gk','kkk'{*}],@unicodeBook2);
+
+    CheckInf(['gg'{*}, 'c'],@unicodeBook2);
+    CheckInf(['gg'{*}, 'cc'],@unicodeBook2);
+    CheckInf(['g'{*}, 'ca'],@unicodeBook2);
+    CheckInf(['gg'{*}, 'ca'],@unicodeBook2);
+    CheckInf(['bb','g'{*}],@unicodeBook2);
+    CheckInf(['bc','g'{*}],@unicodeBook2);
+    CheckInf(['bc','gg'{*}],@unicodeBook2);
+    CheckInf(['bg','bc'{*}],@unicodeBook2);
+    WriteLn('    -- test 2 - ok');
+end;
+
+//-------------------------------------------------------------------------
+
+procedure test5_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,6);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('d'),Ord('a')]);
+    p^.Weights := ToWeight($1609,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+end;
+
+procedure test5_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  ASequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,2));
+  ASequence.Append(TReorderUnit.From([Ord('a'),Ord('d'),Ord('a')],TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From(Ord('d'),TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From(Ord('e'),TReorderWeigthKind.Primary,6));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test5();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  i : Integer;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+  keyA, keyB : TUCASortKey;
+  us : UnicodeString;
+begin
+  statement.Clear();
+  test5_prepareWeigth(wfirst);
+  test5_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test5','first',nil,unicodeBook1);
+  CheckInf(['a','b','ada','c','d','e'],@unicodeBook1);
+  CheckInf(['ba','adaa'],@unicodeBook1);
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,0);
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('c'),TReorderWeigthKind.Deletion,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test5','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf(['a','b','ada','d','e',  'c'{* deleted !}],@unicodeBook2);
+    CheckInf(['ee','ca'],@unicodeBook2);
+    WriteLn('    -- test 1 - ok',sLineBreak);
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,0);
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('a'),Ord('d'),Ord('a')],TReorderWeigthKind.Deletion,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test5','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf(['a', 'adac'{* deleted !}, 'b','c','d','e'],@unicodeBook2);
+    CheckInf(['a','ada'],@unicodeBook2);
+    CheckInf(['ada','b'],@unicodeBook2);
+    CheckInf(['ac','ada'],@unicodeBook2);
+    CheckInf(['ac','adac'],@unicodeBook2);
+    CheckInf(['abe','ada'],@unicodeBook2);
+    CheckInf(['abe','adae'],@unicodeBook2);
+    WriteLn('    -- test 2 - ok',sLineBreak);
+end;
+
+//-------------------------------------------------------------------------
+
+procedure test6_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,7);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('d')]);
+    p^.Weights := ToWeight($1609,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('d'),Ord('a')]);
+    p^.Weights := ToWeight($1613,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+end;
+
+procedure test6_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  ASequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,2));
+  ASequence.Append(TReorderUnit.From([Ord('a'),Ord('d')],TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From([Ord('a'),Ord('d'),Ord('a')],TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From(Ord('d'),TReorderWeigthKind.Primary,6));
+  ASequence.Append(TReorderUnit.From(Ord('e'),TReorderWeigthKind.Primary,7));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test6();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  unicodeBook1, unicodeBook2, unicodeBook3 : unicodedata.TUCA_DataBook;
+begin
+  statement.Clear();
+  test6_prepareWeigth(wfirst);
+  test6_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test6','first',nil,unicodeBook1);
+  CheckInf(['a','b','ad','ada','c','d','e'],@unicodeBook1);
+  CheckInf(['ba','ad'],@unicodeBook1);
+  CheckInf(['ba','adaa'],@unicodeBook1);
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,0);
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('a'),Ord('d')],TReorderWeigthKind.Deletion,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test6','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf(['a', 'ad'{*},'ada', 'b','c','d','e'],@unicodeBook2);
+    CheckInf(['ab','ad'],@unicodeBook2);
+    CheckInf(['ab','adb'],@unicodeBook2);
+    CheckInf(['ad','ba'],@unicodeBook2);
+    CheckInf(['adaa','ba'],@unicodeBook2);
+    WriteLn('    -- test 1 - ok',sLineBreak);
+
+  // --- test 2
+  //sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,0);
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('a'),Ord('d'),Ord('a')],TReorderWeigthKind.Deletion,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test6','second',@unicodeBook2,unicodeBook3);
+  unicodeBook3.Base := @unicodeBook2;
+    CheckInf(['a', 'ad'{*},'ada'{*}, 'b','c','d','e'],@unicodeBook3);
+    CheckInf(['ab','ad'],@unicodeBook3);
+    CheckInf(['ab','adb'],@unicodeBook3);
+    CheckInf(['ab','ada'],@unicodeBook3);
+    WriteLn('    -- test 2 - ok',sLineBreak);
+
+  // --- test 3
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,0);
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('a'),Ord('d'),Ord('a')],TReorderWeigthKind.Deletion,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #3 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test6','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf(['a', 'b', 'ad', 'c','d','e'],@unicodeBook2);
+    CheckInf(['ad','ada'],@unicodeBook2);
+    WriteLn('    -- test 3 - ok',sLineBreak);
+end;
+
+//-------------------------------------------------------------------------
+
+procedure test7_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,8);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray($030A);//030A  ; [.0000.0043.0002.030A] # COMBINING RING ABOVE
+    p^.Weights := ToWeight($0000,$0043,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0327);//0327  ; [.0000.0056.0002.0327] # COMBINING CEDILLA
+    p^.Weights := ToWeight($0000,$0056,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0061);//a
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0062);//b
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0063);//c
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0064);//d
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([$0061,$030A]);//a,030A;COMBINING RING ABOVE
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+end;
+
+procedure test7_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From($030A,TReorderWeigthKind.Tertiary,1));
+  ASequence.Append(TReorderUnit.From($0327,TReorderWeigthKind.Tertiary,2));
+  ASequence.Append(TReorderUnit.From($0061,TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From($0062,TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From($0063,TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From($0064,TReorderWeigthKind.Primary,6));
+  ASequence.Append(TReorderUnit.From([$0061,$030A],TReorderWeigthKind.Primary,7));
+  ASequence.Append(TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,11));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test7();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+begin  // Permutation simple test
+  statement.Clear();
+  test7_prepareWeigth(wfirst);
+  test7_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test7','first',nil,unicodeBook1);
+  CheckInf([#$030A,#$0327,#$0061,#$0062,#$0063,#$0064, #$0061#$030A,'k'],@unicodeBook1);
+  CheckInf([#$0064, #$0061#$030A#$0327#$0062,'k'],@unicodeBook1);// Permutation here $030A <=> #$0327
+  CheckInf([#$0064, #$0061#$0327#$030A#$0062,'k'],@unicodeBook1);
+  CheckEqual(#$0061#$030A#$0327, #$0061#$0327#$030A,@unicodeBook1);
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,0);
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([$0061,$030A],TReorderWeigthKind.Deletion,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test7','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf([#$030A,#$0327,#$0061,#$0061#$030A ,#$0062,#$0063,#$0064,'k'],@unicodeBook2);
+    CheckInf([#$0061, #$0061#$030A#$0327#$0062,#$0062,#$0064],@unicodeBook2);
+    CheckInf([#$0061, #$0061#$030A#$0062#$0327,#$0062,#$0064],@unicodeBook2);
+    WriteLn('    -- test 2 - ok',sLineBreak);
+end;
+
+//-------------------------------------------------------------------------
+
+procedure test8_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,12);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('f'));
+    p^.Weights := ToWeight($1684,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('g'));
+    p^.Weights := ToWeight($1691,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('h'));
+    p^.Weights := ToWeight($16B4,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('i'));
+    p^.Weights := ToWeight($16CD,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('j'));
+    p^.Weights := ToWeight($16E6,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('l'));
+    p^.Weights := ToWeight($1711,$0020,$0002);
+end;
+
+procedure test8_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  ASequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,2));
+  ASequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From(Ord('d'),TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From(Ord('e'),TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From(Ord('f'),TReorderWeigthKind.Primary,6));
+  ASequence.Append(TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,7));
+  ASequence.Append(TReorderUnit.From(Ord('h'),TReorderWeigthKind.Primary,8));
+  ASequence.Append(TReorderUnit.From(Ord('i'),TReorderWeigthKind.Primary,9));
+  ASequence.Append(TReorderUnit.From(Ord('j'),TReorderWeigthKind.Primary,10));
+  ASequence.Append(TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,11));
+  ASequence.Append(TReorderUnit.From(Ord('l'),TReorderWeigthKind.Primary,12));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test8();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+  i : Integer;
+begin
+  statement.Clear();
+  test8_prepareWeigth(wfirst);
+  test8_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test8','first',nil,unicodeBook1);
+  CheckInf('l','-'{* computed are greater},@unicodeBook1);
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('-'),[Ord('c')],TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test8','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckEqual('cc','c-'{*},@unicodeBook2);
+    CheckEqual('ccc','c-c'{*},@unicodeBook2);
+    CheckEqual('cca','c-a'{*},@unicodeBook2);
+    CheckEqual('cce','c-e'{*},@unicodeBook2);
+    CheckInf(['cc','c-c'{*}],@unicodeBook2);
+    CheckInf(['bc','bc-c'{*}],@unicodeBook2);
+    CheckInf('l','-'{* computed are greater},@unicodeBook2);
+    WriteLn('    -- test 1 - ok');
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('-'),[Ord('c')],TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test8','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckInf('cc','c-'{*},@unicodeBook2);
+    CheckInf('ccl','c-'{*},@unicodeBook2);
+    CheckInf('ccc','c-c'{*},@unicodeBook2);
+    CheckInf('cca','c-a'{*},@unicodeBook2);
+    CheckInf('cce','c-e'{*},@unicodeBook2);
+    CheckInf(['cc','c-c'{*}],@unicodeBook2);
+    CheckInf(['bc','bc-c'{*}],@unicodeBook2);
+    CheckInf('l','-'{* computed are greater},@unicodeBook2);
+    WriteLn('    -- test 2 - ok');
+
+  // --- test 3
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('-'),Ord('+')],[Ord('c')],TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test8','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckEqual('cc','c-+'{*},@unicodeBook2);
+    CheckEqual('ccc','c-+c'{*},@unicodeBook2);
+    CheckEqual('cca','c-+a'{*},@unicodeBook2);
+    CheckEqual('cce','c-+e'{*},@unicodeBook2);
+    CheckInf(['cc','c-+c'{*}],@unicodeBook2);
+    CheckInf(['bc','bc-+c'{*}],@unicodeBook2);
+    CheckInf('l','-+'{* computed are greater},@unicodeBook2);
+    WriteLn('    -- test 3 - ok');
+
+  // --- test 4 : '-' has 3 contexts to force the context tree to have at least
+  //                  a "Left" and a "Right"
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('-'),[Ord('c')],TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  statement.Reset[0] := Ord('f');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('-'),[Ord('f')],TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('-'),[Ord('a')],TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test8','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    // Check c
+    CheckEqual('cc','c-'{*},@unicodeBook2);
+    CheckEqual('ccc','c-c'{*},@unicodeBook2);
+    CheckEqual('cca','c-a'{*},@unicodeBook2);
+    CheckEqual('cce','c-e'{*},@unicodeBook2);
+    CheckInf(['cc','c-c'{*}],@unicodeBook2);
+    CheckInf(['bc','bc-c'{*}],@unicodeBook2);
+    //check f
+    CheckEqual('ff','f-'{*},@unicodeBook2);
+    CheckEqual('fff','f-f'{*},@unicodeBook2);
+    CheckEqual('ffa','f-a'{*},@unicodeBook2);
+    CheckEqual('ffe','f-e'{*},@unicodeBook2);
+    CheckInf(['ff','f-f'{*}],@unicodeBook2);
+    CheckInf(['bf','bf-f'{*}],@unicodeBook2);
+    //check c and f
+    CheckEqual('ccf','c-f'{*},@unicodeBook2);
+    CheckEqual('ccff','c-f-'{*},@unicodeBook2);
+    CheckEqual('ccfff','c-f-f'{*},@unicodeBook2);
+    CheckEqual('ffcc','f-c-'{*},@unicodeBook2);
+    CheckEqual('ffccf','f-c-f'{*},@unicodeBook2);
+
+    CheckInf('ffccf','g'{*},@unicodeBook2);
+    CheckInf('a-','ab',@unicodeBook2);
+    // check - alone
+    CheckInf('l','-'{* computed are greater},@unicodeBook2);
+    WriteLn('    -- test 4 - ok');
+
+  // --- test 5 : Add a contraction to force the code path
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('a');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('-'),Ord('h')],TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('-'),[Ord('c')],TReorderWeigthKind.Identity,0);
+  sequence.ApplyStatement(@statement);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test8','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','-h','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckEqual('cc','c-'{*},@unicodeBook2);
+    CheckEqual('ccc','c-c'{*},@unicodeBook2);
+    CheckEqual('cca','c-a'{*},@unicodeBook2);
+    CheckEqual('cce','c-e'{*},@unicodeBook2);
+    CheckInf(['cc','c-c'{*}],@unicodeBook2);
+    CheckInf(['bc','bc-c'{*}],@unicodeBook2);
+    CheckInf(['ab','-hb'{*}],@unicodeBook2);
+    CheckInf(['-hb','ba'],@unicodeBook2);
+    CheckInf('l','-'{* computed are greater},@unicodeBook2);
+    WriteLn('    -- test 5 - ok');
+
+  WriteLn('    -- test - ok',sLineBreak);
+end;
+
+//-------------------------------------------------------------------------
+
+procedure test9_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,8);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray($030A);//030A  ; [.0000.0043.0002.030A] # COMBINING RING ABOVE
+    p^.Weights := ToWeight($0000,$0043,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0327);//0327  ; [.0000.0056.0002.0327] # COMBINING CEDILLA
+    p^.Weights := ToWeight($0000,$0056,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0061);//a
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0062);//b
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0063);//c
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray($0064);//d
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([$0061,$030A]);//a,030A;COMBINING RING ABOVE
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+end;
+
+procedure test9_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From($030A,TReorderWeigthKind.Tertiary,1));
+  ASequence.Append(TReorderUnit.From($0327,TReorderWeigthKind.Tertiary,2));
+  ASequence.Append(TReorderUnit.From($0061,TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From($0062,TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From($0063,TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From($0064,TReorderWeigthKind.Primary,6));
+  ASequence.Append(TReorderUnit.From([$0061,$030A],TReorderWeigthKind.Primary,7));
+  ASequence.Append(TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,11));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test9();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+begin// Permutation with Context
+  statement.Clear();
+  test7_prepareWeigth(wfirst);
+  test7_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test9','first',nil,unicodeBook1);
+  CheckInf([#$030A,#$0327,#$0061,#$0062,#$0063,#$0064, #$0061#$030A,'k'],@unicodeBook1);
+  CheckInf([#$0064, #$0061#$030A#$0327#$0062,'k'],@unicodeBook1);// Permutation here $030A <=> #$0327
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := $0062;
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From([Ord('k'),$032D],[$0061],TReorderWeigthKind.Secondary,0); //032D;COMBINING CIRCUMFLEX ACCENT BELOW;Mn;220;NSM;;;;;N;NON-SPACING CIRCUMFLEX BELOW;;;;
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test9','second',@unicodeBook1,unicodeBook2);
+  unicodeBook2.Base := @unicodeBook1;
+    CheckInf([#$030A,#$0327,#$0061,#$0062,#$0063,#$0064,'k'],@unicodeBook2);
+    CheckInf([#$0061'k'#$032D  ,#$0061#$0063],@unicodeBook2);
+    CheckNotEqual(#$0061'k'#$0327#$032D, #$0061#$0327,@unicodeBook2);
+    CheckInf([#$0061'k'#$0327#$032D  ,#$0061#$0063],@unicodeBook2);
+    WriteLn('    -- test 2 - ok',sLineBreak);
+end;
+
+//------------------------------------------------------
+
+procedure test10_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,12);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('('),Ord('a'),Ord(')')]);
+    p^.Weights := ToWeight($15EF,$0020,$0006); //15EF.0020.0006.24D0
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('A'));
+    p^.Weights := ToWeight($15EF,$0020,$0008); //15EF.0020.0008.0041
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('('),Ord('A'),Ord(')')]);
+    p^.Weights := ToWeight($15EF,$0020,$000C);  //15EF.0020.000C
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('`')]);
+    p^.Weights := ToWeight([$15EF,$0020,$0002,  $0000,$0035,$0002]); //[.15EF.0020.0002.0061][.0000.0035.0002.0300]
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('A'),Ord('`')]);
+    p^.Weights := ToWeight([$15EF,$0020,$0008,  $0000,$0035,$0002]);  //[.15EF.0020.0008.0041][.0000.0035.0002.0300]
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('a'),Ord('e')]);
+    p^.Weights := ToWeight([$15F0,$0020,$0002]);  //[.15EF.0020.0004.00E6][.0000.0139.0004.00E6][.164C.0020.0004.00E6]
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord(UpCase('a')),Ord(UpCase('e'))]);
+    p^.Weights := ToWeight([$15F0,$0020,$0006]);//[.15EF.0020.000A.00C6][.0000.0139.0004.00C6][.164C.0020.000A.00C6]
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray([Ord('('),Ord('b'),Ord(')')]);
+    p^.Weights := ToWeight($1605,$0020,$0006);  //.1605.0020.0006.24D1
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('B'));
+    p^.Weights := ToWeight($1605,$0020,$0008); //1605.0020.0008.0042
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+end;
+
+procedure test10_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  ASequence.Append(TReorderUnit.From([Ord('('),Ord('a'),Ord(')')],TReorderWeigthKind.Tertiary,2));
+  ASequence.Append(TReorderUnit.From(Ord('A'),TReorderWeigthKind.Tertiary,3));
+  ASequence.Append(TReorderUnit.From([Ord('('),Ord('A'),Ord(')')],TReorderWeigthKind.Tertiary,4));
+
+  //ASequence.Append(TReorderUnit.From(Ord('à'),TReorderWeigthKind.Secondary,0));
+  ASequence.Append(TReorderUnit.From([Ord('a'),Ord('`')],TReorderWeigthKind.Secondary,5));
+  //ASequence.Append(TReorderUnit.From(Ord(UpCase('à')),TReorderWeigthKind.Tertiary,0));
+  ASequence.Append(TReorderUnit.From([Ord('A'),Ord('`')],TReorderWeigthKind.Tertiary,6));
+
+  ASequence.Append(TReorderUnit.From([Ord('a'),Ord('e')],TReorderWeigthKind.Primary,7));
+  ASequence.Append(TReorderUnit.From([Ord(UpCase('a')),Ord(UpCase('e'))],TReorderWeigthKind.Tertiary,8));
+
+  ASequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,9));
+  ASequence.Append(TReorderUnit.From([Ord('('),Ord('b'),Ord(')')],TReorderWeigthKind.Tertiary,10));
+  ASequence.Append(TReorderUnit.From(Ord('B'),TReorderWeigthKind.Tertiary,11));
+
+  ASequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,12));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test10();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  unicodeBook1, unicodeBook2, unicodeBook3 : unicodedata.TUCA_DataBook;
+begin
+  statement.Clear();
+  test10_prepareWeigth(wfirst);
+  test10_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test6','first',nil,unicodeBook1);
+  CheckInf(['a','(a)','A','(A)',  'a`','A`',  'ae','AE',  'b','(b)','B', 'c'],@unicodeBook1);
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  statement.Clear();
+  statement.Before := True;
+  SetLength(statement.Reset,2);
+  statement.Reset[0] := Ord('a');
+  statement.Reset[1] := Ord('e');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','(a)','A','(A)',  'a`','A`', 'g'{*},  'ae','AE',  'b','(b)','B', 'c'],@unicodeBook2);
+    CheckInf(['gg','ae'],@unicodeBook2);
+    CheckInf(['gb','ae'],@unicodeBook2);
+    WriteLn('    -- test 1 - ok',sLineBreak);
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  statement.Clear();
+  statement.Before := True;
+  SetLength(statement.Reset,2);
+  statement.Reset[0] := Ord('a');
+  statement.Reset[1] := Ord('e');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Secondary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test2','2',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','(a)','A','(A)',  'a`','A`', 'g'{*},  'ae','AE',  'b','(b)','B', 'c'],@unicodeBook2);
+    CheckInf(['A`B','gg'],@unicodeBook2);
+    CheckInf(['A`b','g'],@unicodeBook2);
+    WriteLn('    -- test 2 - ok',sLineBreak);
+
+  // --- test 3
+  sequence := sequenceClean.Clone();
+  statement.Clear();
+  statement.Before := True;
+  SetLength(statement.Reset,2);
+  statement.Reset[0] := Ord('a');
+  statement.Reset[1] := Ord('e');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Tertiary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #3 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test2','2',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','(a)','A','(A)',  'a`','A`', 'g'{*},  'ae','AE',  'b','(b)','B', 'c'],@unicodeBook2);
+    CheckInf(['A`B','gg'],@unicodeBook2);
+    CheckInf(['A`b','g'],@unicodeBook2);
+    WriteLn('    -- test 3 - ok',sLineBreak);
+
+  // --- test 4
+  sequence := sequenceClean.Clone();
+  statement.Clear();
+  statement.Before := True;
+  SetLength(statement.Reset,2);
+  statement.Reset[0] := Ord('A');
+  statement.Reset[1] := Ord('`');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Tertiary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #4.1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  statement.Clear();
+  statement.Before := True;
+  SetLength(statement.Reset,2);
+  statement.Reset[0] := Ord('A');
+  statement.Reset[1] := Ord('`');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('G'),TReorderWeigthKind.Secondary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #4.2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test2','2',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','(a)','A','(A)', 'G','a`','g'{*},'A`',  'ae','AE',  'b','(b)','B', 'c'],@unicodeBook2);
+    CheckInf(['gg','A`B'],@unicodeBook2);
+    CheckInf(['g','A`b'],@unicodeBook2);
+    CheckInf(['A','gg'],@unicodeBook2);
+    CheckInf(['A','Ga'],@unicodeBook2);
+    WriteLn('    -- test 4 - ok',sLineBreak);
+
+  // --- test 5
+  sequence := sequenceClean.Clone();
+  statement.Clear();
+  statement.Before := True;
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('B');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('Statement #5 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  ConstructUnicodeBook(wresult,'test1','1',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','(a)','A','(A)',  'a`','A`',  'ae','AE', 'g'{*},  'b','(b)','B', 'c'],@unicodeBook2);
+    CheckInf(['gg','b'],@unicodeBook2);
+    CheckInf(['ae','gb'],@unicodeBook2);
+    WriteLn('    -- test 5 - ok',sLineBreak);
+end;
+
+//------------------------------------------------------
+
+procedure test11_prepareWeigth(var AData : TUCA_LineRecArray);
+var
+  p : PUCA_LineRec;
+begin
+  SetLength(AData,12);
+  p := @AData[Low(AData)];
+    p^.CodePoints := CodePointToArray(Ord('a'));
+    p^.Weights := ToWeight($15EF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('b'));
+    p^.Weights := ToWeight($1605,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('c'));
+    p^.Weights := ToWeight($161D,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('d'));
+    p^.Weights := ToWeight($1631,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('e'));
+    p^.Weights := ToWeight($164C,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('f'));
+    p^.Weights := ToWeight($1684,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('g'));
+    p^.Weights := ToWeight($1691,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('h'));
+    p^.Weights := ToWeight($16B4,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('i'));
+    p^.Weights := ToWeight($16CD,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('j'));
+    p^.Weights := ToWeight($16E6,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('k'));
+    p^.Weights := ToWeight($16FF,$0020,$0002);
+  Inc(p);
+    p^.CodePoints := CodePointToArray(Ord('l'));
+    p^.Weights := ToWeight($1711,$0020,$0002);
+end;
+
+procedure test11_PopulateSequence(var ASequence : TOrderedCharacters);
+var
+  i : Integer;
+begin
+  ASequence := TOrderedCharacters.Create();
+  ASequence.Append(TReorderUnit.From(Ord('a'),TReorderWeigthKind.Primary,1));
+  ASequence.Append(TReorderUnit.From(Ord('b'),TReorderWeigthKind.Primary,2));
+  ASequence.Append(TReorderUnit.From(Ord('c'),TReorderWeigthKind.Primary,3));
+  ASequence.Append(TReorderUnit.From(Ord('d'),TReorderWeigthKind.Primary,4));
+  ASequence.Append(TReorderUnit.From(Ord('e'),TReorderWeigthKind.Primary,5));
+  ASequence.Append(TReorderUnit.From(Ord('f'),TReorderWeigthKind.Primary,6));
+  ASequence.Append(TReorderUnit.From(Ord('g'),TReorderWeigthKind.Primary,7));
+  ASequence.Append(TReorderUnit.From(Ord('h'),TReorderWeigthKind.Primary,8));
+  ASequence.Append(TReorderUnit.From(Ord('i'),TReorderWeigthKind.Primary,9));
+  ASequence.Append(TReorderUnit.From(Ord('j'),TReorderWeigthKind.Primary,10));
+  ASequence.Append(TReorderUnit.From(Ord('k'),TReorderWeigthKind.Primary,11));
+  ASequence.Append(TReorderUnit.From(Ord('l'),TReorderWeigthKind.Primary,12));
+  for i := 0 to ASequence.ActualLength - 1 do
+    ASequence.Data[i].Changed := False;
+end;
+
+procedure test11();
+var
+  sequence, sequenceClean : TOrderedCharacters;
+  statement : TReorderSequence;
+  wfirst, wresult : TUCA_LineRecArray;
+  i : Integer;
+  unicodeBook1, unicodeBook2 : unicodedata.TUCA_DataBook;
+  keyA, keyB : TUCASortKey;
+  us : UnicodeString;
+begin
+  statement.Clear();
+  test11_prepareWeigth(wfirst);
+  test11_PopulateSequence(sequenceClean);
+
+  WriteLn('  Initial = ',sLineBreak,'    ',DumpSequenceAnsi(sequenceClean),sLineBreak);
+  WriteLn(DumpLines(wfirst),sLineBreak+sLineBreak);
+  //Generate the original tables
+  ConstructUnicodeBook(wfirst,'test11','first',nil,unicodeBook1);
+  CheckInf(['a','b','c','d','e','f','g','h','i','j','k','l'],@unicodeBook1);
+
+  // --- test 1
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,1);
+  statement.Elements[0] := TReorderUnit.From(Ord('x'),TReorderWeigthKind.Tertiary,0);
+  statement.Elements[0].SetExpansion(Ord('h'));
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #1 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test1','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','x'{*},'d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckInf('ch','x'{*},@unicodeBook2);
+    CheckInf('cha','xa'{*},@unicodeBook2);
+    CheckInf('chk','xka'{*},@unicodeBook2);
+    WriteLn('    -- test 1 - ok');
+
+  // --- test 2
+  sequence := sequenceClean.Clone();
+  SetLength(statement.Reset,1);
+  statement.Reset[0] := Ord('c');
+  SetLength(statement.Elements,2);
+  statement.Elements[0] := TReorderUnit.From(Ord('x'),TReorderWeigthKind.Tertiary,0);
+    statement.Elements[0].SetExpansion(Ord('h'));
+  statement.Elements[1] := TReorderUnit.From(Ord('X'),TReorderWeigthKind.Tertiary,0);
+  sequence.ApplyStatement(@statement);
+  WriteLn('    Statement #2 = ',sLineBreak,'  ',DumpSequenceAnsi(sequence),sLineBreak);
+  wresult := nil;
+  ComputeWeigths(@sequence.Data[0],sequence.ActualLength,wfirst,wresult);
+  WriteLn(DumpLines(wresult),sLineBreak+sLineBreak);
+  //Generate updatet tables
+  ConstructUnicodeBook(wresult,'test2','second',@unicodeBook1,unicodeBook2);
+    CheckInf(['a','b','c','x'{*},'X'{*},'d','e','f','g','h','i','j','k','l'],@unicodeBook2);
+    CheckInf('ch','x'{*},@unicodeBook2);
+    CheckInf('cha','xa'{*},@unicodeBook2);
+    CheckInf('chk','xka'{*},@unicodeBook2);
+    CheckInf('ch','X'{*},@unicodeBook2);
+    CheckInf('cha','Xa'{*},@unicodeBook2);
+    CheckInf('chk','Xka'{*},@unicodeBook2);
+    WriteLn('    -- test 2 - ok');
+
+end;
+
+end.

utils/unicode/cldrxml.pas

@@ -0,0 +1,634 @@
+{   Parser of the CLDR collation xml files.
+
+    Copyright (c) 2013 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+}
+
+unit cldrxml;
+
+{$mode objfpc}{$H+}
+{$TypedAddress on}
+interface
+
+uses
+  Classes, SysUtils, DOM,
+  cldrhelper;
+
+  procedure ParseInitialDocument(ASequence : POrderedCharacters; ADoc : TDOMDocument);overload;
+  procedure ParseInitialDocument(ASequence : POrderedCharacters; AFileName : string);overload;
+
+  procedure ParseCollationDocument(ADoc : TDOMDocument; ACollation : TCldrCollation);
+  procedure ParseCollationDocument(const AFileName : string; ACollation : TCldrCollation);
+
+resourcestring
+  sCaseNothandled = 'This case is not handled : "%s", Position = %d.';
+  sCodePointExpected = 'Code Point node expected as child at this position "%d".';
+  sCollationsNodeNotFound = '"collations" node not found.';
+  sHexAttributeExpected = '"hex" attribute expected at this position "%d".';
+  sInvalidResetClause = 'Invalid "Reset" clause.';
+  sNodeNameAssertMessage = 'Expected NodeName "%s", got "%s".';
+  sRulesNodeNotFound = '"rules" node not found.';
+  sTextNodeChildExpected = '(Child) text node expected at this position "%d", but got "%s".';
+  sUniqueChildNodeExpected = 'Unique child node expected at this position "%d".';
+  sUnknownResetLogicalPosition = 'Unknown reset logical position : "%s".';
+
+implementation
+uses
+  typinfo, XMLRead, XPath, Helper, unicodeset;
+
+const
+  s_AT     = 'at';
+  s_BEFORE = 'before';
+  s_CODEPOINT = 'codepoint';
+  s_COLLATION = 'collation';
+  s_COLLATIONS = 'collations';
+  s_CONTEXT = 'context';
+  s_DEFAULT    = 'default';
+  s_EXTEND = 'extend';
+  s_HEX       = 'hex';
+  s_POSITION = 'position';
+  s_RESET = 'reset';
+  s_RULES = 'rules';
+  s_STANDART = 'standard';
+  s_TYPE     = 'type';
+
+procedure CheckNodeName(ANode : TDOMNode; const AExpectedName : DOMString);
+begin
+  if (ANode.NodeName <> AExpectedName) then
+    raise Exception.CreateFmt(sNodeNameAssertMessage,[AExpectedName,ANode.NodeName]);
+end;
+
+function CharToReorderWeigthKind(const AChar : Char) : TReorderWeigthKind;inline;
+begin
+  case AChar of
+    'p' : Result := TReorderWeigthKind.PriMary;
+    's' : Result := TReorderWeigthKind.Secondary;
+    't' : Result := TReorderWeigthKind.Tertiary;
+    'i' : Result := TReorderWeigthKind.Identity;
+    else
+     Result := TReorderWeigthKind.Identity;
+  end;
+end;
+
+function DomString2UnicodeCodePointArray(const AValue : DOMString): TUnicodeCodePointArray;
+var
+  u4str : UCS4String;
+  k : Integer;
+begin
+  if (Length(AValue) = 0) then
+    exit(nil);
+  if (Length(AValue) = 1) then begin
+    SetLength(Result,1);
+    Result[0] := Ord(AValue[1])
+  end else begin
+    u4str := WideStringToUCS4String(AValue);
+    k := Length(u4str) - 1; // remove the last #0
+    SetLength(Result,k);
+    for k := 0 to k - 1 do
+      Result[k] := u4str[k];
+  end;
+end;
+
+function TryStrToLogicalReorder(
+  const AValue  : string;
+  out   AResult : TReorderLogicalReset
+) : Boolean;
+var
+  s : string;
+  i : Integer;
+begin
+  s := StringReplace(AValue,' ','',[rfReplaceAll]);
+  s := StringReplace(s,'_','',[rfReplaceAll]);
+  i := GetEnumValue(TypeInfo(TReorderLogicalReset),s);
+  Result := (i > -1);
+  if Result then
+    AResult := TReorderLogicalReset(i);
+end;
+
+function ParseStatement(
+      ARules         : TDOMElement;
+      AStartPosition : Integer;
+      AStatement     : PReorderSequence;
+  var ANextPos       : Integer
+) : Boolean;
+var
+  startPosition : Integer;
+  statement : PReorderSequence;
+  elementActualCount : Integer;
+  list : TDOMNodeList;
+  inBlock : Boolean;
+
+  procedure SkipComments();
+  begin
+    while (startPosition < list.Count) do begin
+      if (list[startPosition].NodeType <> COMMENT_NODE) then
+        Break;
+      Inc(startPosition);
+    end;
+  end;
+
+  function parse_reset() : Integer;
+  var
+    n, t : TDOMNode;
+    s : string;
+    logicalPos : TReorderLogicalReset;
+  begin
+    SkipComments();
+    n := list[startPosition];
+    CheckNodeName(n,s_RESET);
+    if n.HasChildNodes() then begin
+      n := n.FirstChild;
+      if (n.NodeType = TEXT_NODE) then begin
+        statement^.Reset := DomString2UnicodeCodePointArray(Trim(TDOMText(n).Data));
+        Result := startPosition+1;
+      end else begin
+        if not TryStrToLogicalReorder(n.NodeName,logicalPos) then
+          raise Exception.CreateFmt(sUnknownResetLogicalPosition,[n.NodeName]);
+        statement^.LogicalPosition := logicalPos;
+        Result := startPosition+1;
+      end;
+    end else if not n.HasChildNodes() then begin
+      if (list[startPosition+1].NodeName = s_POSITION) then begin
+        s := list[startPosition+1].Attributes.GetNamedItem(s_AT).NodeValue;
+        if not TryStrToLogicalReorder(s,logicalPos) then
+          raise Exception.CreateFmt(sUnknownResetLogicalPosition,[s]);
+        statement^.LogicalPosition := logicalPos;
+        Result := startPosition+2;
+      end else begin
+        t := list[startPosition+1];
+        {if (t.NodeType <> TEXT_NODE) then
+          raise Exception.CreateFmt(sTextNodeChildExpected,[(startPosition+1),(t.NodeName+'('+t.ClassName+')')]);}
+        if (t.NodeType = TEXT_NODE) then
+          statement^.Reset := DomString2UnicodeCodePointArray(Trim(TDOMText(t).Data))
+        else
+          statement^.Reset := DomString2UnicodeCodePointArray(' ');
+        Result := startPosition+2;
+      end;
+    end;
+    if (statement^.LogicalPosition = TReorderLogicalReset.None) and
+      (Length(statement^.Reset) = 0)
+    then
+      raise Exception.Create(sInvalidResetClause);
+  end;
+
+  procedure EnsureElementLength(const ALength : Integer);
+  var
+    k, d : Integer;
+  begin
+    k := Length(statement^.Elements);
+    if (k < ALength) then begin
+      k := ALength;
+      if (k = 0) then begin
+        k := 50;
+      end else begin
+        if (k < 10) then
+          d := 10
+        else
+          d := 2;
+        k := k * d;
+      end;
+     SetLength(statement^.Elements,k);
+    end;
+  end;
+
+  {procedure AddElement(AText : DOMString; AWeigthKind : TReorderWeigthKind);overload;
+  var
+    u4str : UCS4String;
+    k : Integer;
+    kp : PReorderUnit;
+  begin
+    u4str := WideStringToUCS4String(AText);
+    EnsureElementLength(elementActualCount+1);
+    kp := @statement^.Elements[elementActualCount];
+    k := Length(u4str) - 1{null terminated};
+   SetLength(kp^.Characters,k);
+    for k := 0 to k - 1 do
+      kp^.Characters[k] := u4str[k];
+    kp^.WeigthKind:= AWeigthKind;
+    elementActualCount := elementActualCount + 1;
+  end;}
+
+  procedure AddElement(
+    const AChars      : array of UCS4Char;
+    const AWeigthKind : TReorderWeigthKind;
+    const AContext    : DOMString
+  );overload;
+  var
+    kp : PReorderUnit;
+    k : Integer;
+  begin
+    EnsureElementLength(elementActualCount+1);
+    kp := @statement^.Elements[elementActualCount];
+    SetLength(kp^.Characters,Length(AChars));
+    for k := 0 to Length(AChars) - 1 do
+     kp^.Characters[k] := AChars[k];
+    kp^.WeigthKind := AWeigthKind;
+    elementActualCount := elementActualCount + 1;
+    if (AContext <> '') then
+      kp^.Context := DomString2UnicodeCodePointArray(AContext);
+  end;
+
+  procedure ReadChars(
+        ANode    : TDOMNode;
+        APos     : Integer;
+    var AChars   : UCS4String
+  );
+  var
+    t : TDOMNode;
+    u4str : UCS4String;
+    s : DOMString;
+  begin
+    if not ANode.HasChildNodes() then begin
+      SetLength(AChars,1);
+      AChars[0] := Ord(UnicodeChar(' '));
+      exit;
+      //raise Exception.CreateFmt(sCodePointExpected + ANode.ClassName,[APos]);
+    end;
+    t := ANode.FindNode(s_CODEPOINT);
+    if (t = nil) then begin
+      if (ANode.ChildNodes.Count <> 1) then
+        raise Exception.CreateFmt(sUniqueChildNodeExpected,[APos]);
+      t := ANode.ChildNodes[0];
+      if not t.InheritsFrom(TDOMText) then
+        raise Exception.CreateFmt(sTextNodeChildExpected,[APos,(t.NodeName+'('+t.ClassName+')')]);
+      s := TDOMText(t).Data;
+      if (Length(s) = 1) then begin
+        SetLength(AChars,1);
+        AChars[0] := Ord(s[1]);
+      end else begin
+        u4str := WideStringToUCS4String(s);
+        AChars := u4str;
+        SetLength(AChars,Length(AChars)-1);
+      end;
+    end else begin
+      t := t.Attributes.GetNamedItem(s_HEX);
+      if (t = nil) then
+        raise Exception.CreateFmt(sHexAttributeExpected,[APos]);
+      SetLength(AChars,1);
+      AChars[0] := StrToInt('$'+t.NodeValue);
+    end
+  end;
+
+  procedure AddPrefixChars(const APrefix : array of UCS4Char; var ADest : TUnicodeCodePointArray);
+  var
+    k : Integer;
+  begin
+    k := Length(ADest);
+    SetLength(ADest,(k+Length(APrefix)));
+    Move(ADest[0],ADest[k+1],(SizeOf(k*ADest[0])));
+    for k := 0 to k - 1 do
+      ADest[k] := APrefix[k];
+  end;
+
+  function ReadNextItem(const APos : Integer) : Integer;
+  var
+    n, t : TDOMNode;
+    s, contextStr : DOMString;
+    w : TReorderWeigthKind;
+    isSimpleCharTag : Boolean;
+    simpleCharTag : AnsiChar;
+    last : PReorderUnit;
+    u4str : UCS4String;
+    k : Integer;
+  begin
+    contextStr := '';
+    Result := APos;
+    n := list[APos];
+    isSimpleCharTag := (Length(n.NodeName) = 1) and (Ord(n.NodeName[1])<=127);
+    if isSimpleCharTag then begin
+      simpleCharTag := AnsiChar(n.NodeName[1]);
+      if (simpleCharTag = 'x') then begin
+        inBlock := True;
+        n := n.FirstChild;
+        if (n.NodeName = s_CONTEXT) then begin
+          if n.HasChildNodes() then begin
+            t := n.FirstChild;
+            if (t.NodeType = TEXT_NODE) then
+              contextStr := TDOMText(t).Data;
+          end;
+          n := n.NextSibling;
+        end;
+        isSimpleCharTag := (Length(n.NodeName) = 1) and (Ord(n.NodeName[1])<=127);
+        if isSimpleCharTag then
+          simpleCharTag := AnsiChar(n.NodeName[1]);
+      end;
+    end;
+    if isSimpleCharTag and (simpleCharTag in ['p','s','t','i']) then begin
+      w := CharToReorderWeigthKind(AnsiChar(n.NodeName[1]));
+      ReadChars(n,APos,u4str);
+      AddElement(u4str,w,contextStr);
+      Result := Result + 1;
+      if not inBlock then
+        exit;
+      last := @statement^.Elements[elementActualCount-1];
+      n := n.NextSibling;
+      if (n <> nil) and (n.NodeName = s_EXTEND) then begin
+        ReadChars(n,APos,u4str);
+        SetLength(last^.ExpansionChars,Length(u4str));
+        for k := 0 to Length(u4str) - 1 do
+          last^.ExpansionChars[k] := u4str[k];
+      end;
+      exit;
+    end;
+    if (Length(n.NodeName) = 2) and (n.NodeName[2] = 'c') and
+       (Ord(n.NodeName[1])<=127) and (AnsiChar(n.NodeName[1]) in ['p','s','t','i'])
+    then begin
+      w := CharToReorderWeigthKind(AnsiChar(n.NodeName[1]));
+      ReadChars(n,APos,u4str);
+      for k := Low(u4str) to High(u4str) do
+        AddElement(u4str[k],w,contextStr);
+      Result := Result + 1;
+      exit;
+    end;
+    raise Exception.CreateFmt(sCaseNothandled,[n.NodeName,APos]);
+  end;
+
+var
+  i, c : Integer;
+  n : TDOMNode;
+begin
+  Result := False;
+  inBlock := False;
+  elementActualCount := 0;
+  if (AStartPosition <= 0) then
+    startPosition := 0
+  else
+    startPosition := AStartPosition;
+  i := startPosition;
+  list := ARules.ChildNodes;
+  c := list.Count;
+  if (c <= i) then
+    exit;
+  statement := AStatement;
+  statement^.Clear();
+  n := list[i];
+  i := parse_reset();
+  while (i < c) do begin
+    n := list[i];
+    if (n.NodeName = s_RESET) then
+      Break;
+    i := ReadNextItem(i);
+  end;
+  SetLength(statement^.Elements,elementActualCount);
+  Result := (i > startPosition);
+  if Result then
+    ANextPos := i;
+end;
+
+procedure ParseInitialDocument(ASequence : POrderedCharacters; ADoc : TDOMDocument);
+var
+  n : TDOMNode;
+  rulesElement : TDOMElement;
+  i, c, nextPost : Integer;
+  statement : TReorderSequence;
+  p : PReorderUnit;
+begin
+  n := ADoc.DocumentElement.FindNode(s_RULES);
+  if (n = nil) then
+    raise Exception.Create(sRulesNodeNotFound);
+  rulesElement := n as TDOMElement;
+  c := rulesElement.ChildNodes.Count;
+  ASequence^.Clear();
+  SetLength(ASequence^.Data,c+100);
+  nextPost := 0;
+  i := 0;
+  while (i < c) do begin
+    statement.Clear();
+    if not ParseStatement(rulesElement,i,@statement,nextPost) then
+      Break;
+    i := nextPost;
+    try
+      ASequence^.ApplyStatement(@statement);
+    except
+      on e : Exception do begin
+        e.Message := Format('%s  Position = %d',[e.Message,i]);
+        raise;
+      end;
+    end;
+  end;
+  if (ASequence^.ActualLength > 0) then begin
+    p := @ASequence^.Data[0];
+    for i := 0 to ASequence^.ActualLength - 1 do begin
+      p^.Changed := False;
+      Inc(p);
+    end;
+  end;
+end;
+
+procedure ParseInitialDocument(ASequence : POrderedCharacters; AFileName : string);
+var
+  doc : TXMLDocument;
+begin
+  ReadXMLFile(doc,AFileName);
+  try
+    ParseInitialDocument(ASequence,doc);
+  finally
+    doc.Free();
+  end;
+end;
+
+function EvaluateXPathStr(const AExpression : string; AContextNode : TDOMNode): DOMString;
+var
+  xv : TXPathVariable;
+begin
+  xv := EvaluateXPathExpression(AExpression,AContextNode);
+  try
+    if (xv <> nil) then
+      Result := xv.AsText
+    else
+      Result := '';
+  finally
+    xv.Free();
+  end;
+end;
+
+function ParseDeletion(
+  const APattern  : DOMString;
+        ASequence : PReorderSequence
+) : Integer;
+var
+  r : array of TReorderUnit;
+  c, i : Integer;
+  uset : TUnicodeSet;
+  it : TUnicodeSet.TIterator;
+  p : PReorderUnit;
+begin
+  if (APattern = '') then
+    exit(0);
+  it := nil;
+  uset := TUnicodeSet.Create();
+  try
+    uset.AddPattern(APattern);
+    it := uset.CreateIterator();
+    c := 0;
+    it.Reset();
+    while it.MoveNext() do begin
+      Inc(c);
+    end;
+    SetLength(r,c);
+    p := @r[0];
+    i := 0;
+    it.Reset();
+    while it.MoveNext() do begin
+      p^.Clear();
+      p^.WeigthKind := TReorderWeigthKind.Deletion;
+      p^.Characters := Copy(it.GetCurrent());
+      Inc(p);
+      Inc(i);
+    end;
+    ASequence^.Clear();
+    ASequence^.Elements := r;
+  finally
+    it.Free();
+    uset.Free();
+  end;
+  SetLength(r,0);
+end;
+
+procedure ParseCollationItem(ACollationNode : TDOMElement; AItem : TCldrCollationItem);
+var
+  n : TDOMNode;
+  rulesElement : TDOMElement;
+  i, c, nextPos : Integer;
+  statementList : TReorderSequenceArray;
+  sal : Integer;//statement actual length
+  statement : PReorderSequence;
+  s : DOMString;
+begin
+  AItem.TypeName := ACollationNode.GetAttribute(s_TYPE);
+  AItem.Base := EvaluateXPathStr('base',ACollationNode);
+  AItem.Backwards := (EvaluateXPathStr('settings/@backwards',ACollationNode) = 'on');
+  if AItem.Backwards then
+    AItem.ChangedFields := AItem.ChangedFields + [TCollationField.BackWard];
+
+  SetLength(statementList,15);
+  sal := 0;
+  statement := @statementList[0];
+  s := EvaluateXPathStr('suppress_contractions',ACollationNode);
+  if (s <> '') then begin
+    if (ParseDeletion(s,statement) > 0) then begin
+      Inc(sal);
+      Inc(statement);
+    end else begin
+      statement^.Clear();
+    end;
+  end;
+  n := ACollationNode.FindNode(s_RULES);
+  if (n <> nil) then begin
+    rulesElement := n as TDOMElement;
+    c := rulesElement.ChildNodes.Count;
+    nextPos := 0;
+    i := 0;
+    while (i < c) do begin
+      statement^.Clear();
+      if not ParseStatement(rulesElement,i,statement,nextPos) then
+        Break;
+      i := nextPos;
+      Inc(statement);
+      Inc(sal);
+      if (sal >= Length(statementList)) then begin
+        SetLength(statementList,(sal*2));
+        statement := @statementList[(sal-1)];
+      end;
+    end;
+  end;
+  SetLength(statementList,sal);
+  AItem.Rules := statementList;
+end;
+
+procedure ParseCollationDocument(ADoc : TDOMDocument; ACollation : TCldrCollation);
+var
+  rulesNodes, n : TDOMNode;
+  collationsElement, rulesElement : TDOMElement;
+  i, c : Integer;
+  item : TCldrCollationItem;
+  nl : TDOMNodeList;
+begin
+  n := ADoc.DocumentElement.FindNode(s_COLLATIONS);
+  if (n = nil) then
+    raise Exception.Create(sCollationsNodeNotFound);
+  collationsElement := n as TDOMElement;
+  ACollation.Clear();
+  ACollation.Language := EvaluateXPathStr('identity/language/@type',ADoc.DocumentElement);
+  ACollation.Version := EvaluateXPathStr('identity/version/@number',ADoc.DocumentElement);
+  ACollation.DefaultType := EvaluateXPathStr('collations/default/@type',ADoc.DocumentElement);
+  if collationsElement.HasChildNodes() then begin
+    nl := collationsElement.ChildNodes;
+    c := nl.Count;
+    item := nil;
+    try
+      for i := 0 to c - 1 do begin
+        n := nl[i];
+        if (n.NodeName = s_COLLATION) then begin
+          item := TCldrCollationItem.Create();
+          ParseCollationItem((n as TDOMElement),item);
+          ACollation.Add(item);
+          item := nil;
+        end
+      end;
+    except
+      FreeAndNil(item);
+      raise;
+    end;
+  end;
+end;
+
+function ReadXMLFile(f: TStream) : TXMLDocument;
+var
+  src : TXMLInputSource;
+  parser: TDOMParser;
+begin
+  src := TXMLInputSource.Create(f);
+  Result := TXMLDocument.Create;
+  parser := TDOMParser.Create();
+  try
+    parser.Options.IgnoreComments := True;
+    parser.Parse(src, Result);
+  finally
+    src.Free();
+    parser.Free;
+  end;
+end;
+
+function ReadXMLFile(const AFilename: String) : TXMLDocument;
+var
+  FileStream: TStream;
+begin
+  Result := nil;
+  FileStream := TFileStream.Create(AFilename, fmOpenRead+fmShareDenyWrite);
+  try
+    Result := ReadXMLFile(FileStream);
+  finally
+    FileStream.Free;
+  end;
+end;
+
+procedure ParseCollationDocument(const AFileName : string; ACollation : TCldrCollation);
+var
+  doc : TXMLDocument;
+begin
+  doc := ReadXMLFile(AFileName);
+  try
+    ParseCollationDocument(doc,ACollation);
+    ACollation.LocalID := ExtractFileName(ChangeFileExt(AFileName,''));
+  finally
+    doc.Free();
+  end;
+end;
+
+end.

utils/unicode/data/readme.txt

@@ -0,0 +1,13 @@
+This folder requires the next files to be present:
+
+  Extracted from http://www.unicode.org/Public/6.2.0/ucd/UCD.zip:
+    * UnicodeData.txt 
+    * HangulSyllableType.txt
+    * PropList.txt
+
+  Extracted from http://www.unicode.org/Public/UCA/6.2.0/CollationAuxiliary.zip:
+    * allkeys.txt : this file is actually the allkeys_CLDR.txt file renamed. It is the CLDR's root collation.
+    * UCA_Rules_SHORT.xml
+
+  Extracted from http://www.unicode.org/Public/cldr/22/core.zip (see the "common\collation" folder):
+    * all the language specific xml files (de.xml, es.xml, ...)

utils/unicode/grbtree.pas

@@ -0,0 +1,667 @@
+{   Red Black Tree implementation.
+
+    Copyright (c) 2013 by Inoussa OUEDRAOGO
+
+    Inspired by ideas of Julienne Walker
+      see http://www.eternallyconfuzzled.com/tuts/datastructures/jsw_tut_bst1.aspx
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. }
+
+unit grbtree;
+
+{$ifdef FPC}
+  {$mode delphi}
+  {$H+}
+{$endif FPC}
+{$TYPEDADDRESS ON}
+{$define RB_DEBUG}
+
+interface
+
+const
+  HEIGHT_LIMIT = 64;
+
+type
+
+
+   {KCOMP = class
+    public
+      // Return
+      //    * if A>B then  1
+      //    * if A=B then  0
+      //    * if A<B then -1
+      class function Compare(const A, B : TRBTreeNodeData) : Integer;
+    end;    }
+
+  TRBTree<T, KCOMP> = class
+  public type
+    TRBTreeNodeData = T;
+    PRBTreeNode = ^TRBTreeNode;
+    PRBTreeAllocator = ^TRBTreeAllocator;
+    TRBTreeNode = record
+      Links : array[Boolean] of PRBTreeNode;
+      Data  : TRBTreeNodeData;
+      Red   : Boolean;
+    end;
+    TRBTreeNodeCreator = function(AContext : Pointer) : PRBTreeNode;
+    TRBTreeNodeDestructor = procedure(ANode : PRBTreeNode; AContext : Pointer);
+    TRBTreeAllocator = record
+      CreateNode : TRBTreeNodeCreator;
+      FreeNode   : TRBTreeNodeDestructor;
+    end;
+
+    TRBTreeNodeComparator = KCOMP;
+    ThisType = TRBTree<T,KCOMP>;
+
+  private type
+    TBaseIterator = record
+      Tree         : ThisType;
+      StartingNode : PRBTreeNode;
+      StartingDir  : Boolean;
+      Current      : PRBTreeNode;
+      Top          : NativeInt;
+      Path         : array[0..(HEIGHT_LIMIT-1)] of PRBTreeNode;
+    end;
+    PBaseIterator = ^TBaseIterator;
+  public type
+
+    TIterator = class
+    private
+      FHandle : PBaseIterator;
+      FResetState : Boolean;
+    private
+    public
+      constructor Create(AHandle : PBaseIterator);
+      destructor Destroy;override;
+      procedure Reset();
+      function MoveNext() : Boolean;inline;
+      function MovePrevious() : Boolean;inline;
+      function GetCurrent : TRBTreeNodeData;inline;
+      function GetCurrentNode : PRBTreeNode;inline;
+    end;
+
+  public var
+    Root       : PRBTreeNode;
+    //FSize      : Integer;
+    Allocator  : TRBTreeAllocator;
+    Comparator : TRBTreeNodeComparator;
+  private
+    class function TreeCreateIterator() : PBaseIterator;static;inline;
+    class procedure TreeFreeIterator(AItem : PBaseIterator);static;inline;
+    class procedure TreeInitIterator(
+            AIterator     : PBaseIterator;
+      const ATree         : ThisType;
+      const AStartingNode : PRBTreeNode;
+      const ADirection    : Boolean
+    );static;
+    class function TreeIteratorMove(
+      AIterator     : PBaseIterator;
+      ADirection    : Boolean
+    ) : PRBTreeNode;static;
+    class function TreeIteratorMoveNext(AIterator : PBaseIterator) : PRBTreeNode;static;inline;
+    class function TreeIteratorMovePrevious(AIterator : PBaseIterator) : PRBTreeNode;static;inline;
+    function CreateIterator(
+      const ANode      : PRBTreeNode;
+      const ADirection : Boolean
+    ) : TIterator;inline;
+  private
+    class function DefaultCreateNode(AContext : Pointer) : PRBTreeNode;static;
+    class procedure DefaultFreeNode(ANode : PRBTreeNode; AContext : Pointer);static;
+
+    function InitNode(ANode : PRBTreeNode; AData : TRBTreeNodeData) : PRBTreeNode;inline;
+    function IsRed(ANode : PRBTreeNode): Boolean;inline;
+    function RotateDouble(ARoot : PRBTreeNode; const ADir : Boolean) : PRBTreeNode;inline;
+    function RotateSingle(ARoot : PRBTreeNode; const ADir : Boolean) : PRBTreeNode;
+  public
+    constructor Create(const AAllocator  : PRBTreeAllocator);overload;
+    constructor Create();overload;
+    destructor Destroy;override;
+    procedure Clear();
+    function FindNode(const AData : TRBTreeNodeData) : PRBTreeNode;
+    function Insert(const AData : TRBTreeNodeData) : PRBTreeNode;
+    function Remove(const AData : TRBTreeNodeData) : Boolean;
+    function CreateForwardIterator(const ANode : PRBTreeNode) : TIterator;overload;inline;
+    function CreateForwardIterator() : TIterator;overload;inline;
+    function CreateBackwardIterator(const ANode : PRBTreeNode) : TIterator;overload;inline;
+    function CreateBackwardIterator() : TIterator;overload;inline;
+{$ifdef RB_DEBUG}
+    function SelfAssert(ARoot : PRBTreeNode; var AErrorMessage : string) : Boolean;overload;
+    function SelfAssert(var AErrorMessage : string) : Boolean;overload;
+{$endif RB_DEBUG}
+  end;
+
+  TOrdinalComparator<T> = class
+  public type
+    TOrdinalType = T;
+  public
+    // Return
+    //    * if A>B then  1
+    //    * if A=B then  0
+    //    * if A<B then -1
+    class function Compare(const A, B : TOrdinalType) : Integer;static;inline;
+  end;
+
+implementation
+
+{ TRBTree<T> }
+
+function TRBTree<T,KCOMP>.IsRed(ANode : PRBTreeNode): Boolean;inline;
+begin
+  Result := (ANode <> nil) and ANode^.Red;
+end;
+
+function TRBTree<T,KCOMP>.InitNode(ANode: PRBTreeNode; AData: TRBTreeNodeData): PRBTreeNode;inline;
+begin
+  Result := ANode;
+  Result^.Data := AData;
+  Result^.Red := True;
+  Result^.Links[False] := nil;
+  Result^.Links[True] := nil;
+end;
+
+function TRBTree<T,KCOMP>.RotateDouble(ARoot: PRBTreeNode; const ADir: Boolean): PRBTreeNode;inline;
+begin
+  ARoot^.Links[not ADir] := RotateSingle(ARoot^.Links[not ADir], not ADir );
+  Result := RotateSingle(ARoot,ADir);
+end;
+
+function TRBTree<T,KCOMP>.RotateSingle(ARoot: PRBTreeNode; const ADir: Boolean): PRBTreeNode;
+var
+  t : PRBTreeNode;
+begin
+  t := ARoot^.Links[not ADir];
+
+  ARoot^.Links[not ADir] := t^.Links[ADir];
+  t^.Links[ADir] := ARoot;
+
+  ARoot^.Red := True;
+  t^.Red := False;
+
+  Result := t;
+end;
+
+class function TRBTree<T,KCOMP>.TreeCreateIterator() : PBaseIterator;static;
+begin
+  Result := AllocMem(SizeOf(Result^));
+end;
+
+class procedure TRBTree<T,KCOMP>.TreeFreeIterator(AItem : PBaseIterator);static;
+begin
+  if (AItem <> nil) then
+    FreeMem(AItem,SizeOf(AItem^));
+end;
+
+class procedure TRBTree<T,KCOMP>.TreeInitIterator(
+        AIterator     : PBaseIterator;
+  const ATree         : ThisType;
+  const AStartingNode : PRBTreeNode;
+  const ADirection    : Boolean
+);static;
+begin
+  AIterator^.Tree := ATree;
+  AIterator^.StartingNode := AStartingNode;
+  AIterator^.StartingDir := ADirection;
+  if (AStartingNode = nil) then
+    AIterator^.Current := AIterator^.Tree.Root
+  else
+    AIterator^.Current := AStartingNode;
+  AIterator^.Top := 0;
+
+  // Save the path for later traversal
+  if (AIterator^.Current <> nil) then begin
+    while (AIterator^.Current^.Links[ADirection] <> nil) do begin
+      AIterator^.Path[AIterator^.Top] := AIterator^.Current;
+      Inc(AIterator^.Top);
+      AIterator^.Current := AIterator^.Current^.Links[ADirection];
+    end;
+  end;
+end;
+
+class function TRBTree<T,KCOMP>.TreeIteratorMove(
+  AIterator  : PBaseIterator;
+  ADirection : Boolean
+) : PRBTreeNode;static;
+var
+  last : PRBTreeNode;
+begin
+  Result := nil;
+  if (AIterator^.Current = nil) then
+    exit;
+
+  if (AIterator^.Current^.Links[ADirection] <> nil) then begin
+    // Continue down this branch
+    AIterator^.Path[AIterator^.Top] := AIterator^.Current;
+    Inc(AIterator^.Top);
+    AIterator^.Current := AIterator^.Current^.Links[ADirection];
+
+    while ( AIterator^.Current^.Links[not ADirection] <> nil) do begin
+      AIterator^.Path[AIterator^.Top] := AIterator^.Current;
+      Inc(AIterator^.Top);
+      AIterator^.Current := AIterator^.Current^.Links[not ADirection];
+    end;
+  end else begin
+    // Move to the next branch
+    repeat
+      if (AIterator^.Top = 0) then begin
+        AIterator^.Current := nil;
+        break;
+      end;
+
+      last := AIterator^.Current;
+      Dec(AIterator^.Top);
+      AIterator^.Current := AIterator^.Path[AIterator^.Top];
+    until (last <> AIterator^.Current^.Links[ADirection]);
+  end;
+
+  Result := AIterator^.Current;
+end;
+
+class function TRBTree<T,KCOMP>.TreeIteratorMoveNext(
+  AIterator : PBaseIterator
+) : PRBTreeNode;static;
+begin
+  Result := TreeIteratorMove(AIterator,True);
+end;
+
+class function TRBTree<T,KCOMP>.TreeIteratorMovePrevious(
+  AIterator : PBaseIterator
+) : PRBTreeNode;static;
+begin
+  Result := TreeIteratorMove(AIterator,False);
+end;
+
+function TRBTree<T,KCOMP>.CreateIterator(
+  const ANode      : PRBTreeNode;
+  const ADirection : Boolean
+) : TIterator;
+var
+  h : PBaseIterator;
+begin
+  h := TreeCreateIterator();
+  TreeInitIterator(h,Self,ANode,ADirection);
+  Result := TIterator.Create(h);
+end;
+
+class function TRBTree<T,KCOMP>.DefaultCreateNode(AContext: Pointer): PRBTreeNode;
+begin
+  New(Result);
+end;
+
+class procedure TRBTree<T,KCOMP>.DefaultFreeNode(ANode: PRBTreeNode; AContext: Pointer);
+begin
+  Dispose(ANode);
+end;
+
+constructor TRBTree<T,KCOMP>.Create(const AAllocator  : PRBTreeAllocator);
+begin
+  Root := nil;
+  Allocator := AAllocator^;
+  //Comparator := TRBTreeNodeComparator.Create();
+end;
+
+constructor TRBTree < T, KCOMP > .Create();
+var
+  a : TRBTreeAllocator;
+begin
+  a.CreateNode := TRBTreeNodeCreator(DefaultCreateNode);
+  a.FreeNode := TRBTreeNodeDestructor(DefaultFreeNode);
+  Create(@a);
+end;
+
+destructor TRBTree<T,KCOMP>.Destroy;
+begin
+  Clear();
+  //Comparator.Free();
+  inherited;
+end;
+
+procedure TRBTree<T,KCOMP>.Clear();
+var
+  it, save : PRBTreeNode;
+begin
+  it := Root;
+
+  while (it <> nil) do begin
+    if (it^.Links[False] <> nil) then begin
+      // Right rotation
+      save := it^.Links[False];
+      it^.Links[False] := save^.Links[True];
+      save^.Links[True] := it;
+    end else begin
+      save := it^.Links[True];
+      Allocator.FreeNode(it,Self);
+    end;
+    it := save;
+  end;
+end;
+
+function TRBTree<T,KCOMP>.FindNode(const AData: TRBTreeNodeData): PRBTreeNode;
+var
+  it : PRBTreeNode;
+  cp : TRBTreeNodeComparator;
+  dir : Boolean;
+begin
+  Result := nil;
+  it := Root;
+  if (it = nil) then
+    exit;
+  cp := Comparator;
+  while (it <> nil) do begin
+    if (cp.Compare(it^.Data,AData) = 0) then begin
+      Result := it;
+      Break;
+    end;
+    dir := (cp.Compare(it^.Data,AData) < 0);
+    it := it^.Links[dir];
+  end;
+end;
+
+function TRBTree<T,KCOMP>.Insert(const AData: TRBTreeNodeData): PRBTreeNode;
+var
+  head : TRBTreeNode;
+  g, t : PRBTreeNode; // Grandparent & parent
+  p, q : PRBTreeNode; // Iterator & parent
+  dir, last, dir2 : Boolean;
+  cp : TRBTreeNodeComparator;
+begin
+  if (Root = nil) then begin
+    // Empty tree case
+    Root := InitNode(Allocator.CreateNode(Self),AData);
+    Result := Root;
+  end else begin
+    FillChar(head,SizeOf(head),0); // False tree root
+
+    dir := False;
+    last := False;
+
+    // Set up helpers
+    t := @head;
+    g := nil;
+    p := nil;
+    t^.Links[True] := Root;
+    q := t^.Links[True];
+
+  // Search down the tree
+    cp := Comparator;
+    while True do begin
+      if (q = nil) then begin
+        // Insert new node at the bottom
+        q := InitNode(Allocator.CreateNode(Self),AData);
+        p^.Links[dir] := q;
+      end else if IsRed(q^.Links[False]) and IsRed(q^.Links[True]) then begin
+        // Color flip
+        q^.Red := True;
+        q^.Links[False]^.Red := False;
+        q^.Links[True]^.Red := False;
+      end;
+
+      // Fix red violation
+      if IsRed(q) and IsRed(p) then begin
+        dir2 := (t^.Links[True] = g);
+        if (q = p^.Links[last]) then
+          t^.Links[dir2] := RotateSingle(g, not last)
+        else
+          t^.Links[dir2] := RotateDouble(g, not last );
+      end;
+
+      // Stop if found
+      if (cp.Compare(q^.Data,AData) = 0) then
+        break;
+
+      last := dir;
+      dir := (cp.Compare(q^.Data,AData) < 0);
+
+      // Update helpers
+      if (g <> nil) then
+        t := g;
+      g := p;
+      p := q;
+      q := q^.Links[dir];
+    end;
+
+    // Update root
+     Root := head.Links[True];
+  end;
+
+  // Make root black
+  Root^.Red := False;
+end;
+
+function TRBTree<T,KCOMP>.Remove(const AData: TRBTreeNodeData): Boolean;
+var
+  head : TRBTreeNode;
+  q, p, g, f, s : PRBTreeNode;
+  dir, last, dir2 : Boolean;
+  cp : TRBTreeNodeComparator;
+begin
+  Result := False;
+  if (Root = nil) then
+    exit;
+
+  FillChar(head,SizeOf(head),0); // False tree root
+  f := nil;
+  dir := True;
+
+  // Set up helpers
+  q := @head;
+  p := nil;
+  g := nil;
+  q^.Links[True] := Root;
+
+  // Search and push a red down
+  cp := Comparator;
+  while (q^.Links[dir] <> nil) do begin
+    last := dir;
+
+    // Update helpers
+    g := p;
+    p := q;
+    q := q^.Links[dir];
+    dir := (cp.Compare(q^.Data,AData) < 0);
+
+    // Save found node
+    if (cp.Compare(q^.Data,AData) = 0) then
+      f := q;
+
+    // Push the red node down
+    if not(IsRed(q)) and not(IsRed(q^.Links[dir])) then begin
+      if IsRed(q^.Links[not dir]) then begin
+        p^.Links[last] := RotateSingle(q,dir);
+        p := p^.Links[last];
+      end else if not IsRed(q^.Links[not dir]) then begin
+        s := p^.Links[not last];
+
+        if (s <> nil) then begin
+          if not(IsRed(s^.Links[not last])) and not(IsRed(s^.Links[last])) then begin
+            // Color flip
+            p^.Red := False;
+            s^.Red := True;
+            q^.Red := True;
+          end else begin
+            dir2 := (g^.Links[True] = p);
+
+            if IsRed(s^.Links[last]) then
+              g^.Links[dir2] := RotateDouble(p,last)
+            else if IsRed(s^.Links[not last]) then
+              g^.Links[dir2] := RotateSingle(p,last);
+
+            // Ensure correct coloring
+            g^.Links[dir2]^.Red := True;
+            q^.Red := g^.Links[dir2]^.Red;
+            g^.Links[dir2]^.Links[False]^.Red := False;
+            g^.Links[dir2]^.Links[True]^.Red := False;
+          end;
+        end;
+      end;
+    end;
+  end;
+
+  // Replace and remove if found
+  if (f <> nil) then begin
+    f^.Data := q^.Data;
+    p^.Links[(p^.Links[True] = q)] :=
+      q^.Links[(q^.Links[False] = nil)];
+    Allocator.FreeNode(q,Self);
+    Result := True;
+  end;
+
+  // Update root and make it black
+  Root := head.Links[True];
+  if (Root <> nil) then
+    Root^.Red := False;
+end;
+
+function TRBTree<T,KCOMP>.CreateForwardIterator(const ANode : PRBTreeNode) : TIterator;
+begin
+  Result := CreateIterator(ANode,False);
+end;
+
+function TRBTree<T,KCOMP>.CreateForwardIterator() : TIterator;
+begin
+  Result := CreateForwardIterator(Root);
+end;
+
+function TRBTree<T,KCOMP>.CreateBackwardIterator(const ANode : PRBTreeNode) : TIterator;
+begin
+  Result := CreateIterator(ANode,True);
+end;
+
+function TRBTree<T,KCOMP>.CreateBackwardIterator() : TIterator;
+begin
+  Result := CreateBackwardIterator(Root);
+end;
+
+{$ifdef RB_DEBUG}
+function TRBTree<T,KCOMP>.SelfAssert(ARoot : PRBTreeNode; var AErrorMessage: string): Boolean;
+var
+  lh, rh : Boolean;
+  ln, rn : PRBTreeNode;
+  e : string;
+begin
+  AErrorMessage := '';
+  if (ARoot = nil) then begin
+    Result := True;
+    exit;
+  end;
+
+  e := '';
+  ln := ARoot^.Links[False];
+  rn := ARoot^.Links[True];
+
+  // Consecutive red links
+  if IsRed(ARoot) then begin
+    if IsRed(ln) or IsRed(rn) then begin
+      AErrorMessage := 'Red violation';
+      Result := False;
+      exit;
+    end;
+  end;
+
+  lh := SelfAssert(ln,e);
+  AErrorMessage := AErrorMessage + ' ' + e;
+
+  rh := SelfAssert(rn,e);
+  AErrorMessage := AErrorMessage + ' ' + e;
+
+  // Invalid binary search tree
+  if ( ( (ln <> nil) and (Comparator.Compare(ln^.Data,ARoot^.Data) >= 0) ) or
+     ( (rn <> nil) and (Comparator.Compare(rn^.Data,ARoot^.Data) <= 0) ) )
+  then begin
+    AErrorMessage := AErrorMessage + ' ' + 'Binary tree violation';
+    Result := False;
+    Exit;
+  end;
+
+  // Black height mismatch
+  if ( lh and rh and (lh <> rh) ) then begin
+    AErrorMessage := AErrorMessage + ' ' + 'Black violation';
+    Result := False;
+    Exit;
+  end;
+
+  Result := lh and rh;
+end;
+
+function TRBTree<T,KCOMP>.SelfAssert(var AErrorMessage: string): Boolean;
+begin
+  Result := Self.SelfAssert(Root, AErrorMessage);
+end;
+
+{$endif RB_DEBUG}
+
+constructor TRBTree<T,KCOMP>.TIterator.Create(AHandle : PBaseIterator);
+begin
+  inherited Create();
+  FHandle := AHandle;
+  FResetState := True;
+end;
+
+destructor TRBTree<T,KCOMP>.TIterator.Destroy();
+begin
+  TreeFreeIterator(FHandle);
+  inherited Destroy;
+end;
+
+function TRBTree<T,KCOMP>.TIterator.MoveNext : Boolean;
+begin
+  if FResetState then begin
+    FResetState := False;
+    Result := (FHandle^.Current <> nil);
+    exit;
+  end;
+  Result := (TreeIteratorMoveNext(FHandle) <> nil);
+end;
+
+function TRBTree<T,KCOMP>.TIterator.MovePrevious : Boolean;
+begin
+  if FResetState then begin
+    FResetState := False;
+    Result := (FHandle^.Current <> nil);
+    exit;
+  end;
+  Result := (TreeIteratorMovePrevious(FHandle) <> nil);
+end;
+
+function TRBTree<T,KCOMP>.TIterator.GetCurrent : TRBTreeNodeData;
+begin
+  Result := GetCurrentNode()^.Data;
+end;
+
+function TRBTree<T,KCOMP>.TIterator.GetCurrentNode : PRBTreeNode;
+begin
+  Result := FHandle^.Current;
+end;
+
+procedure TRBTree<T,KCOMP>.TIterator.Reset();
+begin
+  FResetState := True;
+  TreeInitIterator(FHandle,FHandle^.Tree,FHandle^.StartingNode,FHandle^.StartingDir)
+end;
+
+{ TOrdinalComparator<T> }
+
+class function TOrdinalComparator<T>.Compare(const A, B: TOrdinalType): Integer;
+begin
+  if (A = B) then
+    exit(0);
+  if (A > B) then
+    exit(1);
+  exit(-1);
+end;
+
+end.
+

utils/unicode/helper.pas

@@ -0,0 +1,4036 @@
+{   Unicode parser helper unit.
+
+    Copyright (c) 2012 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. }
+unit helper;
+
+{$mode delphi}
+{$H+}
+{$PACKENUM 1}
+{$pointermath on}
+
+interface
+
+uses
+  Classes, SysUtils, StrUtils;
+
+const
+  SLicenseText =
+    '    {   Unicode implementation tables. ' + sLineBreak +
+    ' ' + sLineBreak +
+    '        Copyright (c) 2013 by Inoussa OUEDRAOGO ' + sLineBreak +
+    ' ' + sLineBreak +
+    '        Permission is hereby granted, free of charge, to any person ' + sLineBreak +
+    '        obtaining a copy of the Unicode data files and any associated ' + sLineBreak +
+    '        documentation (the "Data Files") or Unicode software and any ' + sLineBreak +
+    '        associated documentation (the "Software") to deal in the Data ' + sLineBreak +
+    '        Files or Software without restriction, including without ' + sLineBreak +
+    '        limitation the rights to use, copy, modify, merge, publish, ' + sLineBreak +
+    '        distribute, and/or sell copies of the Data Files or Software, ' + sLineBreak +
+    '        and to permit persons to whom the Data Files or Software are ' + sLineBreak +
+    '        furnished to do so, provided that (a) the above copyright ' + sLineBreak +
+    '        notice(s) and this permission notice appear with all copies ' + sLineBreak +
+    '        of the Data Files or Software, (b) both the above copyright ' + sLineBreak +
+    '        notice(s) and this permission notice appear in associated ' + sLineBreak +
+    '        documentation, and (c) there is clear notice in each modified ' + sLineBreak +
+    '        Data File or in the Software as well as in the documentation ' + sLineBreak +
+    '        associated with the Data File(s) or Software that the data or ' + sLineBreak +
+    '        software has been modified. ' + sLineBreak +
+    ' ' + sLineBreak +
+    ' ' + sLineBreak +
+    '        This program is distributed in the hope that it will be useful, ' + sLineBreak +
+    '        but WITHOUT ANY WARRANTY; without even the implied warranty of ' + sLineBreak +
+    '        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. }';
+
+
+type
+  // Unicode General Category
+  TUnicodeCategory = (
+    ucUppercaseLetter,             // Lu = Letter, uppercase
+    ucLowercaseLetter,             //  Ll = Letter, lowercase
+    ucTitlecaseLetter,             //  Lt = Letter, titlecase
+    ucModifierLetter,              //  Lm = Letter, modifier
+    ucOtherLetter,                 //  Lo = Letter, other
+
+    ucNonSpacingMark,              //  Mn = Mark, nonspacing
+    ucCombiningMark,               //  Mc = Mark, spacing combining
+    ucEnclosingMark,               //  Me = Mark, enclosing
+
+    ucDecimalNumber,               //  Nd = Number, decimal digit
+    ucLetterNumber,                //  Nl = Number, letter
+    ucOtherNumber,                 //  No = Number, other
+
+    ucConnectPunctuation,          //  Pc = Punctuation, connector
+    ucDashPunctuation,             //  Pd = Punctuation, dash
+    ucOpenPunctuation,             //  Ps = Punctuation, open
+    ucClosePunctuation,            //  Pe = Punctuation, close
+    ucInitialPunctuation,          //  Pi = Punctuation, initial quote (may behave like Ps or Pe depending on usage)
+    ucFinalPunctuation,            //  Pf = Punctuation, final quote (may behave like Ps or Pe depending on usage)
+    ucOtherPunctuation,            //  Po = Punctuation, other
+
+    ucMathSymbol,                  //  Sm = Symbol, math
+    ucCurrencySymbol,              //  Sc = Symbol, currency
+    ucModifierSymbol,              //  Sk = Symbol, modifier
+    ucOtherSymbol,                 //  So = Symbol, other
+
+    ucSpaceSeparator,              //  Zs = Separator, space
+    ucLineSeparator,               //  Zl = Separator, line
+    ucParagraphSeparator,          //  Zp = Separator, paragraph
+
+    ucControl,                     //  Cc = Other, control
+    ucFormat,                      //  Cf = Other, format
+    ucSurrogate,                   //  Cs = Other, surrogate
+    ucPrivateUse,                  //  Co = Other, private use
+    ucUnassigned                   //  Cn = Other, not assigned (including noncharacters)
+  );
+
+
+  TUInt24Rec = packed record
+  public
+  {$ifdef FPC_LITTLE_ENDIAN}
+    byte0, byte1, byte2 : Byte;
+  {$else FPC_LITTLE_ENDIAN}
+    byte2, byte1, byte0 : Byte;
+  {$endif FPC_LITTLE_ENDIAN}
+  public
+    class operator Implicit(a : TUInt24Rec) : Cardinal;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Implicit(a : TUInt24Rec) : LongInt;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Implicit(a : TUInt24Rec) : Word;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Implicit(a : TUInt24Rec) : Byte;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Implicit(a : Cardinal) : TUInt24Rec;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Equal(a, b: TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+
+    class operator Equal(a : TUInt24Rec; b : Cardinal): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Equal(a : Cardinal; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+
+    class operator Equal(a : TUInt24Rec; b : LongInt): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Equal(a : LongInt; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+
+    class operator Equal(a : TUInt24Rec; b : Word): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Equal(a : Word; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+
+    class operator Equal(a : TUInt24Rec; b : Byte): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator Equal(a : Byte; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+
+    class operator NotEqual(a, b: TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator NotEqual(a : TUInt24Rec; b : Cardinal): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator NotEqual(a : Cardinal; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator GreaterThan(a, b: TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator GreaterThan(a : TUInt24Rec; b : Cardinal): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator GreaterThan(a : Cardinal; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator GreaterThanOrEqual(a, b: TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator GreaterThanOrEqual(a : TUInt24Rec; b : Cardinal): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator GreaterThanOrEqual(a : Cardinal; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator LessThan(a, b: TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator LessThan(a : TUInt24Rec; b : Cardinal): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator LessThan(a : Cardinal; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator LessThanOrEqual(a, b: TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator LessThanOrEqual(a : TUInt24Rec; b : Cardinal): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+    class operator LessThanOrEqual(a : Cardinal; b : TUInt24Rec): Boolean;{$ifdef USE_INLINE}inline;{$ENDIF}
+  end;
+
+  UInt24 = TUInt24Rec;
+  PUInt24 = ^UInt24;
+  TUnicodeCodePoint = Cardinal;
+  TUnicodeCodePointArray = array of TUnicodeCodePoint;
+  TDecompositionArray = array of TUnicodeCodePointArray;
+  TNumericValue = Double;
+  TNumericValueArray = array of TNumericValue;
+
+  TBlockItemRec = packed record
+    RangeStart    : TUnicodeCodePoint;
+    RangeEnd      : TUnicodeCodePoint;
+    Name          : string[120];
+    CanonicalName : string[120];
+  end;
+  TBlocks = array of TBlockItemRec;
+
+  PPropRec = ^TPropRec;
+
+  { TPropRec }
+
+  TPropRec = packed record
+  private
+    function GetCategory : TUnicodeCategory;inline;
+    procedure SetCategory(AValue : TUnicodeCategory);
+    function GetWhiteSpace : Boolean;inline;
+    procedure SetWhiteSpace(AValue : Boolean);
+    function GetHangulSyllable : Boolean;inline;
+    procedure SetHangulSyllable(AValue : Boolean);
+  public
+    CategoryData    : Byte;
+
+    PropID          : Word;
+    CCC             : Byte; // Canonical Combining Class
+    NumericIndex    : Byte;
+    SimpleUpperCase : UInt24;
+    SimpleLowerCase : UInt24;
+    DecompositionID : SmallInt;
+  public
+    property Category : TUnicodeCategory read GetCategory write SetCategory;
+    property WhiteSpace : Boolean read GetWhiteSpace write SetWhiteSpace;
+    property HangulSyllable : Boolean read GetHangulSyllable write SetHangulSyllable;
+  end;
+  TPropRecArray = array of TPropRec;
+
+  TDecompositionIndexRec = packed record
+    StartPosition : Word;
+    Length        : Byte;
+  end;
+  TDecompositionBook = packed record
+    Index      : array of TDecompositionIndexRec;
+    CodePoints : array of TUnicodeCodePoint;
+  end;
+
+  PDataLineRec = ^TDataLineRec;
+  TDataLineRec = record
+    PropID    : Integer;
+    case LineType : Byte of
+      0 : (CodePoint : TUnicodeCodePoint);
+      1 : (StartCodePoint, EndCodePoint : TUnicodeCodePoint);
+  end;
+  TDataLineRecArray = array of TDataLineRec;
+
+  TCodePointRec = record
+    case LineType : Byte of
+      0 : (CodePoint : TUnicodeCodePoint);
+      1 : (StartCodePoint, EndCodePoint : TUnicodeCodePoint);
+  end;
+  TCodePointRecArray = array of TCodePointRec;
+
+  TPropListLineRec = packed record
+    CodePoint : TCodePointRec;
+    PropName  : string[123];
+  end;
+  TPropListLineRecArray = array of TPropListLineRec;
+
+  TUCA_WeightRec = packed record
+    Weights  : array[0..3] of Cardinal;
+    Variable : Boolean;
+  end;
+  TUCA_WeightRecArray = array of TUCA_WeightRec;
+
+  TUCA_LineContextItemRec = packed record
+  public
+    CodePoints : TUnicodeCodePointArray;
+    Weights    : TUCA_WeightRecArray;
+  public
+    procedure Clear();
+    procedure Assign(ASource : TUCA_LineContextItemRec);
+    function Clone() : TUCA_LineContextItemRec;
+  end;
+  PUCA_LineContextItemRec = ^TUCA_LineContextItemRec;
+
+  TUCA_LineContextRec = packed record
+  public
+    Data : array of TUCA_LineContextItemRec;
+  public
+    procedure Clear();
+    procedure Assign(ASource : TUCA_LineContextRec);
+    function Clone() : TUCA_LineContextRec;
+  end;
+  PUCA_LineContextRec = ^TUCA_LineContextRec;
+
+  { TUCA_LineRec }
+
+  TUCA_LineRec = packed record
+  public
+    CodePoints : TUnicodeCodePointArray;
+    Weights    : TUCA_WeightRecArray;
+    Context    : TUCA_LineContextRec;
+    //Variable   : Boolean;
+    Deleted    : Boolean;
+    Stored     : Boolean;
+  public
+    procedure Clear();
+    procedure Assign(ASource : TUCA_LineRec);
+    function Clone() : TUCA_LineRec;
+    function HasContext() : Boolean;
+  end;
+  PUCA_LineRec = ^TUCA_LineRec;
+  TUCA_VariableKind = (
+    ucaShifted, ucaNonIgnorable, ucaBlanked, ucaShiftedTrimmed,
+    ucaIgnoreSP
+  );
+  TUCA_DataBook = packed record
+    Version        : string;
+    VariableWeight : TUCA_VariableKind;
+    Backwards      : array[0..3] of Boolean;
+    Lines          : array of TUCA_LineRec;
+  end;
+  PUCA_DataBook = ^TUCA_DataBook;
+  TUCA_DataBookIndex = array of Integer;
+
+const
+  BIT_POS_VALIDE = 0;
+  BIT_POS_COMPRESS_WEIGHT_1 = BIT_POS_VALIDE + 1;
+  BIT_POS_COMPRESS_WEIGHT_2 = BIT_POS_COMPRESS_WEIGHT_1 + 1;
+type
+  TWeightLength = 0..24;
+  TUCA_PropWeights = packed record
+    Weights  : array[0..2] of Word;
+    //Variable : Byte;
+  end;
+  PUCA_PropWeights = ^TUCA_PropWeights;
+
+  TUCA_PropItemContextRec = packed record
+    CodePointCount : Byte;
+    WeightCount    : Byte;
+    //CodePoints     : UInt24;
+    //Weights        : TUCA_PropWeights;
+  end;
+  PUCA_PropItemContextRec = ^TUCA_PropItemContextRec;
+  TUCA_PropItemContextTreeNodeRec = packed record
+    Left    : Word;
+    Right   : Word;
+    Data    : TUCA_PropItemContextRec;
+  end;
+  PUCA_PropItemContextTreeNodeRec = ^TUCA_PropItemContextTreeNodeRec;
+
+  TUCA_PropItemContextTreeRec = packed record
+  public
+    Size : UInt24;
+  public
+    function GetData:PUCA_PropItemContextTreeNodeRec;inline;
+    property Data : PUCA_PropItemContextTreeNodeRec read GetData;
+  end;
+  PUCA_PropItemContextTreeRec = ^TUCA_PropItemContextTreeRec;
+
+  { TUCA_PropItemRec }
+
+  TUCA_PropItemRec = packed record
+  private
+    const FLAG_CODEPOINT  = 1;
+    const FLAG_CONTEXTUAL = 2;
+    const FLAG_DELETION   = 3;
+  private
+    function GetWeightLength: TWeightLength;inline;
+    procedure SetWeightLength(AValue: TWeightLength);inline;
+    function GetWeightSize : Word;inline;
+  public
+    //CodePoint    : UInt24;
+    Valid        : Byte;// On First Bit
+    ChildCount   : Byte;
+    Size         : Word;
+    Flags        : Byte;
+  public
+    function GetCodePoint() : UInt24;//inline;
+    property CodePoint : UInt24 read GetCodePoint;
+    //WeightLength is stored in the 5 last bits of "Valid"
+    property WeightLength : TWeightLength read GetWeightLength write SetWeightLength;
+    //Weights    : array[0..WeightLength] of TUCA_PropWeights;
+    procedure GetWeightArray(ADest : PUCA_PropWeights);
+    function GetSelfOnlySize() : Word;inline;
+
+    procedure SetContextual(AValue : Boolean);inline;
+    function GetContextual() : Boolean;inline;
+    property Contextual : Boolean read GetContextual write setContextual;
+    function GetContext() : PUCA_PropItemContextTreeRec;
+    procedure SetDeleted(AValue : Boolean);inline;
+    function IsDeleted() : Boolean;inline;
+  end;
+  PUCA_PropItemRec = ^TUCA_PropItemRec;
+  TUCA_PropIndexItem = packed record
+    CodePoint : Cardinal;
+    Position  : Integer;
+  end;
+  PUCA_PropIndexItem = ^TUCA_PropIndexItem;
+  TUCA_PropBook = packed record
+    ItemSize      : Integer;
+    Index         : array of TUCA_PropIndexItem;
+    Items         : PUCA_PropItemRec;
+    VariableLowLimit  : Word;
+    VariableHighLimit : Word;
+  end;
+  PUCA_PropBook = ^TUCA_PropBook;
+
+  TBmpFirstTable = array[0..255] of Byte;
+  TBmpSecondTableItem = array[0..255] of Word;
+  TBmpSecondTable = array of TBmpSecondTableItem;
+
+  T3lvlBmp1Table = array[0..255] of Byte;
+  T3lvlBmp2TableItem = array[0..15] of Word;
+  T3lvlBmp2Table = array of T3lvlBmp2TableItem;
+  T3lvlBmp3TableItem = array[0..15] of Word;
+  T3lvlBmp3Table = array of T3lvlBmp3TableItem;
+
+  TucaBmpFirstTable = array[0..255] of Byte;
+  TucaBmpSecondTableItem = array[0..255] of Cardinal;
+  TucaBmpSecondTable = array of TucaBmpSecondTableItem;
+  PucaBmpFirstTable = ^TucaBmpFirstTable;
+  PucaBmpSecondTable = ^TucaBmpSecondTable;
+
+const
+  LOW_SURROGATE_BEGIN  = Word($DC00);
+  LOW_SURROGATE_END    = Word($DFFF);
+  LOW_SURROGATE_COUNT  = LOW_SURROGATE_END - LOW_SURROGATE_BEGIN + 1;
+
+  HIGH_SURROGATE_BEGIN = Word($D800);
+  HIGH_SURROGATE_END   = Word($DBFF);
+  HIGH_SURROGATE_COUNT = HIGH_SURROGATE_END - HIGH_SURROGATE_BEGIN + 1;
+type
+  TOBmpFirstTable = array[0..(HIGH_SURROGATE_COUNT-1)] of Word;
+  TOBmpSecondTableItem = array[0..(LOW_SURROGATE_COUNT-1)] of Word;
+  TOBmpSecondTable = array of TOBmpSecondTableItem;
+
+  T3lvlOBmp1Table = array[0..1023] of Byte;
+  T3lvlOBmp2TableItem = array[0..31] of Word;
+  T3lvlOBmp2Table = array of T3lvlOBmp2TableItem;
+  T3lvlOBmp3TableItem = array[0..31] of Word;
+  T3lvlOBmp3Table = array of T3lvlOBmp3TableItem;
+
+  TucaOBmpFirstTable = array[0..(HIGH_SURROGATE_COUNT-1)] of Word;
+  TucaOBmpSecondTableItem = array[0..(LOW_SURROGATE_COUNT-1)] of Cardinal;
+  TucaOBmpSecondTable = array of TucaOBmpSecondTableItem;
+  PucaOBmpFirstTable = ^TucaOBmpFirstTable;
+  PucaOBmpSecondTable = ^TucaOBmpSecondTable;
+
+type
+  TEndianKind = (ekLittle, ekBig);
+const
+  THIS_ENDIAN =
+{$IFDEF ENDIAN_LITTLE}
+    ekLittle;
+{$ENDIF ENDIAN_LITTLE}
+{$IFDEF ENDIAN_BIG}
+    ekBig;
+{$ENDIF ENDIAN_BIG}
+
+  procedure GenerateLicenceText(ADest : TStream);
+
+  function BoolToByte(AValue : Boolean): Byte;inline;
+
+  function IsHangulSyllable(
+    const ACodePoint  : TUnicodeCodePoint;
+    const AHangulList : TCodePointRecArray
+  ) : Boolean;
+  procedure ParseHangulSyllableTypes(
+        ADataAStream   : TMemoryStream;
+    var ACodePointList : TCodePointRecArray
+  );
+
+  procedure ParseProps(
+        ADataAStream   : TMemoryStream;
+    var APropList      : TPropListLineRecArray
+  );
+  function FindCodePointsByProperty(
+    const APropName : string;
+    const APropList : TPropListLineRecArray
+  ) : TCodePointRecArray;
+
+  procedure ParseBlokcs(
+        ADataAStream   : TMemoryStream;
+    var ABlocks        : TBlocks
+  );
+  procedure ParseUCAFile(
+        ADataAStream : TMemoryStream;
+    var ABook        : TUCA_DataBook
+  );
+  procedure MakeUCA_Props(
+          ABook         : PUCA_DataBook;
+    out   AProps        : PUCA_PropBook
+  );
+  procedure FreeUcaBook(var ABook : PUCA_PropBook);
+  procedure MakeUCA_BmpTables(
+    var   AFirstTable   : TucaBmpFirstTable;
+    var   ASecondTable  : TucaBmpSecondTable;
+    const APropBook     : PUCA_PropBook
+  );
+  procedure MakeUCA_OBmpTables(
+    var   AFirstTable   : TucaOBmpFirstTable;
+    var   ASecondTable  : TucaOBmpSecondTable;
+    const APropBook     : PUCA_PropBook
+  );
+  function GetPropPosition(
+    const AHighS,
+          ALowS         : Word;
+    const AFirstTable   : PucaOBmpFirstTable;
+    const ASecondTable  : PucaOBmpSecondTable
+  ): Integer;inline;overload;
+
+  procedure GenerateUCA_Head(
+    ADest  : TStream;
+    ABook  : PUCA_DataBook;
+    AProps : PUCA_PropBook
+  );
+  procedure GenerateUCA_BmpTables(
+          AStream,
+          ABinStream    : TStream;
+    var   AFirstTable   : TucaBmpFirstTable;
+    var   ASecondTable  : TucaBmpSecondTable;
+    const AEndian       : TEndianKind
+  );
+  procedure GenerateUCA_PropTable(
+  // WARNING : files must be generated for each endianess (Little / Big)
+          ADest     : TStream;
+    const APropBook : PUCA_PropBook
+  );
+  procedure GenerateUCA_OBmpTables(
+          AStream,
+          ABinStream    : TStream;
+    var   AFirstTable   : TucaOBmpFirstTable;
+    var   ASecondTable  : TucaOBmpSecondTable;
+    const AEndian       : TEndianKind
+  );
+
+  procedure Parse_UnicodeData(
+          ADataAStream   : TMemoryStream;
+    var   APropList      : TPropRecArray;
+    var   ANumericTable  : TNumericValueArray;
+    var   ADataLineList  : TDataLineRecArray;
+    var   ADecomposition : TDecompositionArray;
+    const AHangulList    : TCodePointRecArray;
+    const AWhiteSpaces   : TCodePointRecArray
+  );
+  procedure MakeDecomposition(
+    const ARawData : TDecompositionArray;
+    var   ABook    : TDecompositionBook
+  );
+
+  procedure MakeBmpTables(
+    var   AFirstTable   : TBmpFirstTable;
+    var   ASecondTable  : TBmpSecondTable;
+    const APropList     : TPropRecArray;
+    const ADataLineList : TDataLineRecArray
+  );
+  procedure MakeBmpTables3Levels(
+    var   AFirstTable   : T3lvlBmp1Table;
+    var   ASecondTable  : T3lvlBmp2Table;
+    var   AThirdTable  : T3lvlBmp3Table;
+    const ADataLineList : TDataLineRecArray
+  );
+  procedure GenerateBmpTables(
+          ADest : TStream;
+    var   AFirstTable   : TBmpFirstTable;
+    var   ASecondTable  : TBmpSecondTable
+  );
+  procedure Generate3lvlBmpTables(
+          ADest : TStream;
+    var   AFirstTable   : T3lvlBmp1Table;
+    var   ASecondTable  : T3lvlBmp2Table;
+    var   AThirdTable   : T3lvlBmp3Table
+  );
+  procedure GeneratePropTable(
+          ADest     : TStream;
+    const APropList : TPropRecArray;
+    const AEndian   : TEndianKind
+  );
+  procedure GenerateNumericTable(
+          ADest         : TStream;
+    const ANumList      : TNumericValueArray;
+    const ACompleteUnit : Boolean
+  );
+  procedure GenerateDecompositionBookTable(
+          ADest   : TStream;
+    const ABook   : TDecompositionBook;
+    const AEndian : TEndianKind
+  );
+  procedure GenerateOutBmpTable(
+          ADest     : TStream;
+    const AList : TDataLineRecArray
+  );
+
+  function Compress(const AData : TDataLineRecArray) : TDataLineRecArray;
+
+  function EvaluateFloat(const AStr : string) : Double;
+  function StrToCategory(const AStr : string) : TUnicodeCategory;
+  function StringToCodePoint(ACP : string) : TUnicodeCodePoint;
+  function IsWhiteSpace(
+    const ACodePoint   : TUnicodeCodePoint;
+    const AWhiteSpaces : TCodePointRecArray
+  ) : Boolean;
+
+  function GetPropID(
+          ACodePoint    : TUnicodeCodePoint;
+    const ADataLineList : TDataLineRecArray
+  ) : Cardinal;
+
+//--------------------
+  procedure MakeOBmpTables(
+    var   AFirstTable   : TOBmpFirstTable;
+    var   ASecondTable  : TOBmpSecondTable;
+    const ADataLineList : TDataLineRecArray
+  );
+  procedure MakeOBmpTables3Levels(
+    var   AFirstTable   : T3lvlOBmp1Table;
+    var   ASecondTable  : T3lvlOBmp2Table;
+    var   AThirdTable  : T3lvlOBmp3Table;
+    const ADataLineList : TDataLineRecArray
+  );
+  procedure GenerateOBmpTables(
+          ADest : TStream;
+    var   AFirstTable   : TOBmpFirstTable;
+    var   ASecondTable  : TOBmpSecondTable
+  );
+  procedure Generate3lvlOBmpTables(
+          ADest : TStream;
+    var   AFirstTable   : T3lvlOBmp1Table;
+    var   ASecondTable  : T3lvlOBmp2Table;
+    var   AThirdTable   : T3lvlOBmp3Table
+  );
+  function GetProp(
+    const AHighS,
+          ALowS         : Word;
+    const AProps        : TPropRecArray;
+    var   AFirstTable   : TOBmpFirstTable;
+    var   ASecondTable  : TOBmpSecondTable
+  ): PPropRec; inline;overload;
+  function GetProp(
+    const AHighS,
+          ALowS         : Word;
+    const AProps        : TPropRecArray;
+    var   AFirstTable   : T3lvlOBmp1Table;
+    var   ASecondTable  : T3lvlOBmp2Table;
+    var   AThirdTable   : T3lvlOBmp3Table
+  ): PPropRec; inline;overload;
+  procedure FromUCS4(const AValue : TUnicodeCodePoint; var AHighS, ALowS : Word);inline;
+  function ToUCS4(const AHighS, ALowS : Word) : TUnicodeCodePoint; inline;
+//--------------------
+
+type
+  TBitOrder = 0..7;
+
+  function IsBitON(const AData : Byte; const ABit : TBitOrder) : Boolean ;{$IFDEF USE_INLINE}inline;{$ENDIF}
+  procedure SetBit(var AData : Byte; const ABit : TBitOrder; const AValue : Boolean);{$IFDEF USE_INLINE}inline;{$ENDIF}
+
+  function GenerateEndianIncludeFileName(const AStoreName : string): string;inline;
+
+resourcestring
+  SInsufficientMemoryBuffer = 'Insufficient Memory Buffer';
+
+implementation
+uses
+  typinfo, Math, AVL_Tree,
+  trie;
+
+
+type
+
+  TCardinalRec = packed record
+  {$ifdef FPC_LITTLE_ENDIAN}
+    byte0, byte1, byte2, byte3 : Byte;
+  {$else FPC_LITTLE_ENDIAN}
+    byte3, byte2, byte1, byte0 : Byte;
+  {$endif FPC_LITTLE_ENDIAN}
+  end;
+
+  TWordRec = packed record
+  {$ifdef FPC_LITTLE_ENDIAN}
+    byte0, byte1 : Byte;
+  {$else FPC_LITTLE_ENDIAN}
+    byte1, byte0 : Byte;
+  {$endif FPC_LITTLE_ENDIAN}
+  end;
+
+{ TUInt24Rec }
+
+class operator TUInt24Rec.Implicit(a : TUInt24Rec) : Cardinal;
+begin
+  TCardinalRec(Result).byte0 := a.byte0;
+  TCardinalRec(Result).byte1 := a.byte1;
+  TCardinalRec(Result).byte2 := a.byte2;
+  TCardinalRec(Result).byte3 := 0;
+end;
+
+class operator TUInt24Rec.Implicit(a : TUInt24Rec) : LongInt;
+begin
+  Result := Cardinal(a);
+end;
+
+class operator TUInt24Rec.Implicit(a : TUInt24Rec) : Word;
+begin
+{$IFOPT R+}
+  if (a > $FFFF) then
+    Error(reIntOverflow);
+{$ENDIF R+}
+  TWordRec(Result).byte0 := a.byte0;
+  TWordRec(Result).byte1 := a.byte1;
+end;
+
+class operator TUInt24Rec.Implicit(a : TUInt24Rec) : Byte;
+begin
+{$IFOPT R+}
+  if (a > $FF) then
+    Error(reIntOverflow);
+{$ENDIF R+}
+  Result := a.byte0;
+end;
+
+class operator TUInt24Rec.Implicit(a : Cardinal) : TUInt24Rec;
+begin
+{$IFOPT R+}
+  if (a > $FFFFFF) then
+    Error(reIntOverflow);
+{$ENDIF R+}
+  Result.byte0 := TCardinalRec(a).byte0;
+  Result.byte1 := TCardinalRec(a).byte1;
+  Result.byte2 := TCardinalRec(a).byte2;
+end;
+
+class operator TUInt24Rec.Equal(a, b : TUInt24Rec) : Boolean;
+begin
+  Result := (a.byte0 = b.byte0) and (a.byte1 = b.byte1) and (a.byte2 = b.byte2);
+end;
+
+class operator TUInt24Rec.Equal(a : TUInt24Rec; b : Cardinal) : Boolean;
+begin
+  Result := (TCardinalRec(b).byte3 = 0) and
+            (a.byte0 = TCardinalRec(b).byte0) and
+            (a.byte1 = TCardinalRec(b).byte1) and
+            (a.byte2 = TCardinalRec(b).byte2);
+end;
+
+class operator TUInt24Rec.Equal(a : Cardinal; b : TUInt24Rec) : Boolean;
+begin
+  Result := (b = a);
+end;
+
+class operator TUInt24Rec.Equal(a : TUInt24Rec; b : LongInt) : Boolean;
+begin
+  Result := (LongInt(a) = b);
+end;
+
+class operator TUInt24Rec.Equal(a : LongInt; b : TUInt24Rec) : Boolean;
+begin
+  Result := (b = a);
+end;
+
+class operator TUInt24Rec.Equal(a : TUInt24Rec; b : Word) : Boolean;
+begin
+  Result := (a.byte2 = 0) and
+            (a.byte0 = TWordRec(b).byte0) and
+            (a.byte1 = TWordRec(b).byte1);
+end;
+
+class operator TUInt24Rec.Equal(a : Word; b : TUInt24Rec) : Boolean;
+begin
+  Result := (b = a);
+end;
+
+class operator TUInt24Rec.Equal(a : TUInt24Rec; b : Byte) : Boolean;
+begin
+  Result := (a.byte2 = 0) and
+            (a.byte1 = 0) and
+            (a.byte0 = b);
+end;
+
+class operator TUInt24Rec.Equal(a : Byte; b : TUInt24Rec) : Boolean;
+begin
+  Result := (b = a);
+end;
+
+class operator TUInt24Rec.NotEqual(a, b : TUInt24Rec) : Boolean;
+begin
+  Result := (a.byte0 <> b.byte0) or (a.byte1 <> b.byte1) or (a.byte2 <> b.byte2);
+end;
+
+class operator TUInt24Rec.NotEqual(a : TUInt24Rec; b : Cardinal) : Boolean;
+begin
+  Result := (TCardinalRec(b).byte3 <> 0) or
+            (a.byte0 <> TCardinalRec(b).byte0) or
+            (a.byte1 <> TCardinalRec(b).byte1) or
+            (a.byte2 <> TCardinalRec(b).byte2);
+end;
+
+class operator TUInt24Rec.NotEqual(a : Cardinal; b : TUInt24Rec) : Boolean;
+begin
+  Result := (b <> a);
+end;
+
+class operator TUInt24Rec.GreaterThan(a, b: TUInt24Rec): Boolean;
+begin
+  Result := (a.byte2 > b.byte2) or
+            ((a.byte2 = b.byte2) and (a.byte1 > b.byte1)) or
+            ((a.byte2 = b.byte2) and (a.byte1 = b.byte1) and (a.byte0 > b.byte0));
+end;
+
+class operator TUInt24Rec.GreaterThan(a: TUInt24Rec; b: Cardinal): Boolean;
+begin
+  Result := Cardinal(a) > b;
+end;
+
+class operator TUInt24Rec.GreaterThan(a: Cardinal; b: TUInt24Rec): Boolean;
+begin
+  Result := a > Cardinal(b);
+end;
+
+class operator TUInt24Rec.GreaterThanOrEqual(a, b: TUInt24Rec): Boolean;
+begin
+  Result := (a.byte2 > b.byte2) or
+            ((a.byte2 = b.byte2) and (a.byte1 > b.byte1)) or
+            ((a.byte2 = b.byte2) and (a.byte1 = b.byte1) and (a.byte0 >= b.byte0));
+end;
+
+class operator TUInt24Rec.GreaterThanOrEqual(a: TUInt24Rec; b: Cardinal): Boolean;
+begin
+  Result := Cardinal(a) >= b;
+end;
+
+class operator TUInt24Rec.GreaterThanOrEqual(a: Cardinal; b: TUInt24Rec): Boolean;
+begin
+  Result := a >= Cardinal(b);
+end;
+
+class operator TUInt24Rec.LessThan(a, b: TUInt24Rec): Boolean;
+begin
+  Result := (b > a);
+end;
+
+class operator TUInt24Rec.LessThan(a: TUInt24Rec; b: Cardinal): Boolean;
+begin
+  Result := Cardinal(a) < b;
+end;
+
+class operator TUInt24Rec.LessThan(a: Cardinal; b: TUInt24Rec): Boolean;
+begin
+  Result := a < Cardinal(b);
+end;
+
+class operator TUInt24Rec.LessThanOrEqual(a, b: TUInt24Rec): Boolean;
+begin
+  Result := (b >= a);
+end;
+
+class operator TUInt24Rec.LessThanOrEqual(a: TUInt24Rec; b: Cardinal): Boolean;
+begin
+  Result := Cardinal(a) <= b;
+end;
+
+class operator TUInt24Rec.LessThanOrEqual(a: Cardinal; b: TUInt24Rec): Boolean;
+begin
+  Result := a <= Cardinal(b);
+end;
+
+function GenerateEndianIncludeFileName(const AStoreName : string): string;inline;
+const
+  ENDIAN_SUFFIX =
+{$IFDEF ENDIAN_LITTLE}
+    'le';
+{$ENDIF ENDIAN_LITTLE}
+{$IFDEF ENDIAN_BIG}
+    'be';
+{$ENDIF ENDIAN_BIG}
+begin
+  Result := ExtractFilePath(AStoreName) +
+            ChangeFileExt(ExtractFileName(AStoreName),Format('_%s.inc',[ENDIAN_SUFFIX]));
+end;
+
+function IsBitON(const AData : Byte; const ABit : TBitOrder) : Boolean ;
+begin
+  Result := ( ( AData and ( 1 shl ABit ) ) <> 0 );
+end;
+
+procedure SetBit(var AData : Byte; const ABit : TBitOrder; const AValue : Boolean);
+begin
+  if AValue then
+    AData := AData or (1 shl (ABit mod 8))
+  else
+    AData := AData and ( not ( 1 shl ( ABit mod 8 ) ) );
+end;
+
+var
+  FS : TFormatSettings;
+
+function EvaluateFloat(const AStr : string) : Double;
+var
+  s, n, d : string;
+  i : Integer;
+begin
+  Result := 0;
+  s := Trim(AStr);
+  if (Length(s) > 0) then begin
+    i := Pos('/',s);
+    if (i < 1) then
+      Result := StrToFloat(s,FS)
+    else begin
+      n := Copy(s,1,i-1);
+      d := Copy(s,i+1,MaxInt);
+      Result := StrToInt(n) / StrToInt(d);
+    end;
+  end;
+end;
+
+function StrToCategory(const AStr : string) : TUnicodeCategory;
+var
+  s : string;
+begin
+  s := UpperCase(Trim(AStr));
+  if (s = 'LU') then
+    Result := ucUppercaseLetter
+  else if (s = 'LL') then
+    Result := ucLowercaseLetter
+  else if (s = 'LT') then
+    Result := ucTitlecaseLetter
+  else if (s = 'LM') then
+    Result := ucModifierLetter
+  else if (s = 'LO') then
+    Result := ucOtherLetter
+  else
+
+  if (s = 'MN') then
+    Result := ucNonSpacingMark
+  else if (s = 'MC') then
+    Result := ucCombiningMark
+  else if (s = 'ME') then
+    Result := ucEnclosingMark
+  else
+
+  if (s = 'ND') then
+    Result := ucDecimalNumber
+  else if (s = 'NL') then
+    Result := ucLetterNumber
+  else if (s = 'NO') then
+    Result := ucOtherNumber
+  else
+
+  if (s = 'PC') then
+    Result := ucConnectPunctuation
+  else if (s = 'PD') then
+    Result := ucDashPunctuation
+  else if (s = 'PS') then
+    Result := ucOpenPunctuation
+  else if (s = 'PE') then
+    Result := ucClosePunctuation
+  else if (s = 'PI') then
+    Result := ucInitialPunctuation
+  else     if (s = 'PF') then
+    Result := ucFinalPunctuation
+  else if (s = 'PO') then
+    Result := ucOtherPunctuation
+  else
+
+  if (s = 'SM') then
+    Result := ucMathSymbol
+  else if (s = 'SC') then
+    Result := ucCurrencySymbol
+  else if (s = 'SK') then
+    Result := ucModifierSymbol
+  else if (s = 'SO') then
+    Result := ucOtherSymbol
+  else
+
+  if (s = 'ZS') then
+    Result := ucSpaceSeparator
+  else if (s = 'ZL') then
+    Result := ucLineSeparator
+  else if (s = 'ZP') then
+    Result := ucParagraphSeparator
+  else
+
+  if (s = 'CC') then
+    Result := ucControl
+  else if (s = 'CF') then
+    Result := ucFormat
+  else if (s = 'CS') then
+    Result := ucSurrogate
+  else if (s = 'CO') then
+    Result := ucPrivateUse
+  else
+    Result := ucUnassigned;
+end;
+
+function StringToCodePoint(ACP : string) : TUnicodeCodePoint;
+var
+  s : string;
+begin
+  s := Trim(ACP);
+  Result := 0;
+  if (Length(s) > 0) and (s <> '#') then
+    Result := StrToInt('$' + s);
+end;
+
+{function IsWhiteSpace(const ACodePoint : TUnicodeCodePoint) : Boolean;
+begin
+  case ACodePoint of
+    $0009..$000D  : Result := True;// White_Space # Cc   [5] <control-0009>..<control-000D>
+    $0020          : Result := True;// White_Space # Zs       SPACE
+    $0085          : Result := True;// White_Space # Cc       <control-0085>
+    $00A0          : Result := True;// White_Space # Zs       NO-BREAK SPACE
+    $1680          : Result := True;// White_Space # Zs       OGHAM SPACE MARK
+    $180E          : Result := True;// White_Space # Zs       MONGOLIAN VOWEL SEPARATOR
+    $2000..$200A   : Result := True;// White_Space # Zs  [11] EN QUAD..HAIR SPACE
+    $2028          : Result := True;// White_Space # Zl       LINE SEPARATOR
+    $2029          : Result := True;// White_Space # Zp       PARAGRAPH SEPARATOR
+    $202F          : Result := True;// White_Space # Zs       NARROW NO-BREAK SPACE
+    $205F          : Result := True;// White_Space # Zs       MEDIUM MATHEMATICAL SPACE
+    $3000          : Result := True;// White_Space # Zs       IDEOGRAPHIC SPACE
+    else
+      Result := False;
+  end;
+end;}
+function IsWhiteSpace(
+  const ACodePoint   : TUnicodeCodePoint;
+  const AWhiteSpaces : TCodePointRecArray
+) : Boolean;
+var
+  i : Integer;
+  p : ^TCodePointRec;
+begin
+  p := @AWhiteSpaces[Low(AWhiteSpaces)];
+  for i := Low(AWhiteSpaces) to High(AWhiteSpaces) do begin
+    if (p^.LineType = 0) then begin
+      if (p^.CodePoint = ACodePoint) then
+        exit(True);
+    end else begin
+      if (ACodePoint >= p^.StartCodePoint) and (ACodePoint <= p^.EndCodePoint) then
+        exit(True);
+    end;
+    Inc(p);
+  end;
+  Result := False;
+end;
+
+function NormalizeBlockName(const AName : string) : string;
+var
+  i, c, k : Integer;
+  s : string;
+begin
+  c := Length(AName);
+  SetLength(Result,c);
+  s := LowerCase(AName);
+  k := 0;
+  for i := 1 to c do begin
+    if (s[1] in ['a'..'z','0'..'9','-']) then begin
+      k := k + 1;
+      Result[k] := s[i];
+    end;
+  end;
+  SetLength(Result,k);
+end;
+
+procedure ParseBlokcs(
+      ADataAStream   : TMemoryStream;
+  var ABlocks        : TBlocks
+);
+const READ_BUFFER_LENGTH = 1024*8;
+      LINE_LENGTH        = 1024;
+      DATA_LENGTH        = 25000;
+var
+  p : PAnsiChar;
+  actualDataLen : Integer;
+  bufferLength, bufferPos, lineLength, linePos : Integer;
+  line : ansistring;
+
+  function NextLine() : Boolean;
+  var
+    k, locOldPos : Integer;
+    locOldPointer : PAnsiChar;
+  begin
+    Result := False;
+    locOldPointer := p;
+    locOldPos := bufferPos;
+    while (bufferPos < bufferLength) and (p^ <> #10) do begin
+      Inc(p);
+      Inc(bufferPos);
+    end;
+    if (locOldPos = bufferPos) and (p^ = #10) then begin
+      lineLength := 0;
+      Inc(p);
+      Inc(bufferPos);
+      linePos := 1;
+      Result := True;
+    end else  if (locOldPos < bufferPos) then begin
+      lineLength := (bufferPos - locOldPos);
+      Move(locOldPointer^,line[1],lineLength);
+      if (p^ = #10) then begin
+        Dec(lineLength);
+        Inc(p);
+        Inc(bufferPos);
+      end;
+      linePos := 1;
+      Result := True;
+    end;
+  end;
+
+  function NextToken() : ansistring;
+  var
+    k : Integer;
+  begin
+    k := linePos;
+    if (linePos < lineLength) and (line[linePos] in [';','#','.']) then begin
+      Inc(linePos);
+      Result := Copy(line,k,(linePos-k));
+      exit;
+    end;
+    while (linePos < lineLength) and not(line[linePos] in [';','#','.']) do
+      Inc(linePos);
+    if (linePos > k) then begin
+      if (line[linePos] in [';','#','.']) then
+        Result := Copy(line,k,(linePos-k))
+      else
+        Result := Copy(line,k,(linePos-k+1));
+      Result := Trim(Result);
+    end else begin
+      Result := '';
+    end;
+  end;
+
+  procedure ParseLine();
+  var
+    locCP : Cardinal;
+    locData : TBlockItemRec;
+    s : ansistring;
+  begin
+    s := NextToken();
+    if (s = '') or (s[1] = '#') then
+      exit;
+    locData.RangeStart := StrToInt('$'+s);
+    s := NextToken();
+    if (s <> '.') then
+      raise Exception.CreateFmt('"." expected but "%s" found.',[s]);
+    s := NextToken();
+    if (s <> '.') then
+      raise Exception.CreateFmt('"." expected but "%s" found.',[s]);
+    s := NextToken();
+    locData.RangeEnd := StrToInt('$'+s);
+    s := NextToken();
+    if (s <> ';') then
+      raise Exception.CreateFmt('";" expected but "%s" found.',[s]);
+    locData.Name := Trim(NextToken());
+    locData.CanonicalName := NormalizeBlockName(locData.Name);
+    if (Length(ABlocks) <= actualDataLen) then
+      SetLength(ABlocks,Length(ABlocks)*2);
+    ABlocks[actualDataLen] := locData;
+    Inc(actualDataLen);
+  end;
+
+  procedure Prepare();
+  var
+    r : TPropRec;
+  begin
+    SetLength(ABlocks,DATA_LENGTH);
+    actualDataLen := 0;
+    bufferLength := ADataAStream.Size;
+    bufferPos := 0;
+    p := ADataAStream.Memory;
+    lineLength := 0;
+    SetLength(line,LINE_LENGTH);
+  end;
+
+begin
+  Prepare();
+  while NextLine() do
+    ParseLine();
+  SetLength(ABlocks,actualDataLen);
+end;
+
+procedure ParseProps(
+      ADataAStream   : TMemoryStream;
+  var APropList      : TPropListLineRecArray
+);
+const READ_BUFFER_LENGTH = 1024*8;
+      LINE_LENGTH        = 1024;
+      DATA_LENGTH        = 25000;
+var
+  p : PAnsiChar;
+  actualDataLen : Integer;
+  bufferLength, bufferPos, lineLength, linePos : Integer;
+  line : ansistring;
+
+  function NextLine() : Boolean;
+  var
+    k, locOldPos : Integer;
+    locOldPointer : PAnsiChar;
+  begin
+    Result := False;
+    locOldPointer := p;
+    locOldPos := bufferPos;
+    while (bufferPos < bufferLength) and (p^ <> #10) do begin
+      Inc(p);
+      Inc(bufferPos);
+    end;
+    if (locOldPos = bufferPos) and (p^ = #10) then begin
+      lineLength := 0;
+      Inc(p);
+      Inc(bufferPos);
+      linePos := 1;
+      Result := True;
+    end else  if (locOldPos < bufferPos) then begin
+      lineLength := (bufferPos - locOldPos);
+      Move(locOldPointer^,line[1],lineLength);
+      if (p^ = #10) then begin
+        Dec(lineLength);
+        Inc(p);
+        Inc(bufferPos);
+      end;
+      linePos := 1;
+      Result := True;
+    end;
+  end;
+
+  function NextToken() : ansistring;
+  var
+    k : Integer;
+  begin
+    k := linePos;
+    if (linePos < lineLength) and (line[linePos] in [';','#','.']) then begin
+      Inc(linePos);
+      Result := Copy(line,k,(linePos-k));
+      exit;
+    end;
+    while (linePos < lineLength) and not(line[linePos] in [';','#','.']) do
+      Inc(linePos);
+    if (linePos > k) then begin
+      if (line[linePos] in [';','#','.']) then
+        Result := Copy(line,k,(linePos-k))
+      else
+        Result := Copy(line,k,(linePos-k+1));
+      Result := Trim(Result);
+    end else begin
+      Result := '';
+    end;
+  end;
+
+  procedure ParseLine();
+  var
+    locCP : Cardinal;
+    locData : TPropListLineRec;
+    s : ansistring;
+  begin
+    s := NextToken();
+    if (s = '') or (s[1] = '#') then
+      exit;
+    locCP := StrToInt('$'+s);
+    s := NextToken();
+    if (s = ';') then begin
+      locData.CodePoint.LineType := 0;
+      locData.CodePoint.CodePoint := locCP;
+    end else begin
+      if (s = '') or (s <> '.') or (NextToken() <> '.') then
+        raise Exception.CreateFmt('Invalid line : "%s".',[Copy(line,1,lineLength)]);
+      locData.CodePoint.LineType := 1;
+      locData.CodePoint.StartCodePoint := locCP;
+      locData.CodePoint.EndCodePoint := StrToInt('$'+NextToken());
+      s := NextToken();
+      if (s <> ';') then
+        raise Exception.CreateFmt('"." expected but "%s" found.',[s]);
+    end;
+    locData.PropName := Trim(NextToken());
+    if (Length(APropList) <= actualDataLen) then
+      SetLength(APropList,Length(APropList)*2);
+    APropList[actualDataLen] := locData;
+    Inc(actualDataLen);
+  end;
+
+  procedure Prepare();
+  var
+    r : TPropRec;
+  begin
+    SetLength(APropList,DATA_LENGTH);
+    actualDataLen := 0;
+    bufferLength := ADataAStream.Size;
+    bufferPos := 0;
+    p := ADataAStream.Memory;
+    lineLength := 0;
+    SetLength(line,LINE_LENGTH);
+  end;
+
+begin
+  Prepare();
+  while NextLine() do
+    ParseLine();
+  SetLength(APropList,actualDataLen);
+end;
+
+function FindCodePointsByProperty(
+  const APropName : string;
+  const APropList : TPropListLineRecArray
+) : TCodePointRecArray;
+var
+  r : TCodePointRecArray;
+  i, k : Integer;
+  s : string;
+begin
+  k := 0;
+  r := nil;
+  s := LowerCase(Trim(APropName));
+  for i := Low(APropList) to High(APropList) do begin
+    if (LowerCase(APropList[i].PropName) = s) then begin
+      if (k >= Length(r)) then begin
+        if (k = 0) then
+          SetLength(r,24)
+        else
+          SetLength(r,(2*Length(r)));
+      end;
+      r[k] := APropList[i].CodePoint;
+      Inc(k);
+    end;
+  end;
+  SetLength(r,k);
+  Result := r;
+end;
+
+procedure ParseHangulSyllableTypes(
+      ADataAStream   : TMemoryStream;
+  var ACodePointList : TCodePointRecArray
+);
+const READ_BUFFER_LENGTH = 1024*8;
+      LINE_LENGTH        = 1024;
+      DATA_LENGTH        = 25000;
+var
+  p : PAnsiChar;
+  actualDataLen : Integer;
+  bufferLength, bufferPos, lineLength, linePos : Integer;
+  line : ansistring;
+
+  function NextLine() : Boolean;
+  var
+    k, locOldPos : Integer;
+    locOldPointer : PAnsiChar;
+  begin
+    Result := False;
+    locOldPointer := p;
+    locOldPos := bufferPos;
+    while (bufferPos < bufferLength) and (p^ <> #10) do begin
+      Inc(p);
+      Inc(bufferPos);
+    end;
+    if (locOldPos = bufferPos) and (p^ = #10) then begin
+      lineLength := 0;
+      Inc(p);
+      Inc(bufferPos);
+      linePos := 1;
+      Result := True;
+    end else  if (locOldPos < bufferPos) then begin
+      lineLength := (bufferPos - locOldPos);
+      Move(locOldPointer^,line[1],lineLength);
+      if (p^ = #10) then begin
+        Dec(lineLength);
+        Inc(p);
+        Inc(bufferPos);
+      end;
+      linePos := 1;
+      Result := True;
+    end;
+  end;
+
+  function NextToken() : ansistring;
+  var
+    k : Integer;
+  begin
+    k := linePos;
+    if (linePos < lineLength) and (line[linePos] = '.') then begin
+      Inc(linePos);
+      while (linePos < lineLength) and (line[linePos] = '.') do begin
+        Inc(linePos);
+      end;
+      Result := Copy(line,k,(linePos-k));
+      exit;
+    end;
+    while (linePos < lineLength) and not(line[linePos] in [';','#','.']) do
+      Inc(linePos);
+    if (linePos > k) then begin
+      if (line[linePos] in [';','#','.']) then
+        Result := Copy(line,k,(linePos-k))
+      else
+        Result := Copy(line,k,(linePos-k+1));
+      Result := Trim(Result);
+    end else begin
+      Result := '';
+    end;
+    //Inc(linePos);
+  end;
+
+  procedure ParseLine();
+  var
+    locCP : Cardinal;
+    locData : TCodePointRec;
+    s : ansistring;
+  begin
+    s := NextToken();
+    if (s = '') or (s[1] = '#') then
+      exit;
+    locData.CodePoint := StrToInt('$'+s);
+    s := NextToken();
+    if (s = '') or (s[1] in [';','#']) then begin
+      locData.LineType := 0;
+    end else begin
+      if (s <> '..') then
+        raise Exception.CreateFmt('Unknown line type : "%s"',[Copy(line,1,lineLength)]);
+      locData.StartCodePoint := locData.CodePoint;
+      locData.EndCodePoint := StrToInt('$'+NextToken());
+      locData.LineType := 1;
+    end;
+    if (Length(ACodePointList) <= actualDataLen) then
+      SetLength(ACodePointList,Length(ACodePointList)*2);
+    ACodePointList[actualDataLen] := locData;
+    Inc(actualDataLen);
+  end;
+
+  procedure Prepare();
+  var
+    r : TPropRec;
+  begin
+    SetLength(ACodePointList,DATA_LENGTH);
+    actualDataLen := 0;
+    bufferLength := ADataAStream.Size;
+    bufferPos := 0;
+    p := ADataAStream.Memory;
+    lineLength := 0;
+    SetLength(line,LINE_LENGTH);
+  end;
+
+begin
+  Prepare();
+  while NextLine() do
+    ParseLine();
+  SetLength(ACodePointList,actualDataLen);
+end;
+
+function IsHangulSyllable(
+  const ACodePoint  : TUnicodeCodePoint;
+  const AHangulList : TCodePointRecArray
+) : Boolean;
+var
+  i : Integer;
+  p : ^TCodePointRec;
+begin
+  Result := False;
+  p := @AHangulList[Low(AHangulList)];
+  for i := Low(AHangulList) to High(AHangulList) do begin
+    if ( (p^.LineType = 0) and (ACodePoint = p^.CodePoint) ) or
+       ( (p^.LineType = 1) and (ACodePoint >= p^.StartCodePoint) and (ACodePoint <= p^.EndCodePoint) )
+    then begin
+      Result := True;
+      Break;
+    end;
+    Inc(p);
+  end;
+end;
+
+function IndexOf(
+  const AProp     : TPropRec;
+  const APropList : TPropRecArray;
+  const AActualLen : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : PPropRec;
+begin
+  Result := -1;
+  if (AActualLen > 0) then begin
+    p := @APropList[0];
+    for i := 0 to AActualLen - 1 do begin
+      if (AProp.Category = p^.Category) and
+         (AProp.CCC = p^.CCC) and
+         (AProp.NumericIndex = p^.NumericIndex) and
+         (AProp.SimpleUpperCase = p^.SimpleUpperCase) and
+         (AProp.SimpleLowerCase = p^.SimpleLowerCase) and
+         (AProp.WhiteSpace = p^.WhiteSpace) and
+         //
+         (AProp.DecompositionID =  p^.DecompositionID) and
+         (*   ( (AProp.DecompositionID = -1 ) and (p^.DecompositionID = -1) ) or
+             ( (AProp.DecompositionID <> -1 ) and (p^.DecompositionID <> -1) )
+         *)
+         (AProp.HangulSyllable = p^.HangulSyllable)
+      then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+function IndexOf(
+  const AItem : TUnicodeCodePointArray;
+  const AList : TDecompositionArray
+) : Integer;overload;
+var
+  p : TUnicodeCodePointArray;
+  i : Integer;
+begin
+  Result := -1;
+  if (Length(AList) = 0) then
+    exit;
+  for i := Low(AList) to High(AList) do begin
+    p := AList[i];
+    if (Length(p) = Length(AItem)) then begin
+      if CompareMem(@p[0],@AItem[0],Length(AItem)*SizeOf(TUnicodeCodePoint)) then
+        exit(i);
+    end;
+  end;
+  Result := -1;
+end;
+
+function IndexOf(
+  const AItem : TNumericValue;
+  const AList : TNumericValueArray;
+  const AActualLen : Integer
+) : Integer;overload;
+var
+  p : ^TNumericValue;
+  i : Integer;
+begin
+  Result := -1;
+  if (AActualLen = 0) then
+    exit;
+  p := @AList[Low(AList)];
+  for i := Low(AList) to AActualLen - 1 do begin
+    if (AItem = p^) then
+      exit(i);
+    Inc(p);
+  end;
+  Result := -1;
+end;
+
+procedure Parse_UnicodeData(
+        ADataAStream   : TMemoryStream;
+  var   APropList      : TPropRecArray;
+  var   ANumericTable  : TNumericValueArray;
+  var   ADataLineList  : TDataLineRecArray;
+  var   ADecomposition : TDecompositionArray;
+  const AHangulList    : TCodePointRecArray;
+  const AWhiteSpaces   : TCodePointRecArray
+);
+const READ_BUFFER_LENGTH = 1024*8;
+      LINE_LENGTH        = 1024;
+      PROP_LENGTH        = 5000;
+      DATA_LENGTH        = 25000;
+var
+  p : PAnsiChar;
+  bufferLength, bufferPos : Integer;
+  actualPropLen, actualDataLen, actualNumLen : Integer;
+  line : ansistring;
+  lineLength, linePos : Integer;
+
+  function NextLine() : Boolean;
+  var
+    k, locOldPos : Integer;
+    locOldPointer : PAnsiChar;
+  begin
+    Result := False;
+    locOldPointer := p;
+    locOldPos := bufferPos;
+    while (bufferPos < bufferLength) and (p^ <> #10) do begin
+      Inc(p);
+      Inc(bufferPos);
+    end;
+    if (locOldPos < bufferPos) then begin
+      lineLength := (bufferPos - locOldPos);
+      Move(locOldPointer^,line[1],lineLength);
+      if (p^ = #10) then begin
+        Dec(lineLength);
+        Inc(p);
+        Inc(bufferPos);
+      end;
+      if (lineLength > 7) then begin
+        linePos := 1;
+        Result := True;
+      end;
+    end;
+  end;
+
+  function NextToken() : ansistring;
+  var
+    k : Integer;
+  begin
+    k := linePos;
+    while (linePos < lineLength) and not(line[linePos] in [';','#']) do
+      Inc(linePos);
+    if (linePos > k) then begin
+      if (line[linePos] in [';','#']) then
+        Result := Copy(line,k,(linePos-k))
+      else
+        Result := Copy(line,k,(linePos-k+1));
+      Result := Trim(Result);
+    end else begin
+      Result := '';
+    end;
+    Inc(linePos);
+  end;
+
+  function ParseCanonicalDecomposition(AStr : ansistring) : TUnicodeCodePointArray;
+  var
+    locStr, ks : ansistring;
+    k0,k, cp : Integer;
+  begin
+    SetLength(Result,0);
+    locStr := UpperCase(Trim(AStr));
+    if (locStr = '') or (locStr[1] = '<') then
+      exit;
+    k0 := 1;
+    k := 1;
+    while (k <= Length(locStr)) do begin
+      while (k <= Length(locStr)) and (locStr[k] in ['0'..'9','A'..'F']) do
+        inc(k);
+      ks := Trim(Copy(locStr,k0,k-k0));
+      SetLength(Result,Length(Result)+1);
+      Result[Length(Result)-1] := StringToCodePoint(ks);
+      Inc(k);
+      k0 := k;
+    end;
+  end;
+
+  procedure ParseLine();
+  var
+    locCP : TUnicodeCodePoint;
+    locProp : TPropRec;
+    locData : TDataLineRec;
+    s : ansistring;
+    locRangeStart, locRangeEnd : Boolean;
+    k : Integer;
+    locDecompItem : TUnicodeCodePointArray;
+    numVal : TNumericValue;
+  begin
+    FillChar(locData,SizeOf(locData),#0);
+    FillChar(locProp,SizeOf(locProp),#0);
+    locCP := StrToInt('$'+NextToken());
+    s := NextToken();
+    locRangeStart := AnsiEndsText(', First>',s);
+    if locRangeStart then
+      locRangeEnd := False
+    else
+      locRangeEnd := AnsiEndsText(', Last>',s);
+    if locRangeStart then begin
+      locData.LineType := 1;
+      locData.StartCodePoint := locCP;
+    end else if locRangeEnd then begin
+      ADataLineList[actualDataLen - 1].EndCodePoint := locCP;
+      exit;
+      //locData.EndCodePoint := locCP;
+    end else begin
+      locData.LineType := 0;
+      locData.CodePoint := locCP;
+    end;
+    locProp.Category := StrToCategory(NextToken());
+    locProp.CCC := StrToInt(NextToken());//Canonical_Combining_Class
+    NextToken();//Bidi_Class
+    s := NextToken();//Decomposition_Type
+    locDecompItem := ParseCanonicalDecomposition(s);
+    if (Length(locDecompItem) = 0) then
+      locProp.DecompositionID := -1
+    else begin
+      locProp.DecompositionID := IndexOf(locDecompItem,ADecomposition);
+      if (locProp.DecompositionID = -1) then begin
+        k := Length(ADecomposition);
+        locProp.DecompositionID := k;
+        SetLength(ADecomposition,k+1);
+        ADecomposition[k] := locDecompItem;
+      end;
+    end;
+
+    numVal := EvaluateFloat(NextToken());
+    if (numVal <> Double(0.0)) then begin
+      NextToken();
+      NextToken();
+    end else begin
+      s := NextToken();
+      if (s <> '') then
+        numVal := EvaluateFloat(s);
+      s := NextToken();
+      if (numVal = Double(0.0)) then
+        numVal := EvaluateFloat(s);
+    end;
+    k := IndexOf(numVal,ANumericTable,actualNumLen);
+    if (k = -1) then begin
+      if (actualNumLen >= Length(ANumericTable)) then
+        SetLength(ANumericTable,(actualNumLen*2));
+      ANumericTable[actualNumLen] := numVal;
+      k := actualNumLen;
+      Inc(actualNumLen);
+    end;
+    locProp.NumericIndex := k;
+
+    NextToken();//Bidi_Mirroed
+    NextToken();//Unicode_l_Name
+    NextToken();//ISO_Comment
+    locProp.SimpleUpperCase := StringToCodePoint(NextToken());
+    locProp.SimpleLowerCase := StringToCodePoint(NextToken());
+    NextToken();//Simple_Title_Case_Mapping
+    locProp.WhiteSpace := IsWhiteSpace(locCP,AWhiteSpaces);
+    locProp.HangulSyllable := IsHangulSyllable(locCP,AHangulList);
+    k := IndexOf(locProp,APropList,actualPropLen);
+    if (k = -1) then begin
+      k := actualPropLen;
+      locProp.PropID := k{ + 1};
+      APropList[k] := locProp;
+      Inc(actualPropLen);
+    end;
+    locData.PropID := k;
+    ADataLineList[actualDataLen] := locData;
+    Inc(actualDataLen);
+  end;
+
+  procedure Prepare();
+  var
+    r : TPropRec;
+  begin
+    SetLength(APropList,PROP_LENGTH);
+    actualPropLen := 0;
+    SetLength(ADataLineList,DATA_LENGTH);
+    actualDataLen := 0;
+    bufferLength := ADataAStream.Size;
+    bufferPos := 0;
+    p := ADataAStream.Memory;
+    lineLength := 0;
+    SetLength(line,LINE_LENGTH);
+    SetLength(ANumericTable,500);
+    actualNumLen := 0;
+
+    FillChar(r,SizeOf(r),#0);
+    r.PropID := 0;
+    r.Category := ucUnassigned;
+    r.DecompositionID := -1;
+    r.NumericIndex := 0;
+    APropList[0] := r;
+    Inc(actualPropLen);
+    ANumericTable[0] := 0;
+    Inc(actualNumLen);
+  end;
+
+begin
+  Prepare();
+  while NextLine() do
+    ParseLine();
+  SetLength(APropList,actualPropLen);
+  SetLength(ADataLineList,actualDataLen);
+  SetLength(ANumericTable,actualNumLen);
+end;
+
+function GetPropID(
+        ACodePoint    : TUnicodeCodePoint;
+  const ADataLineList : TDataLineRecArray
+) : Cardinal;
+var
+  i : Integer;
+  p : PDataLineRec;
+begin
+  Result := 0;
+  p := @ADataLineList[Low(ADataLineList)];
+  for i := Low(ADataLineList) to High(ADataLineList) do begin
+    if (p^.LineType = 0) then begin
+      if (p^.CodePoint = ACodePoint) then begin
+        Result := p^.PropID;
+        Break;
+      end;
+    end else begin
+      if (p^.StartCodePoint <= ACodePoint) and (p^.EndCodePoint >= ACodePoint) then begin
+        Result := p^.PropID;
+        Break;
+      end;
+    end;
+    Inc(p);
+  end;
+end;
+
+procedure MakeDecomposition(
+  const ARawData : TDecompositionArray;
+  var   ABook    : TDecompositionBook
+);
+var
+  i, c, locPos : Integer;
+  locItem : TUnicodeCodePointArray;
+begin
+  c := 0;
+  for i := Low(ARawData) to High(ARawData) do
+    c := c + Length(ARawData[i]);
+  SetLength(ABook.CodePoints,c);
+  SetLength(ABook.Index,Length(ARawData));
+  locPos := 0;
+  for i := Low(ARawData) to High(ARawData) do begin
+    locItem := ARawData[i];
+    ABook.Index[i].StartPosition := locPos;
+    ABook.Index[i].Length := Length(locItem);
+    Move(locItem[0],ABook.CodePoints[locPos],(Length(locItem) * SizeOf(TUnicodeCodePoint)));
+    locPos := locPos + Length(locItem);
+  end;
+end;
+
+type
+  PBmpSecondTableItem = ^TBmpSecondTableItem;
+function IndexOf(
+  const AItem  : PBmpSecondTableItem;
+  const ATable : TBmpSecondTable;
+  const ATableActualLength : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : PBmpSecondTableItem;
+begin
+  Result := -1;
+  if (ATableActualLength > 0) then begin
+    p := @ATable[0];
+    for i := 0 to ATableActualLength - 1 do begin
+      if CompareMem(p,AItem,SizeOf(TBmpSecondTableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+procedure MakeBmpTables(
+  var   AFirstTable   : TBmpFirstTable;
+  var   ASecondTable  : TBmpSecondTable;
+  const APropList     : TPropRecArray;
+  const ADataLineList : TDataLineRecArray
+);
+var
+  locLowByte, locHighByte : Byte;
+  locTableItem : TBmpSecondTableItem;
+  locCP : TUnicodeCodePoint;
+  i, locSecondActualLen : Integer;
+begin
+  SetLength(ASecondTable,120);
+  locSecondActualLen := 0;
+  for locHighByte := 0 to 255 do begin
+    FillChar(locTableItem,SizeOf(locTableItem),#0);
+    for locLowByte := 0 to 255 do begin
+      locCP := (locHighByte * 256) + locLowByte;
+      locTableItem[locLowByte] := GetPropID(locCP,ADataLineList)// - 1;
+    end;
+    i := IndexOf(@locTableItem,ASecondTable,locSecondActualLen);
+    if (i = -1) then begin
+      if (locSecondActualLen = Length(ASecondTable)) then
+        SetLength(ASecondTable,locSecondActualLen + 50);
+      i := locSecondActualLen;
+      ASecondTable[i] := locTableItem;
+      Inc(locSecondActualLen);
+    end;
+    AFirstTable[locHighByte] := i;
+  end;
+  SetLength(ASecondTable,locSecondActualLen);
+end;
+
+type
+  P3lvlBmp3TableItem = ^T3lvlBmp3TableItem;
+function IndexOf(
+  const AItem  : P3lvlBmp3TableItem;
+  const ATable : T3lvlBmp3Table;
+  const ATableActualLength : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : P3lvlBmp3TableItem;
+begin
+  Result := -1;
+  if (ATableActualLength > 0) then begin
+    p := @ATable[0];
+    for i := 0 to ATableActualLength - 1 do begin
+      if CompareMem(p,AItem,SizeOf(T3lvlBmp3TableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+type
+  P3lvlBmp2TableItem = ^T3lvlBmp2TableItem;
+function IndexOf(
+  const AItem  : P3lvlBmp2TableItem;
+  const ATable : T3lvlBmp2Table
+) : Integer;overload;
+var
+  i : Integer;
+  p : P3lvlBmp2TableItem;
+begin
+  Result := -1;
+  if (Length(ATable) > 0) then begin
+    p := @ATable[0];
+    for i := 0 to Length(ATable) - 1 do begin
+      if CompareMem(p,AItem,SizeOf(T3lvlBmp2TableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+procedure MakeBmpTables3Levels(
+  var   AFirstTable   : T3lvlBmp1Table;
+  var   ASecondTable  : T3lvlBmp2Table;
+  var   AThirdTable  : T3lvlBmp3Table;
+  const ADataLineList : TDataLineRecArray
+);
+var
+  locLowByte0, locLowByte1, locHighByte : Byte;
+  locTableItem2 : T3lvlBmp2TableItem;
+  locTableItem3 : T3lvlBmp3TableItem;
+  locCP : TUnicodeCodePoint;
+  i, locThirdActualLen : Integer;
+begin
+  SetLength(AThirdTable,120);
+  locThirdActualLen := 0;
+  for locHighByte := 0 to 255 do begin
+    FillChar(locTableItem2,SizeOf(locTableItem2),#0);
+    for locLowByte0 := 0 to 15 do begin
+      FillChar(locTableItem3,SizeOf(locTableItem3),#0);
+      for locLowByte1 := 0 to 15 do begin
+        locCP := (locHighByte * 256) + (locLowByte0*16) + locLowByte1;
+        locTableItem3[locLowByte1] := GetPropID(locCP,ADataLineList);
+      end;
+      i := IndexOf(@locTableItem3,AThirdTable,locThirdActualLen);
+      if (i = -1) then begin
+        if (locThirdActualLen = Length(AThirdTable)) then
+          SetLength(AThirdTable,locThirdActualLen + 50);
+        i := locThirdActualLen;
+        AThirdTable[i] := locTableItem3;
+        Inc(locThirdActualLen);
+      end;
+      locTableItem2[locLowByte0] := i;
+    end;
+    i := IndexOf(@locTableItem2,ASecondTable);
+    if (i = -1) then begin
+      i := Length(ASecondTable);
+      SetLength(ASecondTable,(i + 1));
+      ASecondTable[i] := locTableItem2;
+    end;
+    AFirstTable[locHighByte] := i;
+  end;
+  SetLength(AThirdTable,locThirdActualLen);
+end;
+
+procedure GenerateLicenceText(ADest : TStream);
+var
+  s : ansistring;
+begin
+  s := SLicenseText + sLineBreak + sLineBreak;
+  ADest.Write(s[1],Length(s));
+end;
+
+procedure GenerateBmpTables(
+        ADest : TStream;
+  var   AFirstTable   : TBmpFirstTable;
+  var   ASecondTable  : TBmpSecondTable
+);
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+begin
+  AddLine('const');
+  AddLine('  UC_TABLE_1 : array[0..255] of Byte = (');
+  locLine := '';
+  for i := Low(AFirstTable) to High(AFirstTable) - 1 do begin
+    locLine := locLine + IntToStr(AFirstTable[i]) + ',';
+    if (((i+1) mod 16) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(AFirstTable[High(AFirstTable)]);
+  locLine := '    ' + locLine;
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+
+  AddLine('  UC_TABLE_2 : array[0..(256*' + IntToStr(Length(ASecondTable)) +'-1)] of Word =(');
+  c := High(ASecondTable);
+  for i := Low(ASecondTable) to c do begin
+    locLine := '';
+    for j := Low(TBmpSecondTableItem) to High(TBmpSecondTableItem) do begin
+      locLine := locLine + IntToStr(ASecondTable[i][j]) + ',';
+      if (((j+1) mod 16) = 0) then begin
+        if (i = c) and (j = 255) then
+          Delete(locLine,Length(locLine),1);
+        locLine := '    ' + locLine;
+        AddLine(locLine);
+        locLine := '';
+      end;
+    end;
+  end;
+  AddLine('  );' + sLineBreak);
+end;
+
+//----------------------------------
+procedure Generate3lvlBmpTables(
+        ADest : TStream;
+  var   AFirstTable   : T3lvlBmp1Table;
+  var   ASecondTable  : T3lvlBmp2Table;
+  var   AThirdTable   : T3lvlBmp3Table
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+begin
+  AddLine('const');
+  AddLine('  UC_TABLE_1 : array[0..255] of Byte = (');
+  locLine := '';
+  for i := Low(AFirstTable) to High(AFirstTable) - 1 do begin
+    locLine := locLine + IntToStr(AFirstTable[i]) + ',';
+    if (((i+1) mod 16) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(AFirstTable[High(AFirstTable)]);
+  locLine := '    ' + locLine;
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+
+  AddLine('  UC_TABLE_2 : array[0..' + IntToStr(Length(ASecondTable)-1) +'] of array[0..15] of Word = (');
+  c := High(ASecondTable);
+  for i := Low(ASecondTable) to c do begin
+    locLine := '(';
+    for j := Low(T3lvlBmp2TableItem) to High(T3lvlBmp2TableItem) do
+      locLine := locLine + IntToStr(ASecondTable[i][j]) + ',';
+    Delete(locLine,Length(locLine),1);
+    locLine := '    ' + locLine + ')';
+    if (i < c) then
+      locLine := locLine + ',';
+    AddLine(locLine);
+  end;
+  AddLine('  );' + sLineBreak);
+
+  AddLine('  UC_TABLE_3 : array[0..' + IntToStr(Length(AThirdTable)-1) +'] of array[0..15] of Word = (');
+  c := High(AThirdTable);
+  for i := Low(AThirdTable) to c do begin
+    locLine := '(';
+    for j := Low(T3lvlBmp3TableItem) to High(T3lvlBmp3TableItem) do
+      locLine := locLine + IntToStr(AThirdTable[i][j]) + ',';
+    Delete(locLine,Length(locLine),1);
+    locLine := '    ' + locLine + ')';
+    if (i < c) then
+      locLine := locLine + ',';
+    AddLine(locLine);
+  end;
+  AddLine('  );' + sLineBreak);
+end;
+
+function UInt24ToStr(const AValue : UInt24; const AEndian : TEndianKind): string;inline;
+begin
+  if (AEndian = ekBig) then
+    Result := Format(
+                '(byte2 : $%s; byte1 : $%s; byte0 : $%s;)',
+                [ IntToHex(AValue.byte2,2), IntToHex(AValue.byte1,2),
+                  IntToHex(AValue.byte0,2)
+                ]
+              )
+  else
+    Result := Format(
+                '(byte0 : $%s; byte1 : $%s; byte2 : $%s;)',
+                [ IntToHex(AValue.byte0,2), IntToHex(AValue.byte1,2),
+                  IntToHex(AValue.byte2,2)
+                ]
+              );
+end;
+
+procedure GeneratePropTable(
+        ADest     : TStream;
+  const APropList : TPropRecArray;
+  const AEndian   : TEndianKind
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i : Integer;
+  locLine : string;
+  pti : PTypeInfo;
+  p : PPropRec;
+begin
+  pti := TypeInfo(TUnicodeCategory);
+  AddLine('');
+  AddLine('const');
+  AddLine('  UC_PROP_REC_COUNT = ' + IntToStr(Length(APropList)) + ';');
+  AddLine('  UC_PROP_ARRAY : array[0..(UC_PROP_REC_COUNT-1)] of TUC_Prop = (');
+  p := @APropList[0];
+  for i := Low(APropList) to High(APropList) - 1 do begin
+    locLine := //'    (Category : TUnicodeCategory.' + GetEnumName(pti,Ord(p^.Category)) + ';' +
+               '    (CategoryData : ' + IntToStr(p^.CategoryData) + ';' +
+               ' CCC : ' + IntToStr(p^.CCC) + ';' +
+               ' NumericIndex : ' + IntToStr(p^.NumericIndex) + ';' +
+               ' SimpleUpperCase : ' + UInt24ToStr(p^.SimpleUpperCase,AEndian) + ';' +
+               ' SimpleLowerCase : ' + UInt24ToStr(p^.SimpleLowerCase,AEndian) + ';' +
+               ' DecompositionID : ' + IntToStr(p^.DecompositionID) + '),';
+    AddLine(locLine);
+    Inc(p);
+  end;
+  locLine := //'    (Category : TUnicodeCategory.' + GetEnumName(pti,Ord(p^.Category)) + ';' +
+             '    (CategoryData : ' + IntToStr(p^.CategoryData) + ';' +
+             ' CCC : ' + IntToStr(p^.CCC) + ';' +
+             ' NumericIndex : ' + IntToStr(p^.NumericIndex) + ';' +
+             ' SimpleUpperCase : ' + UInt24ToStr(p^.SimpleUpperCase,AEndian) + ';' +
+             ' SimpleLowerCase : ' + UInt24ToStr(p^.SimpleLowerCase,AEndian) + ';' +
+             ' DecompositionID : ' + IntToStr(p^.DecompositionID) + ')';
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+end;
+
+procedure GenerateNumericTable(
+        ADest         : TStream;
+  const ANumList      : TNumericValueArray;
+  const ACompleteUnit : Boolean
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i : Integer;
+  locLine : string;
+  p : ^TNumericValue;
+begin
+  if ACompleteUnit then begin
+    GenerateLicenceText(ADest);
+    AddLine('unit unicodenumtable;');
+    AddLine('interface');
+    AddLine('');
+  end;
+  AddLine('');
+  AddLine('const');
+  AddLine('  UC_NUMERIC_COUNT = ' + IntToStr(Length(ANumList)) + ';');
+  AddLine('  UC_NUMERIC_ARRAY : array[0..(UC_NUMERIC_COUNT-1)] of Double = (');
+  locLine := '';
+  p := @ANumList[0];
+  for i := Low(ANumList) to High(ANumList) - 1 do begin
+    locLine := locLine + FloatToStr(p^,FS) + ' ,';
+    if (i > 0) and ((i mod 8) = 0) then begin
+      AddLine('    ' + locLine);
+      locLine := '';
+    end;
+    Inc(p);
+  end;
+  locLine := locLine + FloatToStr(p^,FS);
+  AddLine('    ' + locLine);
+  AddLine('  );' + sLineBreak);
+  if ACompleteUnit then begin
+    AddLine('');
+    AddLine('implementation');
+    AddLine('');
+    AddLine('end.');
+  end;
+end;
+
+procedure GenerateDecompositionBookTable(
+        ADest   : TStream;
+  const ABook   : TDecompositionBook;
+  const AEndian : TEndianKind
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, k : Integer;
+  p : ^TDecompositionIndexRec;
+  cp : ^TUnicodeCodePoint;
+  cp24 : UInt24;
+  locLine : string;
+begin
+  AddLine('const');
+  AddLine('  UC_DEC_BOOK_INDEX_LENGTH = ' + IntToStr(Length(ABook.Index)) + ';');
+  AddLine('  UC_DEC_BOOK_DATA_LENGTH = ' + IntToStr(Length(ABook.CodePoints)) + ';');
+  AddLine('type');
+  AddLine('  TDecompositionIndexRec = packed record');
+  AddLine('    StartPosition : Word;');
+  AddLine('    Length        : Byte;');
+  AddLine('  end;');
+  AddLine('  TDecompositionBookRec = packed record');
+  AddLine('    Index      : array[0..(UC_DEC_BOOK_INDEX_LENGTH-1)] of TDecompositionIndexRec;');
+  AddLine('    CodePoints : array[0..(UC_DEC_BOOK_DATA_LENGTH-1)] of UInt24;');
+  AddLine('  end;');
+  AddLine('const');
+  AddLine('  UC_DEC_BOOK_DATA : TDecompositionBookRec = (');
+  p := @ABook.Index[0];
+  AddLine('    Index : (// Index BEGIN');
+  k := 0;
+  locLine := '      ';
+  for i := Low(ABook.Index) to High(ABook.Index) - 1 do begin
+    locLine := locLine + '(StartPosition : ' + IntToStr(p^.StartPosition) + ';' +
+               ' Length : ' + IntToStr(p^.Length)  + '), ';
+    k := k + 1;
+    if (k >= 2) then begin
+      AddLine(locLine);
+      locLine := '      ';
+      k := 0;
+    end;
+    Inc(p);
+  end;
+  locLine := locLine + '(StartPosition : ' + IntToStr(p^.StartPosition) + ';' +
+             ' Length : ' + IntToStr(p^.Length)  + ')';
+  AddLine(locLine);
+  AddLine('    ); // Index END');
+
+  cp := @ABook.CodePoints[0];
+  AddLine('    CodePoints : (// CodePoints BEGIN');
+  k := 0;
+  locLine := '      ';
+  for i := Low(ABook.CodePoints) to High(ABook.CodePoints) - 1 do begin
+    cp24 := cp^;
+    locLine := locLine + Format('%s,',[UInt24ToStr(cp24,AEndian)]);
+    Inc(k);
+    if (k >= 16) then begin
+      AddLine(locLine);
+      k := 0;
+      locLine := '      ';
+    end;
+    Inc(cp);
+  end;
+  cp24 := cp^;
+  locLine := locLine + Format('%s',[UInt24ToStr(cp24,AEndian)]);
+  AddLine(locLine);
+  AddLine('    ); // CodePoints END');
+  AddLine('  );' + sLineBreak);
+end;
+
+procedure GenerateOutBmpTable(
+        ADest     : TStream;
+  const AList : TDataLineRecArray
+);
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j : Integer;
+  locLine : string;
+  p : PDataLineRec;
+begin
+  AddLine('');
+  //AddLine('  UC_PROP_REC_COUNT = ' + IntToStr(Length(APropList)) + ';');
+  //AddLine('  UC_PROP_ARRAY : array[0..(UC_PROP_REC_COUNT-1)] of TUC_Prop = (');
+  j := -1;
+  p := @AList[0];
+  for i := 0 to Length(AList) - 1 do begin
+    if ((p^.LineType = 0) and (p^.CodePoint >$FFFF)) or
+       (p^.StartCodePoint > $FFFF)
+    then begin
+      j := i;
+      Break;
+    end;
+    Inc(p);
+  end;
+  if (j < 0) then
+    exit;
+
+  for i := j to Length(AList) - 2 do begin
+    locLine := '    (PropID : ' + IntToStr(p^.PropID) + ';' +
+               ' CodePoint : ' + IntToStr(p^.CodePoint) + ';' +
+               ' RangeEnd : ' + IntToStr(p^.EndCodePoint) +  '),' ;
+    AddLine(locLine);
+    Inc(p);
+  end;
+  locLine := '    (PropID : ' + IntToStr(p^.PropID) + ';' +
+             ' CodePoint : ' + IntToStr(p^.CodePoint) + ';' +
+             ' RangeEnd : ' + IntToStr(p^.EndCodePoint) +  ')' ;
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+end;
+
+function Compress(const AData : TDataLineRecArray) : TDataLineRecArray;
+var
+  k, i, locResLen : Integer;
+  q, p, pr : PDataLineRec;
+  k_end : TUnicodeCodePoint;
+begin
+  locResLen := 1;
+  SetLength(Result,Length(AData));
+  FillChar(Result[0],Length(Result),#0);
+  Result[0] := AData[0];
+  q := @AData[0];
+  k := 0;
+  while (k < Length(AData)) do begin
+    if (q^.LineType = 0) then
+      k_end := q^.CodePoint
+    else
+      k_end := q^.EndCodePoint;
+    if ((k+1) = Length(AData)) then begin
+      i := k;
+    end else begin
+      p := @AData[k+1];
+      i := k +1;
+      while (i < (Length(AData) {- 1})) do begin
+        if (p^.PropID <> q^.PropID) then begin
+          i := i - 1;
+          Break;
+        end;
+        if (p^.LineType = 0) then begin
+          if (p^.CodePoint <> (k_end + 1)) then begin
+            i := i - 1;
+            Break;
+          end;
+          Inc(k_end);
+        end else begin
+          if (p^.StartCodePoint <> (k_end + 1)) then begin
+            i := i - 1;
+            Break;
+          end;
+          k_end := p^.EndCodePoint;
+        end;
+        Inc(i);
+        Inc(p);
+      end;
+    end;
+    {if (i = k) then begin
+      Result[locResLen] := q^;
+      Inc(locResLen);
+    end else begin }
+      p := @AData[i];
+      pr := @Result[locResLen];
+      pr^.PropID := q^.PropID;
+      if (q^.LineType = 0) then
+        pr^.StartCodePoint := q^.CodePoint
+      else
+        pr^.StartCodePoint := q^.StartCodePoint;
+      pr^.LineType := 1;
+      if (p^.LineType = 0) then
+        pr^.EndCodePoint := p^.CodePoint
+      else
+        pr^.EndCodePoint := p^.EndCodePoint;
+      Inc(locResLen);
+    //end;
+    k := i + 1;
+    if (k = Length(AData)) then
+      Break;
+    q := @AData[k];
+  end;
+  SetLength(Result,locResLen);
+end;
+
+procedure ParseUCAFile(
+      ADataAStream : TMemoryStream;
+  var ABook        : TUCA_DataBook
+);
+const READ_BUFFER_LENGTH = 1024*8;
+      LINE_LENGTH        = 1024;
+      DATA_LENGTH        = 25000;
+var
+  p : PAnsiChar;
+  actualDataLen : Integer;
+  bufferLength, bufferPos, lineLength, linePos : Integer;
+  line : ansistring;
+
+  function NextLine() : Boolean;
+  var
+    k, locOldPos : Integer;
+    locOldPointer : PAnsiChar;
+  begin
+    Result := False;
+    locOldPointer := p;
+    locOldPos := bufferPos;
+    while (bufferPos < bufferLength) and (p^ <> #10) do begin
+      Inc(p);
+      Inc(bufferPos);
+    end;
+    if (locOldPos = bufferPos) and (p^ = #10) then begin
+      lineLength := 0;
+      Inc(p);
+      Inc(bufferPos);
+      linePos := 1;
+      Result := True;
+    end else  if (locOldPos < bufferPos) then begin
+      lineLength := (bufferPos - locOldPos) + 1;
+      Move(locOldPointer^,line[1],lineLength);
+      if (p^ = #10) then begin
+        Dec(lineLength);
+        Inc(p);
+        Inc(bufferPos);
+      end;
+      linePos := 1;
+      Result := True;
+    end;
+  end;
+
+  procedure SkipSpace();
+  begin
+    while (linePos < lineLength) and (line[linePos] in [' ',#9]) do
+      Inc(linePos);
+  end;
+
+  function NextToken() : ansistring;
+  const C_SEPARATORS  = [';','#','.','[',']','*','@'];
+  var
+    k : Integer;
+  begin
+    SkipSpace();
+    k := linePos;
+    if (linePos <= lineLength) and (line[linePos] in C_SEPARATORS) then begin
+      Result := line[linePos];
+      Inc(linePos);
+      exit;
+    end;
+    while (linePos <= lineLength) and not(line[linePos] in (C_SEPARATORS+[' '])) do
+      Inc(linePos);
+    if (linePos > k) then begin
+      if (line[Min(linePos,lineLength)] in C_SEPARATORS) then
+        Result := Copy(line,k,(linePos-k))
+      else
+        Result := Copy(line,k,(linePos-k+1));
+      Result := Trim(Result);
+    end else begin
+      Result := '';
+    end;
+  end;
+
+  procedure CheckToken(const AToken : string);
+  var
+    a, b : string;
+  begin
+    a := LowerCase(Trim(AToken));
+    b := LowerCase(Trim(NextToken()));
+    if (a <> b) then
+      raise Exception.CreateFmt('Expected token "%s" but found "%s".',[a,b]);
+  end;
+
+  function ReadWeightBlock(var ADest : TUCA_WeightRec) : Boolean;
+  var
+    s :AnsiString;
+    k : Integer;
+  begin
+    Result := False;
+    s := NextToken();
+    if (s <> '[') then
+      exit;
+    s := NextToken();
+    if (s = '.') then
+      ADest.Variable := False
+    else begin
+      if (s <> '*') then
+        raise Exception.CreateFmt('Expected "%s" but found "%s".',['*',s]);
+      ADest.Variable := True;
+    end;
+    ADest.Weights[0] := StrToInt('$'+NextToken());
+    for k := 1 to 3 do begin
+      CheckToken('.');
+      ADest.Weights[k] := StrToInt('$'+NextToken());
+    end;
+    CheckToken(']');
+    Result := True;
+  end;
+
+  procedure ParseHeaderVar();
+  var
+    s,ss : string;
+    k : Integer;
+  begin
+    s := NextToken();
+    if (s = 'version') then begin
+      ss := '';
+      while True do begin
+        s := NextToken();
+        if (s = '') then
+          Break;
+        ss := ss + s;
+      end;
+      ABook.Version := ss;
+    end else if (s = 'variable') then begin
+      if (s = 'blanked') then
+        ABook.VariableWeight := ucaBlanked
+      else if (s = 'non-ignorable') then
+        ABook.VariableWeight := ucaNonIgnorable
+      else if (s = 'shifted') then
+        ABook.VariableWeight := ucaShifted
+      else if (s = 'shift-trimmed') then
+        ABook.VariableWeight := ucaShiftedTrimmed
+      else if (s = 'ignoresp') then
+        ABook.VariableWeight := ucaIgnoreSP
+      else
+        raise Exception.CreateFmt('Unknown "@variable" type : "%s".',[s]);
+    end else if (s = 'backwards') or (s = 'forwards') then begin
+      ss := s;
+      s := NextToken();
+      k := StrToInt(s);
+      if (k < 1) or (k > 4) then
+        raise Exception.CreateFmt('Invalid "%s" position : %d.',[ss,s]);
+      ABook.Backwards[k] := (s = 'backwards');
+    end;
+  end;
+
+  procedure ParseLine();
+  var
+    locCP : Cardinal;
+    locData : ^TUCA_LineRec;
+    s : ansistring;
+    kc : Integer;
+  begin
+    if (Length(ABook.Lines) <= actualDataLen) then
+      SetLength(ABook.Lines,Length(ABook.Lines)*2);
+    locData := @ABook.Lines[actualDataLen];
+    s := NextToken();
+    if (s = '') or (s[1] = '#') then
+      exit;
+    if (s[1] = '@') then begin
+      ParseHeaderVar();
+      exit;
+    end;
+    SetLength(locData^.CodePoints,10);
+    locData^.CodePoints[0] := StrToInt('$'+s);
+    kc := 1;
+    while True do begin
+      s := Trim(NextToken());
+      if (s = '') then
+        exit;
+      if (s = ';') then
+        Break;
+      locData^.CodePoints[kc] := StrToInt('$'+s);
+      Inc(kc);
+    end;
+    if (kc = 0) then
+      exit;
+    SetLength(locData^.CodePoints,kc);
+    SetLength(locData^.Weights,24);
+    kc := 0;
+    while ReadWeightBlock(locData^.Weights[kc]) do begin
+      Inc(kc);
+    end;
+    SetLength(locData^.Weights,kc);
+    Inc(actualDataLen);
+  end;
+
+  procedure Prepare();
+  var
+    r : TPropRec;
+    k : Integer;
+  begin
+    ABook.VariableWeight := ucaShifted;
+    for k := Low(ABook.Backwards) to High(ABook.Backwards) do
+      ABook.Backwards[k] := False;
+    SetLength(ABook.Lines,DATA_LENGTH);
+    actualDataLen := 0;
+    bufferLength := ADataAStream.Size;
+    bufferPos := 0;
+    p := ADataAStream.Memory;
+    lineLength := 0;
+    SetLength(line,LINE_LENGTH);
+  end;
+
+begin
+  Prepare();
+  while NextLine() do
+    ParseLine();
+  SetLength(ABook.Lines,actualDataLen);
+end;
+
+procedure Dump(X : array of TUnicodeCodePoint; const ATitle : string = '');
+var
+  i : Integer;
+begin
+  Write(ATitle, ' ');
+  for i := 0 to Length(X) - 1 do
+    Write(X[i],' ');
+  WriteLn();
+end;
+
+function IsGreaterThan(A, B : PUCA_LineRec) : Integer;
+var
+  i, hb : Integer;
+begin
+  if (A=B) then
+    exit(0);
+  Result := 1;
+  hb := Length(B^.CodePoints) - 1;
+  for i := 0 to Length(A^.CodePoints) - 1 do begin
+    if (i > hb) then
+      exit;
+    if (A^.CodePoints[i] < B^.CodePoints[i]) then
+      exit(-1);
+    if (A^.CodePoints[i] > B^.CodePoints[i]) then
+      exit(1);
+  end;
+  if (Length(A^.CodePoints) = Length(B^.CodePoints)) then
+    exit(0);
+  exit(-1);
+end;
+
+Procedure QuickSort(var AList: TUCA_DataBookIndex; L, R : Longint;
+                     ABook : PUCA_DataBook);
+var
+  I, J : Longint;
+  P, Q : Integer;
+begin
+ repeat
+   I := L;
+   J := R;
+   P := AList[ (L + R) div 2 ];
+   repeat
+     while IsGreaterThan(@ABook^.Lines[P], @ABook^.Lines[AList[i]]) > 0 do
+       I := I + 1;
+     while IsGreaterThan(@ABook^.Lines[P], @ABook^.Lines[AList[J]]) < 0 do
+       J := J - 1;
+     If I <= J then
+     begin
+       Q := AList[I];
+       AList[I] := AList[J];
+       AList[J] := Q;
+       I := I + 1;
+       J := J - 1;
+     end;
+   until I > J;
+   // sort the smaller range recursively
+   // sort the bigger range via the loop
+   // Reasons: memory usage is O(log(n)) instead of O(n) and loop is faster than recursion
+   if J - L < R - I then
+   begin
+     if L < J then
+       QuickSort(AList, L, J, ABook);
+     L := I;
+   end
+   else
+   begin
+     if I < R then
+       QuickSort(AList, I, R, ABook);
+     R := J;
+   end;
+ until L >= R;
+end;
+
+function CreateIndex(ABook : PUCA_DataBook) : TUCA_DataBookIndex;
+var
+  r : TUCA_DataBookIndex;
+  i, c : Integer;
+begin
+  c := Length(ABook^.Lines);
+  SetLength(r,c);
+  for i := 0 to c - 1 do
+    r[i] := i;
+  QuickSort(r,0,c-1,ABook);
+  Result := r;
+end;
+
+function ConstructContextTree(
+  const AContext : PUCA_LineContextRec;
+  var   ADestBuffer;
+  const ADestBufferLength : Integer
+) : PUCA_PropItemContextTreeRec;forward;
+function ConstructItem(
+        AItem         : PUCA_PropItemRec;
+        ACodePoint    : Cardinal;
+        AValid        : Byte;
+        AChildCount   : Byte;
+  const AWeights      : array of TUCA_WeightRec;
+  const AStoreCP      : Boolean;
+  const AContext      : PUCA_LineContextRec;
+  const ADeleted      : Boolean
+) : Integer;
+var
+  i, c : Integer;
+  p : PUCA_PropItemRec;
+  pw : PUCA_PropWeights;
+  w : TUCA_WeightRec;
+  pb : PByte;
+  hasContext : Boolean;
+  contextTree : PUCA_PropItemContextTreeRec;
+begin
+  p := AItem;
+  {if AStoreCP then begin
+    PUInt24(p)^ := ACodePoint;
+    p := PUCA_PropItemRec(PtrUInt(p) + SizeOf(UInt24));
+  end;  }
+  p^.Flags := 0;
+  p^.Valid := 0;
+  SetBit(p^.Valid,BIT_POS_VALIDE,(AValid <> 0));
+  p^.ChildCount := AChildCount;
+  c := Length(AWeights);
+  p^.WeightLength := c;
+  if (c = 0) then begin
+    Result := SizeOf(TUCA_PropItemRec);
+    if ADeleted then
+      SetBit(AItem^.Flags,AItem^.FLAG_DELETION,True);
+  end else begin
+    Result := SizeOf(TUCA_PropItemRec) + (c*SizeOf(TUCA_PropWeights));//PtrUInt(pw) - PtrUInt(AItem);
+    //pw := PUCA_PropWeights(PtrUInt(p) + SizeOf(TUCA_PropItemRec));
+    pb := PByte(PtrUInt(p) + SizeOf(TUCA_PropItemRec));
+    PWord(pb)^ := AWeights[0].Weights[0];
+    pb := pb + 2;
+    if (AWeights[0].Weights[1] > High(Byte)) then begin
+      SetBit(p^.Valid,(BIT_POS_COMPRESS_WEIGHT_1),True);
+      PWord(pb)^ := AWeights[0].Weights[1];
+      pb := pb + 2;
+    end else begin
+      pb^ := AWeights[0].Weights[1];
+      pb := pb + 1;
+      Result := Result - 1;
+    end;
+    if (AWeights[0].Weights[2] > High(Byte)) then begin
+      SetBit(p^.Valid,(BIT_POS_COMPRESS_WEIGHT_2),True);
+      PWord(pb)^ := AWeights[0].Weights[2];
+      pb := pb + 2;
+    end else begin
+      pb^ := AWeights[0].Weights[2];
+      pb := pb + 1;
+      Result := Result - 1;
+    end;
+    pw := PUCA_PropWeights(pb);
+    for i := 1 to c - 1 do begin
+      pw^.Weights[0] := AWeights[i].Weights[0];
+      pw^.Weights[1] := AWeights[i].Weights[1];
+      pw^.Weights[2] := AWeights[i].Weights[2];
+      //pw^.Variable := BoolToByte(AWeights[i].Variable);
+      Inc(pw);
+    end;
+  end;
+  hasContext := (AContext <> nil) and (Length(AContext^.Data) > 0);
+  if AStoreCP or hasContext then begin
+    PUInt24(PtrUInt(AItem)+Result)^ := ACodePoint;
+    Result := Result + SizeOf(UInt24);
+    SetBit(AItem^.Flags,AItem^.FLAG_CODEPOINT,True);
+  end;
+  if hasContext then begin
+    contextTree := ConstructContextTree(AContext,Pointer(PtrUInt(AItem)+Result)^,-1);
+    Result := Result + Cardinal(contextTree^.Size);
+    SetBit(AItem^.Flags,AItem^.FLAG_CONTEXTUAL,True);
+  end;
+  p^.Size := Result;
+end;
+
+function CalcCharChildCount(
+  const AChar           : Cardinal;
+  const ASearchStartPos : Integer;
+  const ALinePos        : Integer;
+  const ABookLines      : PUCA_LineRec;
+  const AMaxLength      : Integer;
+  const ABookIndex      : TUCA_DataBookIndex;
+  out   ALineCount      : Integer
+) : Byte;
+var
+  locLinePos : Integer;
+  p : PUCA_LineRec;
+
+  procedure IncP();
+  begin
+    Inc(locLinePos);
+    p := @ABookLines[ABookIndex[locLinePos]];
+  end;
+
+  procedure DoDump();
+  var
+    px : PUCA_LineRec;
+    k, ki : Integer;
+  begin
+    WriteLn;
+    WriteLn('Dump');
+    for k := ALinePos to ALinePos + 15 do begin
+      px := @ABookLines[ABookIndex[k]];
+      for ki := 0 to Length(px^.CodePoints) -1 do
+        Write(px^.CodePoints[ki],' ');
+      WriteLn;
+    end;
+  end;
+
+var
+  i, locTargetLen, locTargetBufferSize, r : Integer;
+  locTarget : array[0..127] of Cardinal;
+  locLastChar : Cardinal;
+begin
+  locLinePos := ALinePos;
+  p := @ABookLines[ABookIndex[locLinePos]];
+  locTargetLen := ASearchStartPos;
+  locTargetBufferSize := (locTargetLen*SizeOf(Cardinal));
+  Move(p^.CodePoints[0],locTarget[0],locTargetBufferSize);
+  if (Length(p^.CodePoints) = ASearchStartPos) then begin
+    r := 0;
+    locLastChar := High(Cardinal);
+  end else begin
+    r := 1;
+    locLastChar := p^.CodePoints[ASearchStartPos];
+  end;
+  IncP();
+  i := 1;
+  while (i < AMaxLength) do begin
+    if (Length(p^.CodePoints) < locTargetLen) then
+      Break;
+    if not CompareMem(@locTarget[0],@p^.CodePoints[0],locTargetBufferSize) then
+      Break;
+    if (p^.CodePoints[ASearchStartPos] <> locLastChar) then begin
+      Inc(r);
+      locLastChar := p^.CodePoints[ASearchStartPos];
+    end;
+    IncP();
+    Inc(i);
+  end;
+  ALineCount := i;
+  Result := r;
+end;
+
+function BuildTrie(
+  const ALinePos        : Integer;
+  const ABookLines      : PUCA_LineRec;
+  const AMaxLength      : Integer;
+  const ABookIndex      : TUCA_DataBookIndex
+) : PTrieNode;
+var
+  p : PUCA_LineRec;
+  root, n : PTrieNode;
+  ki, k, i : Integer;
+  key : array of TKeyType;
+begin
+  k := ABookIndex[ALinePos];
+  p := @ABookLines[k];
+  if (Length(p^.CodePoints) = 1) then
+    root := CreateNode(p^.CodePoints[0],k)
+  else
+    root := CreateNode(p^.CodePoints[0]);
+
+  for i := ALinePos to ALinePos + AMaxLength - 1 do begin
+    k := ABookIndex[i];
+    p := @ABookLines[k];
+    if (Length(p^.CodePoints) = 1) then begin
+      InsertWord(root,p^.CodePoints[0],k);
+    end else begin
+      SetLength(key,Length(p^.CodePoints));
+      for ki := 0 to Length(p^.CodePoints) - 1 do
+        key[ki] := p^.CodePoints[ki];
+      InsertWord(root,key,k);
+    end;
+  end;
+  Result := root;
+end;
+
+function BoolToByte(AValue : Boolean): Byte;inline;
+begin
+  if AValue then
+    Result := 1
+  else
+    Result := 0;
+end;
+
+function InternalConstructFromTrie(
+  const ATrie  : PTrieNode;
+  const AItem  : PUCA_PropItemRec;
+  const ALines : PUCA_LineRec;
+  const AStoreCp : Boolean
+) : Integer;
+var
+  i, size : Integer;
+  p : PUCA_PropItemRec;
+  n : PTrieNode;
+begin
+  if (ATrie = nil) then
+    exit(0);
+  p := AItem;
+  n := ATrie;
+  if n^.DataNode then
+    size := ConstructItem(p,n^.Key,1,n^.ChildCount,ALines[n^.Data].Weights,AStoreCp,@(ALines[n^.Data].Context),ALines[n^.Data].Deleted)
+  else
+    size := ConstructItem(p,n^.Key,0,n^.ChildCount,[],AStoreCp,nil,False);
+  Result := size;
+  if (n^.ChildCount > 0) then begin
+    for i := 0 to n^.ChildCount - 1 do begin
+      p := PUCA_PropItemRec(PtrUInt(p) + size);
+      size := InternalConstructFromTrie(n^.Children[i],p,ALines,True);
+      Result := Result + size;
+    end;
+  end;
+  AItem^.Size := Result;
+end;
+
+function ConstructFromTrie(
+  const ATrie  : PTrieNode;
+  const AItem  : PUCA_PropItemRec;
+  const ALines : PUCA_LineRec
+) : Integer;
+begin
+  Result := InternalConstructFromTrie(ATrie,AItem,ALines,False);
+end;
+
+procedure MakeUCA_Props(
+        ABook         : PUCA_DataBook;
+  out   AProps        : PUCA_PropBook
+);
+var
+  propIndexCount : Integer;
+
+  procedure CapturePropIndex(AItem : PUCA_PropItemRec; ACodePoint : Cardinal);
+  begin
+    AProps^.Index[propIndexCount].CodePoint := ACodePoint;
+    AProps^.Index[propIndexCount].Position := PtrUInt(AItem) - PtrUInt(AProps^.Items);
+    propIndexCount := propIndexCount + 1;
+  end;
+
+var
+  locIndex : TUCA_DataBookIndex;
+  i, c, k, kc : Integer;
+  p, p1, p2 : PUCA_PropItemRec;
+  lines, pl1, pl2 : PUCA_LineRec;
+  uc : Cardinal;
+  childCount, lineCount, size : Integer;
+  trieRoot : PTrieNode;
+  MaxChildCount, MaxSize : Integer;
+begin
+  locIndex := CreateIndex(ABook);
+  i := Length(ABook^.Lines);
+  i := 30 * i * (SizeOf(TUCA_PropItemRec) + SizeOf(TUCA_PropWeights));
+  GetMem(AProps,SizeOf(TUCA_DataBook));
+  AProps^.ItemSize := i;
+  GetMem(AProps^.Items,i);
+  propIndexCount := 0;
+  SetLength(AProps^.Index,Length(ABook^.Lines));
+  p := AProps^.Items;
+  lines := @ABook^.Lines[0];
+  c := Length(locIndex);
+  i := 0;
+  MaxChildCount := 0; MaxSize := 0;
+  while (i < (c-1)) do begin
+    pl1 := @lines[locIndex[i]];
+    if not pl1^.Stored then begin
+      i := i + 1;
+      Continue;
+    end;
+    pl2 := @lines[locIndex[i+1]];
+    if (pl1^.CodePoints[0] <> pl2^.CodePoints[0]) then begin
+      if (Length(pl1^.CodePoints) = 1) then begin
+        size := ConstructItem(p,pl1^.CodePoints[0],1,0,pl1^.Weights,False,@pl1^.Context,pl1^.Deleted);
+        CapturePropIndex(p,pl1^.CodePoints[0]);
+        p := PUCA_PropItemRec(PtrUInt(p) + size);
+        if (size > MaxSize) then
+          MaxSize := size;
+      end else begin
+        kc := Length(pl1^.CodePoints);
+        for k := 0 to kc - 2 do begin
+          size := ConstructItem(p,pl1^.CodePoints[k],0,1,[],(k>0),@pl1^.Context,pl1^.Deleted);
+          if (k = 0) then
+            CapturePropIndex(p,pl1^.CodePoints[k]);
+          p := PUCA_PropItemRec(PtrUInt(p) + size);
+        end;
+        size := ConstructItem(p,pl1^.CodePoints[kc-1],1,0,pl1^.Weights,True,@pl1^.Context,pl1^.Deleted);
+        p := PUCA_PropItemRec(PtrUInt(p) + size);
+        p2 := p;
+        for k := kc - 2 downto 0 do begin
+          p1 := PUCA_PropItemRec(PtrUInt(p2) - p2^.Size);
+          p1^.Size := p1^.Size + p2^.Size;
+          p2 := p1;
+        end;
+        if (p1^.Size > MaxSize) then
+          MaxSize := p1^.Size;
+      end;
+      lineCount := 1;
+    end else begin
+      childCount := CalcCharChildCount(pl1^.CodePoints[0],1,i,lines,c,locIndex,lineCount);
+      if (childCount < 1) then
+        raise Exception.CreateFmt('Expected "child count > 1" but found %d.',[childCount]);
+      if (lineCount < 2) then
+        raise Exception.CreateFmt('Expected "line count > 2" but found %d.',[lineCount]);
+      if (childCount > MaxChildCount) then
+        MaxChildCount := childCount;
+      trieRoot := BuildTrie(i,lines,lineCount,locIndex);
+      size := ConstructFromTrie(trieRoot,p,lines);
+      CapturePropIndex(p,pl1^.CodePoints[0]);
+      FreeNode(trieRoot);
+      p := PUCA_PropItemRec(PtrUInt(p) + size);
+      if (size > MaxSize) then
+        MaxSize := size;
+    end;
+    i := i + lineCount;
+  end;
+  if (i = (c-1)) then begin
+    pl1 := @lines[locIndex[i]];
+    if (Length(pl1^.CodePoints) = 1) then begin
+      size := ConstructItem(p,pl1^.CodePoints[0],1,0,pl1^.Weights,False,@pl1^.Context,pl1^.Deleted);
+      CapturePropIndex(p,pl1^.CodePoints[0]);
+      p := PUCA_PropItemRec(PtrUInt(p) + size);
+      if (size > MaxSize) then
+        MaxSize := size;
+    end else begin
+      kc := Length(pl1^.CodePoints);
+      for k := 0 to kc - 2 do begin
+        size := ConstructItem(p,pl1^.CodePoints[k],0,1,[],(k>0),@pl1^.Context,pl1^.Deleted);
+        if (k = 0) then
+          CapturePropIndex(p,pl1^.CodePoints[0]);
+        p := PUCA_PropItemRec(PtrUInt(p) + size);
+      end;
+      size := ConstructItem(p,pl1^.CodePoints[kc-1],1,0,pl1^.Weights,True,@pl1^.Context,pl1^.Deleted);
+      p := PUCA_PropItemRec(PtrUInt(p) + size);
+      p2 := p;
+      for k := kc - 2 downto 0 do begin
+        p1 := PUCA_PropItemRec(PtrUInt(p2) - p2^.Size);
+        p1^.Size := p1^.Size + p2^.Size;
+        p2 := p1;
+      end;
+      if (size > MaxSize) then
+        MaxSize := size;
+    end;
+  end;
+  c := Int64(PtrUInt(p)) - Int64(PtrUInt(AProps^.Items));
+  ReAllocMem(AProps^.Items,c);
+  AProps^.ItemSize := c;
+  SetLength(AProps^.Index,propIndexCount);
+
+  k := 0;
+  c := High(Word);
+  for i := 0 to Length(ABook^.Lines) - 1 do begin
+    if (Length(ABook^.Lines[i].Weights) > 0) then begin
+      if (ABook^.Lines[i].Weights[0].Variable) then begin
+        if (ABook^.Lines[i].Weights[0].Weights[0] > k) then
+          k := ABook^.Lines[i].Weights[0].Weights[0];
+        if (ABook^.Lines[i].Weights[0].Weights[0] < c) then
+          c := ABook^.Lines[i].Weights[0].Weights[0];
+      end;
+    end;
+  end;
+  AProps^.VariableHighLimit := k;
+  AProps^.VariableLowLimit := c;
+end;
+
+procedure FreeUcaBook(var ABook : PUCA_PropBook);
+var
+  p : PUCA_PropBook;
+begin
+  if (ABook = nil) then
+    exit;
+  p := ABook;
+  ABook := nil;
+  p^.Index := nil;
+  FreeMem(p^.Items,p^.ItemSize);
+  FreeMem(p,SizeOf(p^));
+end;
+
+function IndexOf(const ACodePoint : Cardinal; APropBook : PUCA_PropBook): Integer;overload;
+var
+  i : Integer;
+begin
+  for i := 0 to Length(APropBook^.Index) - 1 do begin
+    if (ACodePoint = APropBook^.Index[i].CodePoint) then
+      exit(i);
+  end;
+  Result := -1;
+end;
+
+type
+  PucaBmpSecondTableItem = ^TucaBmpSecondTableItem;
+function IndexOf(
+  const AItem  : PucaBmpSecondTableItem;
+  const ATable : TucaBmpSecondTable;
+  const ATableActualLength : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : PucaBmpSecondTableItem;
+begin
+  Result := -1;
+  if (ATableActualLength > 0) then begin
+    p := @ATable[0];
+    for i := 0 to ATableActualLength - 1 do begin
+      if CompareMem(p,AItem,SizeOf(TucaBmpSecondTableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+procedure MakeUCA_BmpTables(
+  var   AFirstTable   : TucaBmpFirstTable;
+  var   ASecondTable  : TucaBmpSecondTable;
+  const APropBook     : PUCA_PropBook
+);
+var
+  locLowByte, locHighByte : Byte;
+  locTableItem : TucaBmpSecondTableItem;
+  locCP : TUnicodeCodePoint;
+  i, locSecondActualLen : Integer;
+  k : Integer;
+begin
+  SetLength(ASecondTable,120);
+  locSecondActualLen := 0;
+  for locHighByte := 0 to 255 do begin
+    FillChar(locTableItem,SizeOf(locTableItem),#0);
+    for locLowByte := 0 to 255 do begin
+      locCP := (locHighByte * 256) + locLowByte;
+      k := IndexOf(locCP,APropBook);
+      if (k = -1) then
+        k := 0
+      else
+        k := APropBook^.Index[k].Position + 1;
+      locTableItem[locLowByte] := k;
+    end;
+    i := IndexOf(@locTableItem,ASecondTable,locSecondActualLen);
+    if (i = -1) then begin
+      if (locSecondActualLen = Length(ASecondTable)) then
+        SetLength(ASecondTable,locSecondActualLen + 50);
+      i := locSecondActualLen;
+      ASecondTable[i] := locTableItem;
+      Inc(locSecondActualLen);
+    end;
+    AFirstTable[locHighByte] := i;
+  end;
+  SetLength(ASecondTable,locSecondActualLen);
+end;
+
+function ToUCS4(const AHighS, ALowS : Word) : TUnicodeCodePoint; inline;
+begin
+  //copied from utf16toutf32
+  Result := (UCS4Char(AHighS)-$d800) shl 10 + (UCS4Char(ALowS)-$dc00) + $10000;
+end;
+
+procedure FromUCS4(const AValue : TUnicodeCodePoint; var AHighS, ALowS : Word);
+begin
+  AHighS := Word((AValue - $10000) shr 10 + $d800);
+  ALowS := Word((AValue - $10000) and $3ff + $dc00);
+end;
+
+type
+  PucaOBmpSecondTableItem = ^TucaOBmpSecondTableItem;
+function IndexOf(
+  const AItem  : PucaOBmpSecondTableItem;
+  const ATable : TucaOBmpSecondTable;
+  const ATableActualLength : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : PucaOBmpSecondTableItem;
+begin
+  Result := -1;
+  if (ATableActualLength > 0) then begin
+    p := @ATable[0];
+    for i := 0 to ATableActualLength - 1 do begin
+      if CompareMem(p,AItem,SizeOf(TucaOBmpSecondTableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+procedure MakeUCA_OBmpTables(
+  var   AFirstTable   : TucaOBmpFirstTable;
+  var   ASecondTable  : TucaOBmpSecondTable;
+  const APropBook     : PUCA_PropBook
+);
+var
+  locLowByte, locHighByte : Word;
+  locTableItem : TucaOBmpSecondTableItem;
+  locCP : TUnicodeCodePoint;
+  i, locSecondActualLen : Integer;
+  k : Integer;
+begin
+  if (Length(ASecondTable) = 0) then
+    SetLength(ASecondTable,2000);
+  locSecondActualLen := 0;
+  for locHighByte := 0 to HIGH_SURROGATE_COUNT - 1 do begin
+    FillChar(locTableItem,SizeOf(locTableItem),#0);
+    for locLowByte := 0 to LOW_SURROGATE_COUNT - 1 do begin
+      locCP := ToUCS4(HIGH_SURROGATE_BEGIN + locHighByte,LOW_SURROGATE_BEGIN + locLowByte);
+      k := IndexOf(locCP,APropBook);
+      if (k = -1) then
+        k := 0
+      else
+        k := APropBook^.Index[k].Position + 1;
+      locTableItem[locLowByte] := k;
+    end;
+    i := IndexOf(@locTableItem,ASecondTable,locSecondActualLen);
+    if (i = -1) then begin
+      if (locSecondActualLen = Length(ASecondTable)) then
+        SetLength(ASecondTable,locSecondActualLen + 50);
+      i := locSecondActualLen;
+      ASecondTable[i] := locTableItem;
+      Inc(locSecondActualLen);
+    end;
+    AFirstTable[locHighByte] := i;
+  end;
+  SetLength(ASecondTable,locSecondActualLen);
+end;
+
+function GetPropPosition(
+  const AHighS,
+        ALowS         : Word;
+  const AFirstTable   : PucaOBmpFirstTable;
+  const ASecondTable  : PucaOBmpSecondTable
+): Integer;inline;overload;
+begin
+  Result := ASecondTable^[AFirstTable^[AHighS-HIGH_SURROGATE_BEGIN]][ALowS-LOW_SURROGATE_BEGIN] - 1;
+end;
+
+procedure GenerateUCA_Head(
+  ADest  : TStream;
+  ABook  : PUCA_DataBook;
+  AProps : PUCA_PropBook
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+begin
+  AddLine('const');
+  AddLine('  VERSION_STRING = ' + QuotedStr(ABook^.Version) + ';');
+  AddLine('  VARIABLE_LOW_LIMIT = ' + IntToStr(AProps^.VariableLowLimit) + ';');
+  AddLine('  VARIABLE_HIGH_LIMIT = ' + IntToStr(AProps^.VariableHighLimit) + ';');
+  AddLine('  VARIABLE_WEIGHT = ' + IntToStr(Ord(ABook^.VariableWeight)) + ';');
+  AddLine('  BACKWARDS_0 = ' + BoolToStr(ABook^.Backwards[0],'True','False') + ';');
+  AddLine('  BACKWARDS_1 = ' + BoolToStr(ABook^.Backwards[1],'True','False') + ';');
+  AddLine('  BACKWARDS_2 = ' + BoolToStr(ABook^.Backwards[2],'True','False') + ';');
+  AddLine('  BACKWARDS_3 = ' + BoolToStr(ABook^.Backwards[3],'True','False') + ';');
+  AddLine('  PROP_COUNT  = ' + IntToStr(Ord(AProps^.ItemSize)) + ';');
+
+  AddLine('');
+end;
+
+procedure GenerateUCA_BmpTables(
+        AStream,
+        ABinStream    : TStream;
+  var   AFirstTable   : TucaBmpFirstTable;
+  var   ASecondTable  : TucaBmpSecondTable;
+  const AEndian       : TEndianKind
+);
+
+  procedure AddLine(AOut : TStream; const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    AOut.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+  value : UInt24;
+begin
+  AddLine(AStream,'const');
+  AddLine(AStream,'  UCA_TABLE_1 : array[0..255] of Byte = (');
+  locLine := '';
+  for i := Low(AFirstTable) to High(AFirstTable) - 1 do begin
+    locLine := locLine + IntToStr(AFirstTable[i]) + ',';
+    if (((i+1) mod 16) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(AStream,locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(AFirstTable[High(AFirstTable)]);
+  locLine := '    ' + locLine;
+  AddLine(AStream,locLine);
+  AddLine(AStream,'  );' + sLineBreak);
+
+  AddLine(ABinStream,'const');
+  AddLine(ABinStream,'  UCA_TABLE_2 : array[0..(256*' + IntToStr(Length(ASecondTable)) +'-1)] of UInt24 =(');
+  c := High(ASecondTable);
+  for i := Low(ASecondTable) to c do begin
+    locLine := '';
+    for j := Low(TucaBmpSecondTableItem) to High(TucaBmpSecondTableItem) do begin
+      value := ASecondTable[i][j];
+      locLine := locLine + UInt24ToStr(value,AEndian) + ',';
+      if (((j+1) mod 2) = 0) then begin
+        if (i = c) and (j = 255) then
+          Delete(locLine,Length(locLine),1);
+        locLine := '    ' + locLine;
+        AddLine(ABinStream,locLine);
+        locLine := '';
+      end;
+    end;
+  end;
+  AddLine(ABinStream,'  );' + sLineBreak);
+end;
+
+procedure GenerateUCA_PropTable(
+// WARNING : files must be generated for each endianess (Little / Big)
+        ADest     : TStream;
+  const APropBook : PUCA_PropBook
+);
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+  p : PByte;
+begin
+  c := APropBook^.ItemSize;
+  AddLine('const');
+  AddLine('  UCA_PROPS : array[0..' + IntToStr(c-1) + '] of Byte = (');
+  locLine := '';
+  p := PByte(APropBook^.Items);
+  for i := 0 to c - 2 do begin
+    locLine := locLine + IntToStr(p[i]) + ',';
+    if (((i+1) mod 60) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(p[c-1]);
+  locLine := '    ' + locLine;
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+end;
+
+procedure GenerateUCA_OBmpTables(
+        AStream,
+        ABinStream    : TStream;
+  var   AFirstTable   : TucaOBmpFirstTable;
+  var   ASecondTable  : TucaOBmpSecondTable;
+  const AEndian       : TEndianKind
+);
+
+  procedure AddLine(AOut : TStream; const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    AOut.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+  value : UInt24;
+begin
+  AddLine(AStream,'const');
+  AddLine(AStream,'  UCAO_TABLE_1 : array[0..' + IntToStr(HIGH_SURROGATE_COUNT-1) + '] of Word = (');
+  locLine := '';
+  for i := Low(AFirstTable) to High(AFirstTable) - 1 do begin
+    locLine := locLine + IntToStr(AFirstTable[i]) + ',';
+    if (((i+1) mod 16) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(AStream,locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(AFirstTable[High(AFirstTable)]);
+  locLine := '    ' + locLine;
+  AddLine(AStream,locLine);
+  AddLine(AStream,'  );' + sLineBreak);
+
+  AddLine(ABinStream,'  UCAO_TABLE_2 : array[0..('+IntToStr(LOW_SURROGATE_COUNT)+'*' + IntToStr(Length(ASecondTable)) +'-1)] of UInt24 =(');
+  c := High(ASecondTable);
+  for i := Low(ASecondTable) to c do begin
+    locLine := '';
+    for j := Low(TucaOBmpSecondTableItem) to High(TucaOBmpSecondTableItem) do begin
+      value := ASecondTable[i][j];
+      locLine := locLine + UInt24ToStr(value,AEndian) + ',';
+      if (((j+1) mod 2) = 0) then begin
+        if (i = c) and (j = High(TucaOBmpSecondTableItem)) then
+          Delete(locLine,Length(locLine),1);
+        locLine := '    ' + locLine;
+        AddLine(ABinStream,locLine);
+        locLine := '';
+      end;
+    end;
+  end;
+  AddLine(ABinStream,'  );' + sLineBreak);
+end;
+
+//-------------------------------------------
+
+type
+  POBmpSecondTableItem = ^TOBmpSecondTableItem;
+function IndexOf(
+  const AItem  : POBmpSecondTableItem;
+  const ATable : TOBmpSecondTable;
+  const ATableActualLength : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : POBmpSecondTableItem;
+begin
+  Result := -1;
+  if (ATableActualLength > 0) then begin
+    p := @ATable[0];
+    for i := 0 to ATableActualLength - 1 do begin
+      if CompareMem(p,AItem,SizeOf(TOBmpSecondTableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+procedure MakeOBmpTables(
+  var   AFirstTable   : TOBmpFirstTable;
+  var   ASecondTable  : TOBmpSecondTable;
+  const ADataLineList : TDataLineRecArray
+);
+var
+  locLowByte, locHighByte : Word;
+  locTableItem : TOBmpSecondTableItem;
+  locCP : TUnicodeCodePoint;
+  i, locSecondActualLen : Integer;
+begin
+  SetLength(ASecondTable,2000);
+  locSecondActualLen := 0;
+  for locHighByte := 0 to HIGH_SURROGATE_COUNT - 1 do begin
+    FillChar(locTableItem,SizeOf(locTableItem),#0);
+    for locLowByte := 0 to LOW_SURROGATE_COUNT - 1 do begin
+      locCP := ToUCS4(HIGH_SURROGATE_BEGIN + locHighByte,LOW_SURROGATE_BEGIN + locLowByte);
+      locTableItem[locLowByte] := GetPropID(locCP,ADataLineList)// - 1;
+    end;
+    i := IndexOf(@locTableItem,ASecondTable,locSecondActualLen);
+    if (i = -1) then begin
+      if (locSecondActualLen = Length(ASecondTable)) then
+        SetLength(ASecondTable,locSecondActualLen + 50);
+      i := locSecondActualLen;
+      ASecondTable[i] := locTableItem;
+      Inc(locSecondActualLen);
+    end;
+    AFirstTable[locHighByte] := i;
+  end;
+  SetLength(ASecondTable,locSecondActualLen);
+end;
+
+type
+  P3lvlOBmp3TableItem = ^T3lvlOBmp3TableItem;
+function IndexOf(
+  const AItem  : P3lvlOBmp3TableItem;
+  const ATable : T3lvlOBmp3Table;
+  const ATableActualLength : Integer
+) : Integer;overload;
+var
+  i : Integer;
+  p : P3lvlOBmp3TableItem;
+begin
+  Result := -1;
+  if (ATableActualLength > 0) then begin
+    p := @ATable[0];
+    for i := 0 to ATableActualLength - 1 do begin
+      if CompareMem(p,AItem,SizeOf(T3lvlOBmp3TableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+
+type
+  P3lvlOBmp2TableItem = ^T3lvlOBmp2TableItem;
+function IndexOf(
+  const AItem  : P3lvlOBmp2TableItem;
+  const ATable : T3lvlOBmp2Table
+) : Integer;overload;
+var
+  i : Integer;
+  p : P3lvlOBmp2TableItem;
+begin
+  Result := -1;
+  if (Length(ATable) > 0) then begin
+    p := @ATable[0];
+    for i := 0 to Length(ATable) - 1 do begin
+      if CompareMem(p,AItem,SizeOf(T3lvlOBmp2TableItem)) then begin
+        Result := i;
+        Break;
+      end;
+      Inc(p);
+    end;
+  end;
+end;
+procedure MakeOBmpTables3Levels(
+  var   AFirstTable   : T3lvlOBmp1Table;
+  var   ASecondTable  : T3lvlOBmp2Table;
+  var   AThirdTable  : T3lvlOBmp3Table;
+  const ADataLineList : TDataLineRecArray
+);
+var
+  locLowByte0, locLowByte1, locHighByte : Word;
+  locTableItem2 : T3lvlOBmp2TableItem;
+  locTableItem3 : T3lvlOBmp3TableItem;
+  locCP : TUnicodeCodePoint;
+  i, locThirdActualLen : Integer;
+begin
+  SetLength(AThirdTable,120);
+  locThirdActualLen := 0;
+  for locHighByte := 0 to 1023 do begin
+    FillChar(locTableItem2,SizeOf(locTableItem2),#0);
+    for locLowByte0 := 0 to 31 do begin
+      FillChar(locTableItem3,SizeOf(locTableItem3),#0);
+      for locLowByte1 := 0 to 31 do begin
+        locCP := ToUCS4(HIGH_SURROGATE_BEGIN + locHighByte,LOW_SURROGATE_BEGIN + (locLowByte0*32) + locLowByte1);
+        locTableItem3[locLowByte1] := GetPropID(locCP,ADataLineList);
+      end;
+      i := IndexOf(@locTableItem3,AThirdTable,locThirdActualLen);
+      if (i = -1) then begin
+        if (locThirdActualLen = Length(AThirdTable)) then
+          SetLength(AThirdTable,locThirdActualLen + 50);
+        i := locThirdActualLen;
+        AThirdTable[i] := locTableItem3;
+        Inc(locThirdActualLen);
+      end;
+      locTableItem2[locLowByte0] := i;
+    end;
+    i := IndexOf(@locTableItem2,ASecondTable);
+    if (i = -1) then begin
+      i := Length(ASecondTable);
+      SetLength(ASecondTable,(i + 1));
+      ASecondTable[i] := locTableItem2;
+    end;
+    AFirstTable[locHighByte] := i;
+  end;
+  SetLength(AThirdTable,locThirdActualLen);
+end;
+
+procedure GenerateOBmpTables(
+        ADest : TStream;
+  var   AFirstTable   : TOBmpFirstTable;
+  var   ASecondTable  : TOBmpSecondTable
+);
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+begin
+  AddLine('const');
+  AddLine('  UCO_TABLE_1 : array[0..' + IntToStr(HIGH_SURROGATE_COUNT-1) + '] of Word = (');
+  locLine := '';
+  for i := Low(AFirstTable) to High(AFirstTable) - 1 do begin
+    locLine := locLine + IntToStr(AFirstTable[i]) + ',';
+    if (((i+1) mod 16) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(AFirstTable[High(AFirstTable)]);
+  locLine := '    ' + locLine;
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+
+  AddLine('  UCO_TABLE_2 : array[0..('+IntToStr(LOW_SURROGATE_COUNT)+'*' + IntToStr(Length(ASecondTable)) +'-1)] of Word =(');
+  c := High(ASecondTable);
+  for i := Low(ASecondTable) to c do begin
+    locLine := '';
+    for j := Low(TOBmpSecondTableItem) to High(TOBmpSecondTableItem) do begin
+      locLine := locLine + IntToStr(ASecondTable[i][j]) + ',';
+      if (((j+1) mod 16) = 0) then begin
+        if (i = c) and (j = High(TOBmpSecondTableItem)) then
+          Delete(locLine,Length(locLine),1);
+        locLine := '    ' + locLine;
+        AddLine(locLine);
+        locLine := '';
+      end;
+    end;
+  end;
+  AddLine('  );' + sLineBreak);
+end;
+
+
+//----------------------------------
+procedure Generate3lvlOBmpTables(
+        ADest : TStream;
+  var   AFirstTable   : T3lvlOBmp1Table;
+  var   ASecondTable  : T3lvlOBmp2Table;
+  var   AThirdTable   : T3lvlOBmp3Table
+);
+
+  procedure AddLine(const ALine : ansistring);
+  var
+    buffer : ansistring;
+  begin
+    buffer := ALine + sLineBreak;
+    ADest.Write(buffer[1],Length(buffer));
+  end;
+
+var
+  i, j, c : Integer;
+  locLine : string;
+begin
+  AddLine('const');
+  AddLine('  UCO_TABLE_1 : array[0..1023] of Word = (');
+  locLine := '';
+  for i := Low(AFirstTable) to High(AFirstTable) - 1 do begin
+    locLine := locLine + IntToStr(AFirstTable[i]) + ',';
+    if (((i+1) mod 16) = 0) then begin
+      locLine := '    ' + locLine;
+      AddLine(locLine);
+      locLine := '';
+    end;
+  end;
+  locLine := locLine + IntToStr(AFirstTable[High(AFirstTable)]);
+  locLine := '    ' + locLine;
+  AddLine(locLine);
+  AddLine('  );' + sLineBreak);
+
+  AddLine('  UCO_TABLE_2 : array[0..' + IntToStr(Length(ASecondTable)-1) +'] of array[0..31] of Word = (');
+  c := High(ASecondTable);
+  for i := Low(ASecondTable) to c do begin
+    locLine := '(';
+    for j := Low(T3lvlOBmp2TableItem) to High(T3lvlOBmp2TableItem) do
+      locLine := locLine + IntToStr(ASecondTable[i][j]) + ',';
+    Delete(locLine,Length(locLine),1);
+    locLine := '    ' + locLine + ')';
+    if (i < c) then
+      locLine := locLine + ',';
+    AddLine(locLine);
+  end;
+  AddLine('  );' + sLineBreak);
+
+  AddLine('  UCO_TABLE_3 : array[0..' + IntToStr(Length(AThirdTable)-1) +'] of array[0..31] of Word = (');
+  c := High(AThirdTable);
+  for i := Low(AThirdTable) to c do begin
+    locLine := '(';
+    for j := Low(T3lvlOBmp3TableItem) to High(T3lvlOBmp3TableItem) do
+      locLine := locLine + IntToStr(AThirdTable[i][j]) + ',';
+    Delete(locLine,Length(locLine),1);
+    locLine := '    ' + locLine + ')';
+    if (i < c) then
+      locLine := locLine + ',';
+    AddLine(locLine);
+  end;
+  AddLine('  );' + sLineBreak);
+end;
+
+function GetProp(
+  const AHighS,
+        ALowS         : Word;
+  const AProps        : TPropRecArray;
+  var   AFirstTable   : TOBmpFirstTable;
+  var   ASecondTable  : TOBmpSecondTable
+): PPropRec;
+begin
+  Result := @AProps[ASecondTable[AFirstTable[AHighS-HIGH_SURROGATE_BEGIN]][ALowS-LOW_SURROGATE_BEGIN]];
+end;
+
+function GetProp(
+  const AHighS,
+        ALowS         : Word;
+  const AProps        : TPropRecArray;
+  var   AFirstTable   : T3lvlOBmp1Table;
+  var   ASecondTable  : T3lvlOBmp2Table;
+  var   AThirdTable   : T3lvlOBmp3Table
+): PPropRec;
+begin
+  Result := @AProps[AThirdTable[ASecondTable[AFirstTable[AHighS]][ALowS div 32]][ALowS mod 32]];
+  //Result := @AProps[ASecondTable[AFirstTable[AHighS-HIGH_SURROGATE_BEGIN]][ALowS-LOW_SURROGATE_BEGIN]];
+end;
+
+{ TUCA_PropItemContextTreeRec }
+
+function TUCA_PropItemContextTreeRec.GetData : PUCA_PropItemContextTreeNodeRec;
+begin
+  if (Size = 0) then
+    Result := nil
+  else
+    Result := PUCA_PropItemContextTreeNodeRec(
+                PtrUInt(
+                  PtrUInt(@Self) + SizeOf(UInt24){Size}
+                )
+              );
+end;
+
+{ TUCA_LineContextRec }
+
+procedure TUCA_LineContextRec.Clear;
+begin
+  Data := nil
+end;
+
+procedure TUCA_LineContextRec.Assign(ASource : TUCA_LineContextRec);
+var
+  c, i : Integer;
+begin
+  c := Length(ASource.Data);
+  SetLength(Self.Data,c);
+  for i := 0 to c-1 do
+    Self.Data[i].Assign(ASource.Data[i]);
+end;
+
+function TUCA_LineContextRec.Clone : TUCA_LineContextRec;
+begin
+  Result.Clear();
+  Result.Assign(Self);
+end;
+
+{ TUCA_LineContextItemRec }
+
+procedure TUCA_LineContextItemRec.Clear();
+begin
+  CodePoints := nil;
+  Weights := nil;
+end;
+
+procedure TUCA_LineContextItemRec.Assign(ASource : TUCA_LineContextItemRec);
+begin
+  Self.CodePoints := Copy(ASource.CodePoints);
+  Self.Weights := Copy(ASource.Weights);
+end;
+
+function TUCA_LineContextItemRec.Clone() : TUCA_LineContextItemRec;
+begin
+  Result.Clear();
+  Result.Assign(Self);
+end;
+
+{ TUCA_LineRec }
+
+procedure TUCA_LineRec.Clear;
+begin
+  CodePoints := nil;
+  Weights := nil;
+  Deleted := False;
+  Stored := False;
+  Context.Clear();
+end;
+
+procedure TUCA_LineRec.Assign(ASource : TUCA_LineRec);
+begin
+  Self.CodePoints := Copy(ASource.CodePoints);
+  Self.Weights := Copy(ASource.Weights);
+  Self.Deleted := ASource.Deleted;
+  Self.Stored := ASource.Stored;
+  Self.Context.Assign(ASource.Context);
+end;
+
+function TUCA_LineRec.Clone : TUCA_LineRec;
+begin
+  Result.Clear();
+  Result.Assign(Self);
+end;
+
+function TUCA_LineRec.HasContext() : Boolean;
+begin
+  Result := (Length(Context.Data) > 0);
+end;
+
+{ TPropRec }
+
+function TPropRec.GetCategory: TUnicodeCategory;
+begin
+  Result := TUnicodeCategory((CategoryData and Byte($F8)) shr 3);
+end;
+
+procedure TPropRec.SetCategory(AValue: TUnicodeCategory);
+var
+  b : Byte;
+begin
+  b := Ord(AValue);
+  b := b shl 3;
+  CategoryData := CategoryData or b;
+  //CategoryData := CategoryData or Byte(Byte(Ord(AValue)) shl 3);
+end;
+
+function TPropRec.GetWhiteSpace: Boolean;
+begin
+  Result := IsBitON(CategoryData,0);
+end;
+
+procedure TPropRec.SetWhiteSpace(AValue: Boolean);
+begin
+  SetBit(CategoryData,0,AValue);
+end;
+
+function TPropRec.GetHangulSyllable: Boolean;
+begin
+  Result := IsBitON(CategoryData,1);
+end;
+
+procedure TPropRec.SetHangulSyllable(AValue: Boolean);
+begin
+   SetBit(CategoryData,1,AValue);
+end;
+
+{ TUCA_PropItemRec }
+
+function TUCA_PropItemRec.GetWeightLength: TWeightLength;
+begin
+  //Result := TWeightLength(Valid and Byte($FC) shr 3);
+  Result := TWeightLength((Valid and Byte($F8)) shr 3);
+end;
+
+procedure TUCA_PropItemRec.SetWeightLength(AValue: TWeightLength);
+begin
+  Valid := Valid or Byte(Byte(AValue) shl 3);
+end;
+
+function TUCA_PropItemRec.GetWeightSize : Word;
+var
+  c : Integer;
+begin
+  c := WeightLength;
+  if (c = 0) then
+    exit(0);
+  Result := c*SizeOf(TUCA_PropWeights);
+  if IsBitON(Self.Valid,BIT_POS_COMPRESS_WEIGHT_1) then
+    Result := Result - 1;
+  if IsBitON(Self.Valid,BIT_POS_COMPRESS_WEIGHT_2) then
+    Result := Result - 1;
+end;
+
+procedure TUCA_PropItemRec.GetWeightArray(ADest: PUCA_PropWeights);
+var
+  i, c : Integer;
+  p : PByte;
+  pd : PUCA_PropWeights;
+begin
+  c := WeightLength;
+  p := PByte(PtrUInt(@Self) + SizeOf(TUCA_PropItemRec));
+  pd := ADest;
+  pd^.Weights[0] := PWord(p)^;
+  p := p + 2;
+  if IsBitON(Self.Valid,BIT_POS_COMPRESS_WEIGHT_1) then begin
+    pd^.Weights[1] := PWord(p)^;
+    p := p + 2;
+  end else begin
+    pd^.Weights[1] := p^;
+    p := p + 1;
+  end;
+  if IsBitON(Self.Valid,BIT_POS_COMPRESS_WEIGHT_2) then begin
+    pd^.Weights[2] := PWord(p)^;
+    p := p + 2;
+  end else begin
+    pd^.Weights[2] := p^;
+    p := p + 1;
+  end;
+  if (c > 1) then
+    Move(p^, (pd+1)^, ((c-1)*SizeOf(TUCA_PropWeights)));
+end;
+
+function TUCA_PropItemRec.GetSelfOnlySize: Word;
+begin
+  Result := SizeOf(TUCA_PropItemRec);
+  if (WeightLength > 0) then begin
+    Result := Result + (WeightLength * Sizeof(TUCA_PropWeights));
+    if not IsBitON(Self.Valid,BIT_POS_COMPRESS_WEIGHT_1) then
+      Result := Result - 1;
+    if not IsBitON(Self.Valid,BIT_POS_COMPRESS_WEIGHT_2) then
+      Result := Result - 1;
+  end;
+end;
+
+procedure TUCA_PropItemRec.SetContextual(AValue : Boolean);
+begin
+  SetBit(Flags,FLAG_CONTEXTUAL,AValue);
+end;
+
+function TUCA_PropItemRec.GetContextual : Boolean;
+begin
+  Result := IsBitON(Flags,FLAG_CONTEXTUAL);
+end;
+
+function TUCA_PropItemRec.GetContext() : PUCA_PropItemContextTreeRec;
+var
+  p : PtrUInt;
+begin
+  if not Contextual then
+    exit(nil);
+  p := PtrUInt(@Self) + SizeOf(TUCA_PropItemRec);
+  if IsBitON(Flags,FLAG_CODEPOINT) then
+    p := p + SizeOf(UInt24);
+  Result := PUCA_PropItemContextTreeRec(p);
+end;
+
+procedure TUCA_PropItemRec.SetDeleted(AValue: Boolean);
+begin
+  SetBit(Flags,FLAG_DELETION,AValue);
+end;
+
+function TUCA_PropItemRec.IsDeleted: Boolean;
+begin
+  Result := IsBitON(Flags,FLAG_DELETION);
+end;
+
+function TUCA_PropItemRec.GetCodePoint: UInt24;
+begin
+  Result := PUInt24(PtrUInt(@Self) + Self.GetSelfOnlySize())^;
+end;
+
+function avl_CompareCodePoints(Item1, Item2: Pointer): Integer;
+var
+  a, b : PUCA_LineContextItemRec;
+  i, hb : Integer;
+begin
+  if (Item1 = Item2) then
+    exit(0);
+  if (Item1 = nil) then
+    exit(-1);
+  if (Item2 = nil) then
+    exit(1);
+  a := Item1;
+  b := Item2;
+  if (a^.CodePoints = b^.CodePoints) then
+    exit(0);
+  Result := 1;
+  hb := Length(b^.CodePoints) - 1;
+  for i := 0 to Length(a^.CodePoints) - 1 do begin
+    if (i > hb) then
+      exit;
+    if (a^.CodePoints[i] < b^.CodePoints[i]) then
+      exit(-1);
+    if (a^.CodePoints[i] > b^.CodePoints[i]) then
+      exit(1);
+  end;
+  if (Length(a^.CodePoints) = Length(b^.CodePoints)) then
+    exit(0);
+  exit(-1);
+end;
+
+function ConstructAvlContextTree(AContext : PUCA_LineContextRec) : TAVLTree;
+var
+  r : TAVLTree;
+  i : Integer;
+begin
+  r := TAVLTree.Create(@avl_CompareCodePoints);
+  try
+    for i := 0 to Length(AContext^.Data) - 1 do
+      r.Add(@AContext^.Data[i]);
+    Result := r;
+  except
+    FreeAndNil(r);
+    raise;
+  end;
+end;
+
+function ConstructContextTree(
+  const AContext : PUCA_LineContextRec;
+  var   ADestBuffer;
+  const ADestBufferLength : Integer
+) : PUCA_PropItemContextTreeRec;
+
+  function CalcItemOnlySize(AItem : TAVLTreeNode) : Cardinal;
+  var
+    kc : Integer;
+    kitem : PUCA_LineContextItemRec;
+  begin
+    if (AItem = nil) then
+      exit(0);
+    kitem := AItem.Data;
+    Result := SizeOf(PUCA_PropItemContextTreeNodeRec^.Left) +
+              SizeOf(PUCA_PropItemContextTreeNodeRec^.Right) +
+              SizeOf(PUCA_PropItemContextRec^.CodePointCount) +
+                (Length(kitem^.CodePoints)*SizeOf(UInt24)) +
+              SizeOf(PUCA_PropItemContextRec^.WeightCount) +
+                (Length(kitem^.Weights)*SizeOf(TUCA_PropWeights));
+  end;
+
+  function CalcItemSize(AItem : TAVLTreeNode) : Cardinal;
+  begin
+    if (AItem = nil) then
+      exit(0);
+    Result := CalcItemOnlySize(AItem);
+    if (AItem.Left <> nil) then
+      Result := Result + CalcItemSize(AItem.Left);
+    if (AItem.Right <> nil) then
+      Result := Result + CalcItemSize(AItem.Right);
+  end;
+
+  function CalcSize(AData : TAVLTree) : Cardinal;
+  begin
+    Result := SizeOf(PUCA_PropItemContextTreeRec^.Size) + CalcItemSize(AData.Root);
+  end;
+
+  function ConstructItem(ASource : TAVLTreeNode; ADest : PUCA_PropItemContextTreeNodeRec) : Cardinal;
+  var
+    k : Integer;
+    kitem : PUCA_LineContextItemRec;
+    kpcp : PUInt24;
+    kpw : PUCA_PropWeights;
+    pextra : PtrUInt;
+    pnext : PUCA_PropItemContextTreeNodeRec;
+  begin
+    kitem := ASource.Data;
+    ADest^.Data.CodePointCount := Length(kitem^.CodePoints);
+    ADest^.Data.WeightCount := Length(kitem^.Weights);
+    pextra := PtrUInt(ADest)+SizeOf(ADest^.Left)+SizeOf(ADest^.Right)+
+              SizeOf(ADest^.Data.CodePointCount)+SizeOf(ADest^.Data.WeightCount);
+    if (ADest^.Data.CodePointCount > 0) then begin
+      kpcp := PUInt24(pextra);
+      for k := 0 to ADest^.Data.CodePointCount - 1 do begin
+        kpcp^ := kitem^.CodePoints[k];
+        Inc(kpcp);
+      end;
+    end;
+    if (ADest^.Data.WeightCount > 0) then begin
+      kpw := PUCA_PropWeights(pextra + (ADest^.Data.CodePointCount*SizeOf(UInt24)));
+      for k := 0 to ADest^.Data.WeightCount - 1 do begin
+        kpw^.Weights[0] := kitem^.Weights[k].Weights[0];
+        kpw^.Weights[1] := kitem^.Weights[k].Weights[1];
+        kpw^.Weights[2] := kitem^.Weights[k].Weights[2];
+        Inc(kpw);
+      end;
+    end;
+    Result := CalcItemOnlySize(ASource);
+    if (ASource.Left <> nil) then begin
+      pnext := PUCA_PropItemContextTreeNodeRec(PtrUInt(ADest) + Result);
+      ADest^.Left := Result;
+      Result := Result + ConstructItem(ASource.Left,pnext);
+    end else begin
+      ADest^.Left := 0;
+    end;
+    if (ASource.Right <> nil) then begin
+      pnext := PUCA_PropItemContextTreeNodeRec(PtrUInt(ADest) + Result);
+      ADest^.Right := Result;
+      Result := Result + ConstructItem(ASource.Right,pnext);
+    end else begin
+      ADest^.Right := 0;
+    end;
+  end;
+
+var
+  c : PtrUInt;
+  r : PUCA_PropItemContextTreeRec;
+  p : PUCA_PropItemContextTreeNodeRec;
+  tempTree : TAVLTree;
+begin
+  tempTree := ConstructAvlContextTree(AContext);
+  try
+    c := CalcSize(tempTree);
+    if (ADestBufferLength > 0) and (c > ADestBufferLength) then
+      raise Exception.Create(SInsufficientMemoryBuffer);
+    r := @ADestBuffer;
+    r^.Size := c;
+    p := PUCA_PropItemContextTreeNodeRec(PtrUInt(r) + SizeOf(r^.Size));
+    ConstructItem(tempTree.Root,p);
+  finally
+    tempTree.Free();
+  end;
+  Result := r;
+end;
+
+initialization
+  FS := DefaultFormatSettings;
+  FS.DecimalSeparator := '.';
+
+end.

utils/unicode/parse-collations.bat

@@ -0,0 +1,17 @@
+cldrparser.exe de.xml -d.\data -o.\data
+echo
+cldrparser es.xml -d.\data -o.\data
+echo
+cldrparser fr_CA.xml -d.\data -o.\data
+echo
+cldrparser ja.xml -d.\data -o.\data
+echo
+cldrparser ko.xml -d.\data -o.\data
+echo
+cldrparser ru.xml -d.\data -o.\data
+echo
+cldrparser sv.xml -d.\data -o.\data
+echo
+cldrparser zh.xml -d.\data -o.\data
+
+pause

utils/unicode/trie.pas

@@ -0,0 +1,158 @@
+{   Simple TRIE implementation.
+
+    Copyright (c) 2012 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. }
+unit trie;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  SysUtils;
+
+const
+  MAX_CHILD_COUNT = 256;
+type
+  TKeyType = Cardinal;
+  TDataType = Integer;
+  PTrieNode = ^TTrieNode;
+  TTrieNode = packed record
+    Key        : TKeyType;
+    DataNode   : Boolean;
+    Data       : TDataType;
+    ChildCount : Byte;
+    Children   : array[0..(MAX_CHILD_COUNT-1)] of PTrieNode;
+  end;
+
+  function CreateNode(
+    const AKey      : TKeyType;
+    const AData     : TDataType
+  ) : PTrieNode; overload;
+  function CreateNode(const AKey : TKeyType) : PTrieNode;overload;
+  procedure FreeNode(ANode : PTrieNode);
+  function InsertWord(
+    const ARoot   : PTrieNode;
+    const AWord   : array of TKeyType;
+    const AValue  : TDataType
+  ) : Boolean;overload;
+  function InsertWord(
+    const ARoot   : PTrieNode;
+    const AWord   : TKeyType;
+    const AValue  : TDataType
+  ) : Boolean;overload;
+
+implementation
+
+function CreateNode(
+  const AKey      : TKeyType;
+  const AData     : TDataType
+) : PTrieNode;
+begin
+  New(Result);
+  Result^.Key := AKey;
+  Result^.DataNode := True;
+  Result^.Data := AData;
+  Result^.ChildCount := 0;
+end;
+
+function CreateNode(const AKey : TKeyType) : PTrieNode;
+begin
+  New(Result);
+  Result^.Key := AKey;
+  Result^.DataNode := False;
+  Result^.ChildCount := 0;
+end;
+
+procedure FreeNode(ANode : PTrieNode);
+var
+  p : PTrieNode;
+  i : Integer;
+begin
+  if (ANode = nil) then
+    exit;
+  p := ANode;
+  for i := 0 to p^.ChildCount - 1 do
+    FreeNode(p^.Children[i]);
+  Dispose(p);
+end;
+
+function InsertWord(
+  const ARoot   : PTrieNode;
+  const AWord   : TKeyType;
+  const AValue  : TDataType
+) : Boolean;
+begin
+  Result := InsertWord(ARoot,[AWord],AValue);
+end;
+
+function InsertWord(
+  const ARoot   : PTrieNode;
+  const AWord   : array of TKeyType;
+  const AValue  : TDataType
+) : Boolean;
+var
+  p, q : PTrieNode;
+  i, k, c : Integer;
+  searching : TKeyType;
+  found : Boolean;
+begin
+  Result := False;
+  if (ARoot^.Key <> AWord[0]) then
+    exit;
+  p := ARoot;
+  q := p;
+  i := 1;
+  c := Length(AWord);
+  while (i < c) do begin
+    searching := AWord[i];
+    found := False;
+    for k := 0 to p^.ChildCount - 1 do begin
+      if (p^.Children[k]^.Key = searching) then begin
+        q := p;
+        p :=  p^.Children[k];
+        found := True;
+        Break;
+      end;
+    end;
+    if not found then
+      Break;
+    Inc(i);
+  end;
+  if (i < c) then begin
+    if (i = c) then
+      i := i - 1;
+    for i := i to c - 2 do begin
+      k := p^.ChildCount;
+      p^.Children[k] := CreateNode(AWord[i]);
+      p^.ChildCount := k + 1;
+      p := p^.Children[k];
+    end;
+    i := c - 1;
+    k := p^.ChildCount;
+    p^.Children[k] := CreateNode(AWord[i],AValue);
+    p^.ChildCount := k + 1;
+    p := p^.Children[k];
+    Result := True;
+  end;
+end;
+
+
+
+end.
+

utils/unicode/uca_test.pas

@@ -0,0 +1,282 @@
+{   Unicode Collation Algorithm test routines for generated data.
+
+    Copyright (c) 2012 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+}
+
+unit uca_test;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  SysUtils,
+  helper;
+
+  procedure uca_CheckProp_1(
+    ABook : TUCA_DataBook;
+    APropBook : PUCA_PropBook
+  );
+  procedure uca_CheckProp_x(
+    ABook : TUCA_DataBook;
+    APropBook : PUCA_PropBook
+  );
+
+  procedure uca_CheckProp_1y(
+    const ABook         : TUCA_DataBook;
+    const APropBook     : PUCA_PropBook;
+    const AFirstTable   : PucaBmpFirstTable;
+    const ASecondTable  : PucaBmpSecondTable
+  );
+  procedure uca_CheckProp_2y(
+    const ABook         : TUCA_DataBook;
+    const APropBook     : PUCA_PropBook;
+    const AFirstTable   : PucaOBmpFirstTable;
+    const ASecondTable  : PucaOBmpSecondTable
+  );
+
+implementation
+
+function IndexOf(const ACodePoint : Cardinal; APropBook : PUCA_PropBook): Integer;
+var
+  i : Integer;
+begin
+  for i := 0 to Length(APropBook^.Index) - 1 do begin
+    if (ACodePoint = APropBook^.Index[i].CodePoint) then
+      exit(i);
+  end;
+  Result := -1;
+end;
+
+function CompareWeigth(AExpect : PUCA_LineRec; AActual : PUCA_PropItemRec) : Boolean;
+var
+  i, k : Integer;
+  p : PUCA_PropWeights;
+  pw : array of TUCA_PropWeights;
+begin
+  Result := False;
+  if (Length(AExpect^.Weights) <> AActual^.WeightLength) then
+    exit;
+  //p := PUCA_PropWeights(PtrUInt(AActual) + SizeOf(TUCA_PropItemRec));
+  SetLength(pw,AActual^.WeightLength);
+  p := @pw[0];
+  AActual^.GetWeightArray(p);
+  for i := 0 to Length(AExpect^.Weights) - 1 do begin
+    //if (BoolToByte(AExpect^.Weights[i].Variable) <> p^.Variable) then
+      //exit;
+    for k := 0 to 3 - 1 do begin
+      if (AExpect^.Weights[i].Weights[k] <> p^.Weights[k]) then
+        exit;
+    end;
+    Inc(p);
+  end;
+  Result := True;
+end;
+
+procedure uca_CheckProp_1(
+  ABook : TUCA_DataBook;
+  APropBook : PUCA_PropBook
+);
+var
+  i, c, k : Integer;
+  line : PUCA_LineRec;
+  uc : Cardinal;
+  p : PUCA_PropItemRec;
+begin
+  WriteLn('uca_CheckProp_1 Start ... ');
+  line := @ABook.Lines[0];
+  c := Length(ABook.Lines);
+  for i := 0 to c - 1 do begin
+    if line^.Stored and (Length(line^.CodePoints) = 1) then begin
+      uc := line^.CodePoints[0];
+      k := IndexOf(uc,APropBook);
+      if (k = -1) then begin
+        WriteLn('Property not found for Code Point : ' + Format('%x',[uc]));
+      end else begin
+        p := PUCA_PropItemRec(PtrUInt(APropBook^.Items)+APropBook^.Index[k].Position);
+        if not CompareWeigth(line,p) then
+          WriteLn('CompareWeigth fail for Code Point : ' + Format('%x',[uc]));
+      end;
+    end;
+    Inc(line);
+  end;
+  WriteLn('uca_CheckProp_1 End');
+end;
+
+function FindWord(
+  const AWord : array of Cardinal;
+  const APropBook : PUCA_PropItemRec
+) : PUCA_PropItemRec;
+var
+  cc : Cardinal;
+  p : PUCA_PropItemRec;
+  i, k, kc : Integer;
+  ok : Boolean;
+begin
+  Result := nil;
+  p := APropBook;
+  for i := 1 to Length(AWord) - 1 do begin
+    ok := False;
+    kc := p^.ChildCount - 1;
+    p := PUCA_PropItemRec(PtrUInt(p) + p^.GetSelfOnlySize());
+    if (i > 1) then
+      p := PUCA_PropItemRec(PtrUInt(p) + SizeOf(UInt24));
+    for k := 0 to kc do begin
+      if (AWord[i] = p^.CodePoint) then begin
+        ok := True;
+        Break;
+      end;
+      p := PUCA_PropItemRec(PtrUInt(p) + p^.Size);
+    end;
+    if not ok then
+      exit;
+  end;
+  Result := p;
+end;
+
+function DumpCodePoints(const AValues : array of Cardinal) : string;
+var
+  i : Integer;
+begin
+  Result := '';
+  for i := 0 to Length(AValues) - 1 do
+    Result := Format('%s %x',[Result,AValues[i]]);
+  Result := Trim(Result);
+end;
+
+procedure uca_CheckProp_x(
+  ABook : TUCA_DataBook;
+  APropBook : PUCA_PropBook
+);
+var
+  i, c, k : Integer;
+  line : PUCA_LineRec;
+  uc : Cardinal;
+  p, q : PUCA_PropItemRec;
+begin
+  WriteLn('uca_CheckProp_x Start ... ');
+  line := @ABook.Lines[0];
+  c := Length(ABook.Lines);
+  for i := 0 to c - 1 do begin
+    if line^.Stored and (Length(line^.CodePoints) > 1) then begin
+      //WriteLn('  Code Point sequence  : ' + DumpCodePoints(line^.CodePoints));
+      uc := line^.CodePoints[0];
+      k := IndexOf(uc,APropBook);
+      if (k = -1) then begin
+        WriteLn('    Property not found for Code Point : ' + Format('%x',[uc]));
+      end else begin
+        q := PUCA_PropItemRec(PtrUInt(APropBook^.Items)+APropBook^.Index[k].Position);
+        p := FindWord(line^.CodePoints,q);
+        if (p = nil) then
+          WriteLn('    Data not found for Code Point sequence  : ' + DumpCodePoints(line^.CodePoints))
+        else if not CompareWeigth(line,p) then
+          WriteLn('    CompareWeigth fail for Code Point sequence  : ' + DumpCodePoints(line^.CodePoints));
+      end;
+    end;
+    Inc(line);
+  end;
+  WriteLn('uca_CheckProp_x End');
+end;
+
+
+function GetPropPosition(
+  const ABMPCodePoint : Word;
+  const AFirstTable   : PucaBmpFirstTable;
+  const ASecondTable  : PucaBmpSecondTable
+) : Integer; inline;overload;
+begin
+  Result:=
+      ASecondTable^[AFirstTable^[WordRec(ABMPCodePoint).Hi]][WordRec(ABMPCodePoint).Lo] - 1
+end;
+
+procedure uca_CheckProp_1y(
+  const ABook         : TUCA_DataBook;
+  const APropBook     : PUCA_PropBook;
+  const AFirstTable   : PucaBmpFirstTable;
+  const ASecondTable  : PucaBmpSecondTable
+);
+var
+  i, c, k : Integer;
+  line : PUCA_LineRec;
+  uc : Cardinal;
+  p : PUCA_PropItemRec;
+  ucw : Word;
+begin
+  WriteLn('uca_CheckProp_1y Start (BMP) ... ');
+  line := @ABook.Lines[0];
+  c := Length(ABook.Lines);
+  for i := 0 to c - 1 do begin
+    if line^.Stored and (Length(line^.CodePoints) = 1) then begin
+      uc := line^.CodePoints[0];
+      if (uc <= High(Word)) then begin
+        ucw := uc;
+        k := GetPropPosition(ucw,AFirstTable,ASecondTable);
+        if (k = -1) then begin
+          WriteLn('Property not found for Code Point : ' + Format('%x',[uc]));
+        end else begin
+          p := PUCA_PropItemRec(PtrUInt(APropBook^.Items)+k);
+          if not CompareWeigth(line,p) then
+            WriteLn('CompareWeigth fail for Code Point : ' + Format('%x',[uc]));
+        end;
+      end;
+    end;
+    Inc(line);
+  end;
+  WriteLn('uca_CheckProp_1y End');
+end;
+
+procedure uca_CheckProp_2y(
+  const ABook         : TUCA_DataBook;
+  const APropBook     : PUCA_PropBook;
+  const AFirstTable   : PucaOBmpFirstTable;
+  const ASecondTable  : PucaOBmpSecondTable
+);
+var
+  i, c, k : Integer;
+  line : PUCA_LineRec;
+  uc : Cardinal;
+  p : PUCA_PropItemRec;
+  uchs, ucls : Word;
+begin
+  WriteLn('uca_CheckProp_2y Start (>BMP) ... ');
+  line := @ABook.Lines[0];
+  c := Length(ABook.Lines);
+  for i := 0 to c - 1 do begin
+    if line^.Stored and (Length(line^.CodePoints) = 1) then begin
+      uc := line^.CodePoints[0];
+      if (uc > High(Word)) then begin
+        FromUCS4(uc,uchs,ucls);
+        k := GetPropPosition(uchs,ucls,AFirstTable,ASecondTable);
+        if (k = -1) then begin
+          WriteLn('Property not found for Code Point : ' + Format('%x',[uc]));
+        end else begin
+          p := PUCA_PropItemRec(PtrUInt(APropBook^.Items)+k);
+          if not CompareWeigth(line,p) then
+            WriteLn('CompareWeigth fail for Code Point : ' + Format('%x',[uc]));
+        end;
+      end;
+    end;
+    Inc(line);
+  end;
+  WriteLn('uca_CheckProp_2y End');
+end;
+
+
+end.
+

utils/unicode/unicodeset.pas

@@ -0,0 +1,426 @@
+{   UnicodeSet implementation.
+
+    Copyright (c) 2013 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+}
+unit unicodeset;
+
+{$mode delphi}{$H+}
+{$scopedenums on}
+
+interface
+
+uses
+  SysUtils,
+  grbtree, helper;
+
+type
+
+  EUnicodeSetException = class(Exception)
+  end;
+
+  TUnicodeSet = class;
+
+  TPatternParser = class
+  private
+    FBufferStr : UnicodeString;
+    FBuffer : PUnicodeChar;
+    FBufferLength : Integer;
+    FSet : TUnicodeSet;
+    FPosition : Integer;
+  private
+    procedure Error(const AMsg : string; const AArgs : array of const);overload;inline;
+    procedure Error(const AMsg : string);overload;inline;
+    procedure SetBuffer(const APattern : PUnicodeChar; const ALength : Integer);
+    procedure CheckEOF();inline;overload;
+    procedure CheckEOF(ALength : Integer);overload;inline;
+    procedure UnexpectedEOF();inline;
+    function IsThis(AItem : UnicodeString; const APosition : Integer) : Boolean;overload;
+    function IsThis(AItem : UnicodeString) : Boolean;overload;inline;
+    procedure Expect(AItem : UnicodeString; const APosition : Integer);overload;inline;
+    procedure Expect(AItem : UnicodeString);overload;inline;
+    procedure SkipSpaces();inline;
+    function NextChar() : TUnicodeCodePoint;
+    procedure ParseItem();
+    procedure DoParse();
+  public
+    procedure Parse(const APattern : PUnicodeChar; const ALength : Integer);overload;
+    procedure Parse(const APattern : UnicodeString);overload;inline;
+    property CurrentSet : TUnicodeSet read FSet write FSet;
+  end;
+
+  TUnicodeCodePointArrayComparator = class
+  public
+    // Return
+    //    * if A>B then  1
+    //    * if A=B then  0
+    //    * if A<B then -1
+    class function Compare(const A, B : TUnicodeCodePointArray) : Integer;static;inline;
+  end;
+
+  TUnicodeSet = class
+  private type
+      TItem = TUnicodeCodePointArray;
+      TTree = TRBTree<TItem,TUnicodeCodePointArrayComparator>;
+  public type
+    TIterator = TTree.TIterator;
+  private
+    FTree : TTree;
+    FParser : TPatternParser;
+  private
+    procedure CreateParser();inline;
+  public
+    constructor Create();
+    destructor Destroy;override;
+    procedure Add(AChar : TUnicodeCodePoint);inline;overload;
+    procedure Add(AString : TUnicodeCodePointArray);inline;overload;
+    procedure AddRange(const AStart, AEnd : TUnicodeCodePoint);inline;
+    procedure AddPattern(const APattern : UnicodeString);inline;
+    function CreateIterator() : TIterator;
+    function Contains(const AString : array of TUnicodeCodePoint) : Boolean;overload;
+    function Contains(const AChar : TUnicodeCodePoint) : Boolean;inline;overload;
+    function Contains(const AChar : UnicodeChar) : Boolean;inline;overload;
+    function Contains(const AChar : AnsiChar) : Boolean;inline;overload;
+  end;
+
+resourcestring
+  SInvalidLength = 'Invalid length value : "%d".';
+  SInvalidPosition = 'Invalid position : "%d".';
+  SInvalidRangeLimits = 'Invalid range limits : ["%x" , "%x"].';
+  SExpectedBut = 'Expects "%s" but got "%s..." .';
+  SUnexpectedEOF = 'Unexpected end of file.';
+
+implementation
+uses
+  unicodedata;
+
+function ToArray(const AItem : TUnicodeCodePoint) : TUnicodeCodePointArray;inline;
+begin
+  SetLength(Result,1);
+  Result[Low(Result)] := AItem;
+end;
+
+function CompareItem(const Item1, Item2 : TUnicodeCodePointArray): Integer;
+var
+  a, b : ^TUnicodeCodePoint;
+  i, ha, hb : Integer;
+begin
+  if (Pointer(Item1) = Pointer(Item2)) then
+    exit(0);
+  if (Item1 = nil) then
+    exit(-1);
+  if (Item2 = nil) then
+    exit(1);
+  a := @Item1[0];
+  b := @Item2[0];
+  Result := 1;
+  ha := Length(Item1) - 1;
+  hb := Length(Item2) - 1;
+  for i := 0 to ha do begin
+    if (i > hb) then
+      exit;
+    if (a^ < b^) then
+      exit(-1);
+    if (a^ > b^) then
+      exit(1);
+    Inc(a);
+    Inc(b);
+  end;
+  if (ha = hb) then
+    exit(0);
+  exit(-1);
+end;
+
+{ TUnicodeCodePointArrayComparator }
+
+class function TUnicodeCodePointArrayComparator.Compare(const A, B : TUnicodeCodePointArray): Integer;
+begin
+  Result := CompareItem(A,B);
+end;
+
+{ TPatternParser }
+
+procedure TPatternParser.Error(const AMsg: string; const AArgs: array of const);
+begin
+  raise EUnicodeSetException.CreateFmt(AMsg,AArgs);
+end;
+
+procedure TPatternParser.Error(const AMsg: string);
+begin
+  raise EUnicodeSetException.Create(AMsg);
+end;
+
+procedure TPatternParser.SetBuffer(
+  const APattern : PUnicodeChar;
+  const ALength  : Integer
+);
+begin
+  FPosition := 0;
+  if (ALength <= 1) then begin
+    FBufferStr := '';
+    FBuffer := nil;
+    FBufferLength := 0;
+    exit;
+  end;
+  FBufferLength := ALength;
+  SetLength(FBufferStr,FBufferLength);
+  FBuffer := @FBufferStr[1];
+  Move(APattern^,FBuffer^,(FBufferLength*SizeOf(FBuffer^)));
+end;
+
+procedure TPatternParser.CheckEOF();
+begin
+  CheckEOF(0);
+end;
+
+procedure TPatternParser.CheckEOF(ALength : Integer);
+begin
+  if (ALength < 0) then
+    Error(SInvalidLength,[ALength]);
+  if ((FPosition+ALength) >= FBufferLength) then
+    UnexpectedEOF();
+end;
+
+procedure TPatternParser.UnexpectedEOF();
+begin
+  Error(SUnexpectedEOF);
+end;
+
+function TPatternParser.IsThis(AItem: UnicodeString; const APosition: Integer): Boolean;
+var
+  i, k, c : Integer;
+begin
+  if (APosition < 0) then
+    Error(SInvalidPosition,[APosition]);
+  Result := False;
+  c := Length(AItem);
+  if (c = 0) then
+    exit;
+  i := APosition;
+  k := i + c;
+  if (k >= FBufferLength) then
+    exit;
+  if CompareMem(@AItem[1], @FBuffer[APosition],c) then
+    Result := True;
+end;
+
+function TPatternParser.IsThis(AItem : UnicodeString) : Boolean;
+begin
+  Result := IsThis(AItem,FPosition);
+end;
+
+procedure TPatternParser.Expect(AItem: UnicodeString; const APosition: Integer);
+begin
+  if not IsThis(AItem,APosition) then
+    Error(SExpectedBut,[AItem,Copy(FBuffer,APosition,Length(AItem))]);
+end;
+
+procedure TPatternParser.Expect(AItem: UnicodeString);
+begin
+  Expect(AItem,FPosition);
+end;
+
+procedure TPatternParser.SkipSpaces();
+begin
+  while (FPosition < FBufferLength) do begin
+    if (FBuffer[FPosition] <> ' ') then
+      Break;
+    Inc(FPosition);
+  end;
+end;
+
+function TPatternParser.NextChar(): TUnicodeCodePoint;
+var
+  i : Integer;
+  c : UnicodeChar;
+  cp : TUnicodeCodePoint;
+  s : UnicodeString;
+begin
+  SkipSpaces();
+  CheckEOF();
+  c := FBuffer[FPosition];
+  cp := Ord(c);
+  Inc(FPosition);
+  cp := Ord(c);
+  if (c = '\') and (FPosition < FBufferLength) then begin
+    if IsThis('\') then begin
+      Inc(FPosition);
+      CheckEOF();
+      cp := Ord(FBuffer[FPosition]);
+      Inc(FPosition);
+    end else if IsThis('u') then begin
+      Inc(FPosition);
+      CheckEOF(4);
+      s := Copy(FBufferStr,(FPosition+1),4);
+      Inc(FPosition,4);
+      if not TryStrToInt('$'+s,i) then
+        Error(SExpectedBut,['\uXXXX',s]);
+      cp := i;
+    end;
+  end;
+  if (cp <= MAX_WORD) and UnicodeIsLowSurrogate(UnicodeChar(Word(cp))) then begin
+    SkipSpaces();
+    CheckEOF();
+    c := UnicodeChar(Word(cp));
+    if UnicodeIsSurrogatePair(c,FBuffer[FPosition]) then begin
+      cp := ToUCS4(c,FBuffer[FPosition]);
+      Inc(FPosition);
+    end;
+  end;
+  Result := cp;
+end;
+
+function CompareTo(const A : TUnicodeCodePoint; const B : UnicodeChar) : Boolean;inline;
+begin
+  Result := (A = Ord(B));
+end;
+
+procedure TPatternParser.ParseItem();
+var
+  cp, lastCp : TUnicodeCodePoint;
+  charCount, i : Integer;
+begin
+  SkipSpaces();
+  Expect('[');
+  charCount := 0;
+  Inc(FPosition);
+  while (FPosition < FBufferLength) do begin
+    lastCp := cp;
+    cp := NextChar();
+    if CompareTo(cp,']') then
+      Break;
+    if CompareTo(cp,'-') then begin
+      if (charCount = 0) then
+        Error(SExpectedBut,['<char>','-']);
+      cp := NextChar();
+      FSet.AddRange(lastCp,cp);
+    end else begin
+      FSet.Add(cp);
+    end;
+    Inc(charCount);
+  end;
+end;
+
+procedure TPatternParser.DoParse();
+begin
+  SkipSpaces();
+  while (FPosition < FBufferLength) do begin
+    ParseItem();
+    SkipSpaces();
+  end;
+end;
+
+procedure TPatternParser.Parse(const APattern: PUnicodeChar; const ALength: Integer);
+begin
+  if (ALength < 2) then
+    exit;
+  SetBuffer(APattern,ALength);
+  DoParse();
+end;
+
+procedure TPatternParser.Parse(const APattern : UnicodeString);
+begin
+  Parse(@APattern[1],Length(APattern));
+end;
+
+{ TUnicodeSet }
+
+procedure TUnicodeSet.CreateParser();
+begin
+  if (FParser = nil) then begin
+    FParser := TPatternParser.Create();
+    FParser.CurrentSet := Self;
+  end;
+end;
+
+constructor TUnicodeSet.Create;
+begin
+  FTree := TTree.Create();
+end;
+
+destructor TUnicodeSet.Destroy;
+begin
+  FParser.Free();
+  FTree.Free();
+  inherited Destroy;
+end;
+
+procedure TUnicodeSet.Add(AChar: TUnicodeCodePoint);
+begin
+  FTree.Insert(ToArray(AChar));
+end;
+
+procedure TUnicodeSet.Add(AString: TUnicodeCodePointArray);
+begin
+  if (AString <> nil) then
+    FTree.Insert(AString);
+end;
+
+procedure TUnicodeSet.AddRange(const AStart, AEnd : TUnicodeCodePoint);
+var
+  i : Integer;
+begin
+  if (AStart > AEnd) then
+    raise EUnicodeSetException.CreateFmt(SInvalidRangeLimits,[AStart,AEnd]);
+  for i := AStart to AEnd do
+    Add(i);
+end;
+
+procedure TUnicodeSet.AddPattern(const APattern : UnicodeString);
+begin
+  CreateParser();
+  FParser.Parse(APattern);
+end;
+
+function TUnicodeSet.CreateIterator() : TIterator;
+begin
+  Result := FTree.CreateForwardIterator();
+end;
+
+function TUnicodeSet.Contains(const AString : array of TUnicodeCodePoint) : Boolean;
+var
+  c : Integer;
+  x : TUnicodeCodePointArray;
+begin
+  Result := False;
+  c := Length(AString);
+  if (c = 0) then
+    exit;
+  SetLength(x,c);
+  Move(AString[Low(AString)],x[Low(x)],(c*SizeOf(x[0])));
+  if (FTree.FindNode(x) <> nil) then
+    Result := True;
+end;
+
+function TUnicodeSet.Contains(const AChar : TUnicodeCodePoint) : Boolean;
+begin
+  Result := Contains([AChar]);
+end;
+
+function TUnicodeSet.Contains(const AChar : UnicodeChar) : Boolean;
+begin
+  Result := Contains(TUnicodeCodePoint(Ord(AChar)));
+end;
+
+function TUnicodeSet.Contains(const AChar : AnsiChar) : Boolean;
+begin
+  Result := Contains(TUnicodeCodePoint(Ord(AChar)));
+end;
+
+end.
+

utils/unicode/unihelper.lpi

@@ -0,0 +1,74 @@
+<?xml version="1.0"?>
+<CONFIG>
+  <ProjectOptions>
+    <Version Value="9"/>
+    <PathDelim Value="\"/>
+    <General>
+      <Flags>
+        <MainUnitHasCreateFormStatements Value="False"/>
+        <MainUnitHasTitleStatement Value="False"/>
+      </Flags>
+      <SessionStorage Value="InProjectDir"/>
+      <MainUnit Value="0"/>
+      <Title Value="unihelper"/>
+      <UseAppBundle Value="False"/>
+      <ResourceType Value="res"/>
+    </General>
+    <i18n>
+      <EnableI18N LFM="False"/>
+    </i18n>
+    <VersionInfo>
+      <StringTable ProductVersion=""/>
+    </VersionInfo>
+    <BuildModes Count="1">
+      <Item1 Name="Default" Default="True"/>
+    </BuildModes>
+    <PublishOptions>
+      <Version Value="2"/>
+      <IncludeFileFilter Value="*.(pas|pp|inc|lfm|lpr|lrs|lpi|lpk|sh|xml)"/>
+      <ExcludeFileFilter Value="*.(bak|ppu|o|so);*~;backup"/>
+    </PublishOptions>
+    <RunParams>
+      <local>
+        <FormatVersion Value="1"/>
+      </local>
+    </RunParams>
+    <Units Count="1">
+      <Unit0>
+        <Filename Value="unihelper.lpr"/>
+        <IsPartOfProject Value="True"/>
+      </Unit0>
+    </Units>
+  </ProjectOptions>
+  <CompilerOptions>
+    <Version Value="11"/>
+    <PathDelim Value="\"/>
+    <Target>
+      <Filename Value="unihelper"/>
+    </Target>
+    <SearchPaths>
+      <IncludeFiles Value="$(ProjOutDir)"/>
+      <OtherUnitFiles Value="..\trie"/>
+      <UnitOutputDirectory Value="lib\$(TargetCPU)-$(TargetOS)"/>
+    </SearchPaths>
+    <Other>
+      <CompilerMessages>
+        <MsgFileName Value=""/>
+      </CompilerMessages>
+      <CompilerPath Value="$(CompPath)"/>
+    </Other>
+  </CompilerOptions>
+  <Debugging>
+    <Exceptions Count="3">
+      <Item1>
+        <Name Value="EAbort"/>
+      </Item1>
+      <Item2>
+        <Name Value="ECodetoolError"/>
+      </Item2>
+      <Item3>
+        <Name Value="EFOpenError"/>
+      </Item3>
+    </Exceptions>
+  </Debugging>
+</CONFIG>

utils/unicode/unihelper.lpr

@@ -0,0 +1,386 @@
+{   Unicode tables parser.
+
+    Copyright (c) 2012 by Inoussa OUEDRAOGO
+
+    The source code is distributed under the Library GNU
+    General Public License with the following modification:
+
+        - object files and libraries linked into an application may be
+          distributed without source code.
+
+    If you didn't receive a copy of the file COPYING, contact:
+          Free Software Foundation
+          675 Mass Ave
+          Cambridge, MA  02139
+          USA
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. }
+
+{ This program generates tables as include-files for use
+  with the unicode related sources. It expects the following
+  unicode.org's files to be present in the same folder :
+    * HangulSyllableType.txt
+    * PropList.txt
+    * UnicodeData.txt
+    * allkeys.txt
+}
+
+{$DEFINE UCA_TEST}
+program unihelper;
+
+{$mode objfpc}{$H+}
+
+uses
+  SysUtils, Classes,
+  helper, uca_test;
+
+const
+  SUsage =
+    'This program generates tables as include-files for use ' + sLineBreak +
+    '  with the unicode related sources. It expects the following ' + sLineBreak +
+    '  unicode.org''s files to be present in the same folder : ' + sLineBreak +
+    '    * HangulSyllableType.txt ' + sLineBreak +
+    '    * PropList.txt ' + sLineBreak +
+    '    * UnicodeData.txt ' + sLineBreak +
+    '    * allkeys.txt : Note that this file is the one provided for the CLDR root.' + sLineBreak +
+    '' + sLineBreak +
+    'Usage : unihelper [<dataDir> <outputDir>] ' + sLineBreak +
+    '  where ' + sLineBreak +
+    '    dataDir : the directory where are stored the unicode files. The default' + sLineBreak +
+    '              value is the program''s directory.' + sLineBreak +
+    '    outputDir : The directory where the generated files will be stored. The' + sLineBreak +
+    '                default value is the program''s directory.'+sLineBreak;
+
+function DumpCodePoint(ACodePoint : TCodePointRec) : string;
+begin
+  Result := '';
+  if (ACodePoint.LineType = 0) then
+    WriteStr(Result,IntToHex(ACodePoint.CodePoint,4))
+  else
+    WriteStr(Result,IntToHex(ACodePoint.StartCodePoint,4),'..',IntToHex(ACodePoint.EndCodePoint,4));
+end;
+
+var
+  dataPath, outputPath : string;
+  stream, binStream, binStream2 : TMemoryStream;
+  hangulSyllables : TCodePointRecArray;
+  ucaBook : TUCA_DataBook;
+  ucaPropBook : PUCA_PropBook;
+  propList : TPropListLineRecArray;
+  whiteSpaceCodePoints : TCodePointRecArray;
+  props : TPropRecArray;
+  numericTable : TNumericValueArray;
+  decomposition : TDecompositionArray;
+  decompositionBook : TDecompositionBook;
+  data : TDataLineRecArray;
+  //----------------
+  lvl3table1 : T3lvlBmp1Table;
+  lvl3table2 : T3lvlBmp2Table;
+  lvl3table3 : T3lvlBmp3Table;
+  //----------------
+  s : ansistring;
+  i, k, h : Integer;
+  p : PDataLineRec;
+  r : TDataLineRecArray;
+  olvl3table1 : T3lvlOBmp1Table;
+  olvl3table2 : T3lvlOBmp2Table;
+  olvl3table3 : T3lvlOBmp3Table;
+  //----------------
+  hs, ls : Word;
+  ucaFirstTable   : TucaBmpFirstTable;
+  ucaSecondTable  : TucaBmpSecondTable;
+  ucaoFirstTable   : TucaoBmpFirstTable;
+  ucaoSecondTable  : TucaOBmpSecondTable;
+  WL : Integer;
+begin
+  WriteLn(SUsage+sLineBreak);
+  if (ParamCount > 0) then
+    dataPath := IncludeTrailingPathDelimiter(ParamStr(1))
+  else
+    dataPath := ExtractFilePath(ParamStr(0));
+  if (ParamCount > 1) then
+    outputPath := IncludeTrailingPathDelimiter(ParamStr(2))
+  else
+    outputPath := dataPath;
+  if not DirectoryExists(outputPath) then begin
+    WriteLn('Directory not found : ',outputPath);
+    if ForceDirectories(outputPath) then begin
+      WriteLn('  directory created successfully');
+    end else begin
+      WriteLn('  fail to create directory.');
+      Halt(1);
+    end;
+  end;
+  if not(
+       FileExists(dataPath + 'HangulSyllableType.txt') and
+       FileExists(dataPath + 'PropList.txt') and
+       FileExists(dataPath + 'UnicodeData.txt') and
+       FileExists(dataPath + 'allkeys.txt')
+     )
+  then begin
+    WriteLn('File(s) not found : HangulSyllableType.txt or PropList.txt or UnicodeData.txt or allkeys.txt .');
+    Halt(1);
+  end;
+
+  binStream2 := nil;
+  binStream := nil;
+  stream := TMemoryStream.Create();
+  try
+    binStream := TMemoryStream.Create();
+    binStream2 := TMemoryStream.Create();
+    WriteLn('Load file HangulSyllableType.txt ...', DateTimeToStr(Now));
+    stream.LoadFromFile(dataPath + 'HangulSyllableType.txt');
+    stream.Position := 0;
+    hangulSyllables := nil;
+    ParseHangulSyllableTypes(stream,hangulSyllables);
+    stream.Clear();
+
+    WriteLn('Load file PropList.txt ...', DateTimeToStr(Now));
+    stream.LoadFromFile(dataPath + 'PropList.txt');
+    stream.Position := 0;
+    propList := nil;
+    ParseProps(stream,propList);
+    stream.Clear();
+    whiteSpaceCodePoints := FindCodePointsByProperty('White_Space',propList);
+    writeln('  PropList Length = ',Length(propList));
+    writeln('  White_Space Length = ',Length(whiteSpaceCodePoints));
+    for i := Low(whiteSpaceCodePoints) to High(whiteSpaceCodePoints) do
+      WriteLn('      ',DumpCodePoint(whiteSpaceCodePoints[i]):12,' , IsWhiteSpace = ',IsWhiteSpace(whiteSpaceCodePoints[i].CodePoint,whiteSpaceCodePoints));
+
+    WriteLn('Load file UnicodeData.txt ...', DateTimeToStr(Now));
+    stream.LoadFromFile(dataPath + 'UnicodeData.txt');
+    stream.Position := 0;
+    WriteLn('Parse file ...', DateTimeToStr(Now));
+    data := nil;
+    props := nil;
+    Parse_UnicodeData(stream,props,numericTable,data,decomposition,hangulSyllables,whiteSpaceCodePoints);
+    WriteLn('Decomposition building ...');
+    MakeDecomposition(decomposition,decompositionBook);
+
+    WriteLn('Load file UCA allkeys.txt ...', DateTimeToStr(Now));
+    stream.LoadFromFile(dataPath + 'allkeys.txt');
+    stream.Position := 0;
+    ParseUCAFile(stream,ucaBook);
+ { $IFDEF UCA_TEST}
+    k := 0;  WL := 0; ;
+    for i := 0 to Length(ucaBook.Lines) - 1 do begin
+      h := GetPropID(ucaBook.Lines[i].CodePoints[0],data);
+      if (h <> -1) and
+         ({props[h].HangulSyllable or} (props[h].DecompositionID <> -1))
+      then begin
+        Inc(k);
+        ucaBook.Lines[i].Stored := False;
+      end else begin
+        ucaBook.Lines[i].Stored := True;
+        if Length(ucaBook.Lines[i].Weights) > WL then
+          WL := Length(ucaBook.Lines[i].Weights);
+      end;
+    end;
+    WriteLn(
+      'UCA, Version = ',ucaBook.Version,'; entries count = ',Length(ucaBook.Lines),' ; Hangul # = ',k,
+      'Max Weights Length = ',WL
+    );
+{ $ENDIF UCA_TEST}
+    WriteLn('Construct UCA Property Book ...');
+    ucaPropBook := nil;
+    MakeUCA_Props(@ucaBook,ucaPropBook);
+{$IFDEF UCA_TEST}
+    uca_CheckProp_1(ucaBook,ucaPropBook);
+    uca_CheckProp_x(ucaBook,ucaPropBook);
+{$ENDIF UCA_TEST}
+    WriteLn('Construct UCA BMP tables ...');
+    MakeUCA_BmpTables(ucaFirstTable,ucaSecondTable,ucaPropBook);
+    WriteLn('  UCA BMP Second Table Length = ',Length(ucaSecondTable));
+{$IFDEF UCA_TEST}
+    uca_CheckProp_1y(ucaBook,ucaPropBook,@ucaFirstTable,@ucaSecondTable);
+{$ENDIF UCA_TEST}
+
+    WriteLn('Construct UCA OBMP tables ...');
+    MakeUCA_OBmpTables(ucaoFirstTable,ucaoSecondTable,ucaPropBook);
+    WriteLn('  UCA OBMP Second Table Length = ',Length(ucaoSecondTable));
+{$IFDEF UCA_TEST}
+    uca_CheckProp_2y(ucaBook,ucaPropBook,@ucaoFirstTable,@ucaoSecondTable);
+{$ENDIF UCA_TEST}
+    WriteLn('Generate UCA Props tables ...');
+    binStream.Clear();
+    GenerateLicenceText(binStream);
+    GenerateUCA_PropTable(binStream,ucaPropBook);
+    WriteLn('Generate UCA BMP tables ...');
+    stream.Clear();
+    GenerateLicenceText(stream);
+    GenerateUCA_Head(stream,@ucaBook,ucaPropBook);
+    GenerateUCA_BmpTables(stream,binStream,ucaFirstTable,ucaSecondTable,THIS_ENDIAN);
+    WriteLn('Generate UCA OBMP tables ...');
+    GenerateUCA_OBmpTables(stream,binStream,ucaoFirstTable,ucaoSecondTable,THIS_ENDIAN);
+    stream.SaveToFile(outputPath + 'ucadata.inc');
+    s := outputPath + 'ucadata.inc';
+    s := GenerateEndianIncludeFileName(s);
+    binStream.SaveToFile(s);
+    binStream.Clear();
+
+
+    stream.Clear();
+    GenerateLicenceText(stream);
+    WriteLn('File parsed ...', DateTimeToStr(Now));
+    WriteLn('  Props Len = ',Length(props));
+    WriteLn('  Data Len = ',Length(data));
+
+    {WriteLn('BMP Tables building ...', DateTimeToStr(Now));
+    MakeBmpTables(firstTable,secondTable,props,data);
+    WriteLn('   First Table length = ',Length(firstTable));
+    WriteLn('   Second Table length = ',Length(secondTable));}
+
+    WriteLn('BMP Tables building ...', DateTimeToStr(Now));
+    MakeBmpTables3Levels(lvl3table1,lvl3table2,lvl3table3,data);
+    WriteLn(' 3 Levels Tables :');
+    WriteLn('       Len 1 = ',Length(lvl3table1));
+    WriteLn('       Len 2 = ',Length(lvl3table2));
+    WriteLn('       Len 3 = ',Length(lvl3table3));
+    for i := 0 to 255 do begin
+      for k := 0 to 15 do begin
+        for h := 0 to 15 do begin
+          if lvl3table3[lvl3table2[lvl3table1[i]][k]][h] <>
+             GetPropID(256*i + 16*k +h,data)
+          then begin
+            writeln('3 levels errors, i=',i,'; k=',k,'; h=',h);
+          end;
+        end;
+      end;
+    end;
+
+    binStream2.Clear();
+    WriteLn('Source generation ...', DateTimeToStr(Now));
+    WriteLn('BMP Tables sources ...', DateTimeToStr(Now));
+      Generate3lvlBmpTables(stream,lvl3table1,lvl3table2,lvl3table3);
+    WriteLn('Properties Table sources ...', DateTimeToStr(Now));
+      binStream.Clear();
+      GenerateNumericTable(binStream,numericTable,True);
+      binStream.SaveToFile(outputPath + 'unicodenumtable.pas');
+      binStream.Clear();
+      GeneratePropTable(binStream,props,ekLittle);
+      GeneratePropTable(binStream2,props,ekBig);
+//-------------------------------------------
+
+   r := Compress(data);
+
+//-------------------
+    WriteLn('OBMP Tables building ...', DateTimeToStr(Now));
+    MakeOBmpTables3Levels(olvl3table1,olvl3table2,olvl3table3,r);
+    WriteLn(' 3 Levels Tables :');
+    WriteLn('       Len 1 = ',Length(olvl3table1));
+    WriteLn('       Len 2 = ',Length(olvl3table2));
+    WriteLn('       Len 3 = ',Length(olvl3table3));
+    for i := 0 to 1023 do begin
+      for k := 0 to 31 do begin
+        for h := 0 to 31 do begin
+          if olvl3table3[olvl3table2[olvl3table1[i]][k]][h] <>
+             GetPropID(ToUCS4(HIGH_SURROGATE_BEGIN + i,LOW_SURROGATE_BEGIN + (k*32) + h),data)
+          then begin
+            writeln('3, OBMP levels errors, i=',i,'; k=',k,'; h=',h);
+          end;
+        end;
+      end;
+    end;
+    WriteLn('OBMP Tables sources ...', DateTimeToStr(Now));
+    Generate3lvlOBmpTables(stream,olvl3table1,olvl3table2,olvl3table3);
+
+  //---------------------
+    WriteLn('Decomposition  Table sources ...', DateTimeToStr(Now));
+    GenerateDecompositionBookTable(binStream,decompositionBook,ekLittle);
+    GenerateDecompositionBookTable(binStream2,decompositionBook,ekBig);
+    stream.SaveToFile(outputPath + 'unicodedata.inc');
+    binStream.SaveToFile(outputPath + 'unicodedata_le.inc');
+    binStream2.SaveToFile(outputPath + 'unicodedata_be.inc');
+    binStream.Clear();
+    binStream2.Clear();
+
+
+    h := -1;
+    for i := Low(data) to High(data) do
+      if (data[i].CodePoint > $FFFF) then begin
+        h := i;
+        Break;
+      end;
+    stream.Clear();
+    for i := h to High(data) do begin
+      p := @data[i];
+      if (p^.LineType = 0) then begin
+        FromUCS4(p^.CodePoint,hs,ls);
+        //k := GetProp(hs,ls,props,ofirstTable,osecondTable)^.PropID;
+        k := GetProp(
+               (hs-HIGH_SURROGATE_BEGIN),(ls-LOW_SURROGATE_BEGIN),
+               props,olvl3table1,olvl3table2,olvl3table3
+             )^.PropID;
+        if (p^.PropID <> k) then begin
+          s := Format('#%d-%d  #%d',[p^.CodePoint,p^.PropID,k]) + sLineBreak;
+          stream.Write(s[1],Length(s));
+        end;
+      end else begin
+        for h := p^.StartCodePoint to p^.EndCodePoint do begin
+          FromUCS4(h,hs,ls);
+          //k := GetProp(hs,ls,props,ofirstTable,osecondTable)^.PropID;
+          k := GetProp(
+                 (hs-HIGH_SURROGATE_BEGIN),(ls-LOW_SURROGATE_BEGIN),
+                 props,olvl3table1,olvl3table2,olvl3table3
+               )^.PropID;
+          if (p^.PropID <> k) then begin
+            s := Format('##%d;%d-%d  #%d',[p^.StartCodePoint,p^.EndCodePoint,p^.PropID,k]) + sLineBreak;
+            stream.Write(s[1],Length(s));
+            Break
+          end;
+        end;
+      end;
+    end;
+    stream.SaveToFile(outputPath + 'diff-obmp.txt');
+
+    stream.Clear();
+    for i := Low(data) to High(data) do begin
+      p := @data[i];
+      if (p^.LineType = 0) then begin
+        k := GetPropID(p^.CodePoint,r);
+        if (p^.PropID <> k) then begin
+          s := Format('#%d-%d  #%d',[p^.CodePoint,p^.PropID,k]) + sLineBreak;
+          stream.Write(s[1],Length(s));
+        end;
+      end else begin
+        for h := p^.StartCodePoint to p^.EndCodePoint do begin
+          k := GetPropID(h,r);
+          if (p^.PropID <> k) then begin
+            s := Format('##%d;%d-%d  #%d',[p^.StartCodePoint,p^.EndCodePoint,p^.PropID,k]) + sLineBreak;
+            stream.Write(s[1],Length(s));
+            Break
+          end;
+        end;
+      end;
+    end;
+    stream.SaveToFile(outputPath + 'diff.txt');
+    stream.Clear();
+    for i := Low(r) to High(r) do begin
+      p := @r[i];
+      if (p^.LineType = 0) then begin
+        k := GetPropID(p^.CodePoint,data);
+        if (p^.PropID <> k) then begin
+          s := Format('#%d-%d  #%d',[p^.CodePoint,p^.PropID,k]) + sLineBreak;
+          stream.Write(s[1],Length(s));
+        end;
+      end else begin
+        for h := p^.StartCodePoint to p^.EndCodePoint do begin
+          k := GetPropID(h,r);
+          if (p^.PropID <> k) then begin
+            s := Format('##%d;%d-%d  #%d',[p^.StartCodePoint,p^.EndCodePoint,p^.PropID,k]) + sLineBreak;
+            stream.Write(s[1],Length(s));
+            Break
+          end;
+        end;
+      end;
+    end;
+    stream.SaveToFile(outputPath + 'diff2.txt');
+  finally
+    binStream2.Free();
+    binStream.Free();
+    stream.Free();
+  end;
+end.
+

utils/unicode/weight_derivation.inc

@@ -0,0 +1,66 @@
+
+function IsCJK_Unified_Ideographs(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $4E00) and (ACodePoint <= $9FCC); // $9FFF
+end;
+
+function IsCJK_Compatibility_Ideographs(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $F900) and (ACodePoint <= $FAFF);
+end;
+
+function IsCJK_Unified_Ideographs_Extension_A(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $3400) and (ACodePoint <= $4DB5);  // $4DBF
+end;
+
+function IsCJK_Unified_Ideographs_Extension_B(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $20000) and (ACodePoint <= $2A6D6); // $2A6DF
+end;
+
+function IsCJK_Unified_Ideographs_Extension_C(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $2A700) and (ACodePoint <= $2B734); // $2B73F
+end;
+
+function IsCJK_Unified_Ideographs_Extension_D(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $2B740) and (ACodePoint <= $2B81D); // $2B81F
+end;
+
+function IsCJK_Compatibility_Ideographs_Supplement(ACodePoint : Cardinal) : Boolean;inline;
+begin
+  Result := (ACodePoint >= $2F800) and (ACodePoint <= $2FA1F);
+end;
+
+procedure DeriveWeight(const ACodePoint : Cardinal; AResult : PUCA_PropWeights);
+const
+  BASE_1 = Word($FB40);
+  BASE_2 = Word($FB80);
+  BASE_3 = Word($FBC0);
+var
+  base : Word;
+begin
+  if IsCJK_Unified_Ideographs(ACodePoint) or IsCJK_Compatibility_Ideographs(ACodePoint) then
+    base := BASE_1
+  else if IsCJK_Unified_Ideographs_Extension_A(ACodePoint) or
+          IsCJK_Unified_Ideographs_Extension_B(ACodePoint) or
+          IsCJK_Unified_Ideographs_Extension_C(ACodePoint) or
+          IsCJK_Unified_Ideographs_Extension_D(ACodePoint) or
+          IsCJK_Compatibility_Ideographs_Supplement(ACodePoint)
+  then begin
+    base := BASE_2;
+  end else begin
+    base := BASE_3;
+  end;
+
+
+  AResult[0].Weights[0] := base + (ACodePoint shr 15);
+  AResult[0].Weights[1] := $20;
+  AResult[0].Weights[2] := $2;
+
+  AResult[1].Weights[0] := (ACodePoint and $7FFF) or $8000;
+  AResult[1].Weights[1] := 0;
+  AResult[1].Weights[2] := 0;
+end;