cldrparser.lpr 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321
  1. { Unicode CLDR's collation parser.
  2. Copyright (c) 2013-2017 by Inoussa OUEDRAOGO
  3. It creates units from CLDR's collation files.
  4. The source code is distributed under the Library GNU
  5. General Public License with the following modification:
  6. - object files and libraries linked into an application may be
  7. distributed without source code.
  8. If you didn't receive a copy of the file COPYING, contact:
  9. Free Software Foundation
  10. 675 Mass Ave
  11. Cambridge, MA 02139
  12. USA
  13. This program is distributed in the hope that it will be useful,
  14. but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  16. }
  17. program cldrparser;
  18. {$mode objfpc}{$H+}
  19. { $define WINCE_TEST}
  20. {$TYPEDADDRESS ON}
  21. uses //heaptrc,
  22. SysUtils, classes, getopts,{$ifdef WINCE}StreamIO,{$endif}
  23. cldrhelper, helper, cldrtest, cldrxml, unicodeset, cldrtxt;
  24. const
  25. SROOT_RULES_FILE = 'UCA_Rules_SHORT.txt';
  26. SUsageText =
  27. 'This program creates pascal units from CLDR''s collation files for usage ' + sLineBreak +
  28. 'with the FreePascal Native Unicode Manager.' + sLineBreak + sLineBreak +
  29. 'Usage : cldrparser <collationFileName> [<typeName>] [-a<alt>] [-d<dataDir>] [-o<outputDir>] [-t<HaltOnFail>]' + sLineBreak + sLineBreak +
  30. ' where :' + sLineBreak +
  31. ' ' + sLineBreak +
  32. ' - collationFileName : specify the target file.' + sLineBreak +
  33. ' - typeName : optional, specify the collation'' type-name to be parse;' + sLineBreak +
  34. ' If this argument is not supplied, a default type-name' + sLineBreak +
  35. ' is chosen from the type-name list in the following order : ' + sLineBreak +
  36. ' * the "default" specified type-name indicated by the collation' + sLineBreak +
  37. ' * the type named "standard" ' + sLineBreak +
  38. ' * the type named "search" ' + sLineBreak +
  39. ' * the first type.' + sLineBreak +
  40. ' - a : this provides the "alt" property to select specific "type".' + sLineBreak +
  41. ' - dataDir : specify the directory that contains the collation files.' + sLineBreak +
  42. ' The default value is the program''s directory.' + sLineBreak +
  43. ' - outputDir : specify the directory where the generated files will be stored.' + sLineBreak +
  44. ' The default value is the program''s directory.' + sLineBreak +
  45. ' - t : to execute parser the test suite. The program will execute only the test suite and exit.' + sLineBreak +
  46. ' <HaltOnFail> may be one of (y, Y, t, T, 1) to halt the execution on the first failing.' + sLineBreak +
  47. ' ' + sLineBreak +
  48. ' The program expects some files to be present in the <dataDir> folder : ' + sLineBreak +
  49. ' - UCA_Rules_SHORT.xml ' + sLineBreak +
  50. ' - allkeys.txt this is the file allkeys_CLDR.txt renamed to allkeys.txt' + sLineBreak +
  51. ' These files are in the core.zip file of the CLDR release files. The CLDR''version used should be synchronized the' + sLineBreak +
  52. ' version of the Unicode version used, for example for Uniocde 7 it will be CLDR 26.' + sLineBreak +
  53. ' The CLDR files are provided by the Unicode Consortium at http://cldr.unicode.org/index/downloads';
  54. function ParseOptions(
  55. var ADataDir,
  56. AOuputDir,
  57. ACollationFileName,
  58. ACollationTypeName,
  59. ACollationTypeAlt : string;
  60. var AExecTestSuite,
  61. ATestHaltOnFail : Boolean
  62. ) : Boolean;
  63. var
  64. c : Char;
  65. idx, k : Integer;
  66. s : string;
  67. begin
  68. {$ifdef WINCE_TEST}
  69. ADataDir := ExtractFilePath(ParamStr(0))+'data';
  70. AOuputDir := ADataDir;
  71. ACollationFileName := 'sv.xml';
  72. exit(True);
  73. {$endif WINCE_TEST}
  74. if (ParamCount() = 0) then
  75. exit(False);
  76. Result := True;
  77. AExecTestSuite := False;
  78. repeat
  79. c := GetOpt('a:d:o:ht:');
  80. case c of
  81. 'a' : ACollationTypeAlt := Trim(OptArg);
  82. 'd' : ADataDir := ExpandFileName(Trim(OptArg));
  83. 'o' : AOuputDir := ExpandFileName(Trim(OptArg));
  84. 'h', '?' :
  85. begin
  86. WriteLn(SUsageText);
  87. Result := False;
  88. end;
  89. 't' :
  90. begin
  91. AExecTestSuite := True;
  92. s := Trim(OptArg);
  93. ATestHaltOnFail := (s <> '') and CharInSet(s[1],['y','Y','t','T','1']);
  94. end;
  95. end;
  96. until (c = EndOfOptions);
  97. idx := 0;
  98. for k := 1 to ParamCount() do begin
  99. s := Trim(ParamStr(k));
  100. if (s <> '') and (s[1] <> '-') then begin
  101. if (idx = 0) then
  102. ACollationFileName := s
  103. else if (idx = 1) then
  104. ACollationTypeName := s;
  105. Inc(idx);
  106. if (idx >= 2) then
  107. Break;
  108. end;
  109. end;
  110. end;
  111. procedure Main;
  112. var
  113. orderedChars : TOrderedCharacters;
  114. settings : TSettingRecArray;
  115. ucaBook : TUCA_DataBook;
  116. stream, streamNE, streamOE, binaryStreamNE, binaryStreamOE : TMemoryStream;
  117. s, collationFileName, collationTypeName, collationTypeAlt : string;
  118. i , c: Integer;
  119. repository : TCldrCollationRepository;
  120. collation : TCldrCollation;
  121. dataPath, outputPath : string;
  122. collationItem : TCldrCollationItem;
  123. testSuiteFlag, testSuiteHaltOnFailFlag : Boolean;
  124. {$ifdef WINCE}
  125. fs : TFileStream;
  126. {$endif WINCE}
  127. begin
  128. {$ifdef WINCE}
  129. s := ExtractFilePath(ParamStr(0))+'cldr-log.txt';
  130. DeleteFile(s);
  131. fs := TFileStream.Create(s,fmCreate);
  132. AssignStream(Output,fs);
  133. Rewrite(Output);
  134. s := ExtractFilePath(ParamStr(0))+'cldr-err.txt';
  135. DeleteFile(s);
  136. fs := TFileStream.Create(s,fmCreate);
  137. AssignStream(ErrOutput,fs);
  138. Rewrite(ErrOutput);
  139. {$endif WINCE}
  140. {$ifdef WINCE_TEST}
  141. testSuiteFlag := True;
  142. try
  143. exec_tests();
  144. except
  145. on e : Exception do begin
  146. WriteLn('Exception : '+e.Message);
  147. raise;
  148. end;
  149. end;
  150. exit;
  151. {$endif WINCE_TEST}
  152. dataPath := '';
  153. outputPath := '';
  154. collationFileName := '';
  155. collationTypeName := '';
  156. collationTypeAlt := '';
  157. testSuiteFlag := False;
  158. testSuiteHaltOnFailFlag := True;
  159. if not ParseOptions(
  160. dataPath,outputPath,collationFileName,collationTypeName,
  161. collationTypeAlt,testSuiteFlag,testSuiteHaltOnFailFlag
  162. )
  163. then begin
  164. WriteLn(SUsageText);
  165. Halt(1);
  166. end;
  167. if testSuiteFlag then begin
  168. WriteLn('Executing the test suite ...');
  169. exec_tests(testSuiteHaltOnFailFlag);
  170. Halt;
  171. end;
  172. if (dataPath <> '') and not(DirectoryExists(dataPath)) then begin
  173. WriteLn('This directory does not exist : ',dataPath);
  174. Halt(1);
  175. end;
  176. if (dataPath = '') then
  177. dataPath := ExtractFilePath(ParamStr(0))
  178. else
  179. dataPath := IncludeTrailingPathDelimiter(dataPath);
  180. if (outputPath = '') then
  181. outputPath := dataPath
  182. else
  183. outputPath := IncludeTrailingPathDelimiter(outputPath);
  184. {$ifndef WINCE_TEST}
  185. if (ParamCount() = 0) then begin
  186. WriteLn(SUsageText);
  187. Halt(1);
  188. end;
  189. {$endif WINCE_TEST}
  190. if not(
  191. FileExists(dataPath+SROOT_RULES_FILE) and
  192. FileExists(dataPath+'allkeys.txt')
  193. )
  194. then begin
  195. WriteLn(Format('File not found : %s or %s.',[dataPath+SROOT_RULES_FILE,dataPath+'allkeys.txt']));
  196. Halt(1);
  197. end;
  198. {collationFileName := dataPath + collationFileName;
  199. if not FileExists(collationFileName) then begin
  200. WriteLn('File not found : "',collationFileName,'"');
  201. Halt(1);
  202. end;}
  203. WriteLn(sLineBreak,'Collation Parsing ',QuotedStr(collationFileName),' ...');
  204. stream := nil;
  205. streamNE := nil;
  206. streamOE := nil;
  207. binaryStreamNE := nil;
  208. binaryStreamOE := nil;
  209. repository := TCldrCollationRepository.Create(
  210. TCldrCollationFileLoader.Create(dataPath) as ICldrCollationLoader
  211. );
  212. try
  213. collation := repository.Load(collationFileName,TCldrParserMode.HeaderParsing);
  214. WriteLn(Format(' Collation Count = %d',[collation.FindPublicItemCount()]));
  215. if (collation.FindPublicItemCount() = 0) then begin
  216. WriteLn('No collation in this file.');
  217. end else begin
  218. for i := 0 to collation.ItemCount - 1 do begin
  219. if not collation.Items[i].IsPrivate() then begin
  220. s := collation.Items[i].TypeName;
  221. if (collation.Items[i].Alt <> '') then
  222. s := s + ', Alt = ' + collation.Items[i].Alt;
  223. WriteLn(Format(' Item[%d] = (Type = %s)',[i,s]));
  224. end;
  225. end;
  226. if (collationTypeAlt = '') then
  227. collationItem := collation.Find(collationTypeName)
  228. else
  229. collationItem := collation.Find(collationTypeName,collationTypeAlt);
  230. if (collationItem = nil) then begin
  231. collationTypeName := FindCollationDefaultItemName(collation);
  232. collationItem := collation.Find(collationTypeName);
  233. collationTypeAlt := collationItem.Alt;
  234. end;
  235. s := collationTypeName;
  236. if (collationTypeAlt <> '') then
  237. s := Format('%s (%s)',[s,collationTypeAlt]);
  238. WriteLn(Format('Parsing Collation Item "%s" ...',[s]));
  239. collationItem := repository.LoadType(collationFileName,collationTypeName,collationTypeAlt);
  240. s := dataPath + SROOT_RULES_FILE;
  241. WriteLn;
  242. WriteLn('Parsing ',QuotedStr(s),' ...');
  243. FillByte(orderedChars,SizeOf(orderedChars),0);
  244. orderedChars.Clear();
  245. SetLength(settings,0);
  246. ParseInitialDocument(@orderedChars,s,settings);
  247. WriteLn('File parsed, ',orderedChars.ActualLength,' characters.');
  248. WriteLn('Loading CLDR root''s key table ...');
  249. stream := TMemoryStream.Create();
  250. s := dataPath + 'allkeys.txt';
  251. stream.LoadFromFile(s);
  252. FillChar(ucaBook,SizeOf(ucaBook),#0);
  253. ParseUCAFile(stream,ucaBook);
  254. //WriteLn(' LEVEL-2''s items Value = ',CalcMaxLevel2Value(ucaBook.Lines));
  255. //RewriteLevel2Values(@ucaBook.Lines[0],Length(ucaBook.Lines));
  256. //WriteLn(' LEVEL-2''s items Value (after rewrite) = ',CalcMaxLevel2Value(ucaBook.Lines));
  257. c := FillInitialPositions(@orderedChars.Data[0],orderedChars.ActualLength,ucaBook.Lines);
  258. if (c > 0) then
  259. WriteLn(' Missed Initial Positions = ',c);
  260. WriteLn(' Loaded.');
  261. WriteLn('Start generation ...');
  262. stream.Clear();
  263. streamNE := TMemoryStream.Create();
  264. streamOE := TMemoryStream.Create();
  265. binaryStreamNE := TMemoryStream.Create();
  266. binaryStreamOE := TMemoryStream.Create();
  267. s := COLLATION_FILE_PREFIX + ChangeFileExt(LowerCase(ExtractFileName(collationFileName)),'.pas');
  268. GenerateCdlrCollation(
  269. collation,collationTypeName,s,stream,streamNE,streamOE,
  270. binaryStreamNE,binaryStreamOE,
  271. orderedChars,ucaBook.Lines
  272. );
  273. stream.SaveToFile(outputPath+s);
  274. if (streamNE.Size > 0) then begin
  275. streamNE.SaveToFile(outputPath+GenerateEndianIncludeFileName(s,ENDIAN_NATIVE));
  276. streamOE.SaveToFile(outputPath+GenerateEndianIncludeFileName(s,ENDIAN_NON_NATIVE));
  277. end;
  278. if (binaryStreamNE.Size > 0) then begin
  279. binaryStreamNE.SaveToFile(
  280. outputPath +
  281. ChangeFileExt(s,Format('_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NATIVE]]))
  282. );
  283. binaryStreamOE.SaveToFile(
  284. outputPath +
  285. ChangeFileExt(s,Format('_%s.bco',[ENDIAN_SUFFIX[ENDIAN_NON_NATIVE]]))
  286. );
  287. end;
  288. end;
  289. repository.Clear();
  290. finally
  291. binaryStreamOE.Free();
  292. binaryStreamNE.Free();
  293. streamOE.Free();
  294. streamNE.Free();
  295. stream.Free();
  296. repository.Free();
  297. end;
  298. WriteLn(sLineBreak,'Finished.');
  299. end;
  300. begin
  301. Main();
  302. end.