wstrings.inc 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692
  1. {
  2. This file is part of the Free Pascal run time library.
  3. Copyright (c) 1999-2005 by Florian Klaempfl,
  4. member of the Free Pascal development team.
  5. This file implements support routines for WideStrings with FPC
  6. See the file COPYING.FPC, included in this distribution,
  7. for details about the copyright.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11. **********************************************************************}
  12. {
  13. This file contains the implementation of the WideString type,
  14. and all things that are needed for it.
  15. WideString is defined as a 'silent' pwidechar :
  16. a pwidechar that points to :
  17. @-8 : SizeInt for reference count;
  18. @-4 : SizeInt for size; size=number of bytes, not the number of chars. Divide or multiply
  19. with sizeof(WideChar) to convert. This is needed to be compatible with Delphi and
  20. Windows COM BSTR.
  21. @ : String + Terminating #0;
  22. Pwidechar(Widestring) is a valid typecast.
  23. So WS[i] is converted to the address @WS+i-1.
  24. Constants should be assigned a reference count of -1
  25. Meaning that they can't be disposed of.
  26. }
  27. Type
  28. PWideRec = ^TWideRec;
  29. TWideRec = Packed Record
  30. Len : DWord;
  31. First : WideChar;
  32. end;
  33. Const
  34. WideRecLen = SizeOf(TWideRec);
  35. WideFirstOff = SizeOf(TWideRec)-sizeof(WideChar);
  36. {
  37. Default WideChar <-> Char conversion is to only convert the
  38. lower 127 chars, all others are translated to spaces.
  39. These routines can be overwritten for the Current Locale
  40. }
  41. procedure DefaultWide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
  42. var
  43. i : SizeInt;
  44. destp: PChar;
  45. begin
  46. setlength(dest,len);
  47. destp := PChar(Pointer(dest));
  48. for i:=1 to len do
  49. begin
  50. if word(source^)<256 then
  51. destp^:=char(word(source^))
  52. else
  53. destp^:='?';
  54. inc(source);
  55. inc(destp);
  56. end;
  57. end;
  58. procedure DefaultAnsi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
  59. var
  60. i : SizeInt;
  61. begin
  62. setlength(dest,len);
  63. for i:=1 to len do
  64. begin
  65. dest[i]:=widechar(byte(source^));
  66. inc(source);
  67. end;
  68. end;
  69. (*
  70. Procedure UniqueWideString(Var S : WideString); [Public,Alias : 'FPC_WIDESTR_UNIQUE'];
  71. {
  72. Make sure reference count of S is 1,
  73. using copy-on-write semantics.
  74. }
  75. begin
  76. end;
  77. *)
  78. {****************************************************************************
  79. Internal functions, not in interface.
  80. ****************************************************************************}
  81. procedure WideStringError;
  82. begin
  83. HandleErrorFrame(204,get_frame);
  84. end;
  85. {$ifdef WideStrDebug}
  86. Procedure DumpWideRec(S : Pointer);
  87. begin
  88. If S=Nil then
  89. Writeln ('String is nil')
  90. Else
  91. Begin
  92. With PWideRec(S-WideFirstOff)^ do
  93. begin
  94. Write ('(Len:',len);
  95. Writeln (' Ref: ',ref,')');
  96. end;
  97. end;
  98. end;
  99. {$endif}
  100. Function NewWideString(Len : SizeInt) : Pointer;
  101. {
  102. Allocate a new WideString on the heap.
  103. initialize it to zero length and reference count 1.
  104. }
  105. Var
  106. P : Pointer;
  107. begin
  108. {$ifdef MSWINDOWS}
  109. if winwidestringalloc then
  110. begin
  111. P:=SysAllocStringLen(nil,Len);
  112. if P=nil then
  113. WideStringError;
  114. end
  115. else
  116. {$endif MSWINDOWS}
  117. begin
  118. GetMem(P,Len*sizeof(WideChar)+WideRecLen);
  119. If P<>Nil then
  120. begin
  121. PWideRec(P)^.Len:=Len*2; { Initial length }
  122. PWideRec(P)^.First:=#0; { Terminating #0 }
  123. inc(p,WideFirstOff); { Points to string now }
  124. end
  125. else
  126. WideStringError;
  127. end;
  128. NewWideString:=P;
  129. end;
  130. Procedure DisposeWideString(Var S : Pointer);
  131. {
  132. Deallocates a WideString From the heap.
  133. }
  134. begin
  135. If S=Nil then
  136. exit;
  137. {$ifndef MSWINDOWS}
  138. Dec (S,WideFirstOff);
  139. Freemem(S);
  140. {$else MSWINDOWS}
  141. if winwidestringalloc then
  142. SysFreeString(S)
  143. else
  144. begin
  145. Dec (S,WideFirstOff);
  146. Freemem(S);
  147. end;
  148. {$endif MSWINDOWS}
  149. S:=Nil;
  150. end;
  151. Procedure fpc_WideStr_Decr_Ref (Var S : Pointer);[Public,Alias:'FPC_WIDESTR_DECR_REF']; compilerproc;
  152. {
  153. Decreases the ReferenceCount of a non constant widestring;
  154. If the reference count is zero, deallocate the string;
  155. }
  156. Begin
  157. DisposeWideString(S); { does test for nil }
  158. end;
  159. { alias for internal use }
  160. Procedure fpc_WideStr_Decr_Ref (Var S : Pointer);[external name 'FPC_WIDESTR_DECR_REF'];
  161. Procedure fpc_WideStr_Incr_Ref(Var S : Pointer);[Public,Alias:'FPC_WIDESTR_INCR_REF']; compilerproc;
  162. var
  163. p : pointer;
  164. Begin
  165. If S=Nil then
  166. exit;
  167. p:=NewWidestring(length(WideString(S)));
  168. move(s^,p^,(length(WideString(s))+1)*sizeof(widechar)); // double #0 too
  169. s:=p;
  170. end;
  171. { alias for internal use }
  172. Procedure fpc_WideStr_Incr_Ref (Var S : Pointer);[external name 'FPC_WIDESTR_INCR_REF'];
  173. {$ifndef FPC_STRTOSHORTSTRINGPROC}
  174. function fpc_WideStr_To_ShortStr (high_of_res: SizeInt;const S2 : WideString): shortstring;[Public, alias: 'FPC_WIDESTR_TO_SHORTSTR']; compilerproc;
  175. {
  176. Converts a WideString to a ShortString;
  177. }
  178. Var
  179. Size : SizeInt;
  180. temp : ansistring;
  181. begin
  182. result:='';
  183. Size:=Length(S2);
  184. if Size>0 then
  185. begin
  186. If Size>high_of_res then
  187. Size:=high_of_res;
  188. widestringmanager.Wide2AnsiMoveProc(PWideChar(S2),temp,Size);
  189. result:=temp;
  190. end;
  191. end;
  192. {$else FPC_STRTOSHORTSTRINGPROC}
  193. procedure fpc_WideStr_To_ShortStr (out res: ShortString;const S2 : WideString); [Public, alias: 'FPC_WIDESTR_TO_SHORTSTR'];compilerproc;
  194. {
  195. Converts a WideString to a ShortString;
  196. }
  197. Var
  198. Size : SizeInt;
  199. temp : ansistring;
  200. begin
  201. res:='';
  202. Size:=Length(S2);
  203. if Size>0 then
  204. begin
  205. If Size>high(res) then
  206. Size:=high(res);
  207. widestringmanager.Wide2AnsiMoveProc(PWideChar(S2),temp,Size);
  208. res:=temp;
  209. end;
  210. end;
  211. {$endif FPC_STRTOSHORTSTRINGPROC}
  212. Function fpc_ShortStr_To_WideStr (Const S2 : ShortString): WideString;compilerproc;
  213. {
  214. Converts a ShortString to a WideString;
  215. }
  216. Var
  217. Size : SizeInt;
  218. begin
  219. result:='';
  220. Size:=Length(S2);
  221. if Size>0 then
  222. begin
  223. widestringmanager.Ansi2WideMoveProc(PChar(@S2[1]),result,Size);
  224. { Terminating Zero }
  225. PWideChar(Pointer(fpc_ShortStr_To_WideStr)+Size*sizeof(WideChar))^:=#0;
  226. end;
  227. end;
  228. Function fpc_WideStr_To_AnsiStr (const S2 : WideString): AnsiString; compilerproc;
  229. {
  230. Converts a WideString to an AnsiString
  231. }
  232. Var
  233. Size : SizeInt;
  234. begin
  235. result:='';
  236. Size:=Length(S2);
  237. if Size>0 then
  238. widestringmanager.Wide2AnsiMoveProc(PWideChar(Pointer(S2)),result,Size);
  239. end;
  240. Function fpc_AnsiStr_To_WideStr (Const S2 : AnsiString): WideString; compilerproc;
  241. {
  242. Converts an AnsiString to a WideString;
  243. }
  244. Var
  245. Size : SizeInt;
  246. begin
  247. result:='';
  248. Size:=Length(S2);
  249. if Size>0 then
  250. widestringmanager.Ansi2WideMoveProc(PChar(S2),result,Size);
  251. end;
  252. Function fpc_PWideChar_To_WideStr(const p : pwidechar): widestring; compilerproc;
  253. var
  254. Size : SizeInt;
  255. begin
  256. result:='';
  257. if p=nil then
  258. exit;
  259. Size := IndexWord(p^, -1, 0);
  260. Setlength(result,Size);
  261. if Size>0 then
  262. begin
  263. Move(p^,PWideChar(Pointer(result))^,Size*sizeof(WideChar));
  264. { Terminating Zero }
  265. PWideChar(Pointer(result)+Size*sizeof(WideChar))^:=#0;
  266. end;
  267. end;
  268. { checked against the ansistring routine, 2001-05-27 (FK) }
  269. Procedure fpc_WideStr_Assign (Var S1 : Pointer;S2 : Pointer);[Public,Alias:'FPC_WIDESTR_ASSIGN']; compilerproc;
  270. {
  271. Assigns S2 to S1 (S1:=S2), taking in account reference counts.
  272. }
  273. begin
  274. if S1=S2 then exit;
  275. if S2<>nil then
  276. begin
  277. {$ifdef MSWINDOWS}
  278. if winwidestringalloc then
  279. begin
  280. if SysReAllocStringLen(S1, S2, Length(WideString(S2))) = 0 then
  281. WideStringError;
  282. end
  283. else
  284. {$endif MSWINDOWS}
  285. begin
  286. SetLength(WideString(S1),length(WideString(S2)));
  287. move(s2^,s1^,(length(WideString(s1))+1)*sizeof(widechar));
  288. end;
  289. end
  290. else
  291. begin
  292. { Free S1 }
  293. fpc_widestr_decr_ref (S1);
  294. S1:=nil;
  295. end;
  296. end;
  297. { alias for internal use }
  298. Procedure fpc_WideStr_Assign (Var S1 : Pointer;S2 : Pointer);[external name 'FPC_WIDESTR_ASSIGN'];
  299. {$ifndef STR_CONCAT_PROCS}
  300. function fpc_WideStr_Concat (const S1,S2 : WideString): WideString; compilerproc;
  301. Var
  302. Size,Location : SizeInt;
  303. pc : pwidechar;
  304. begin
  305. { only assign if s1 or s2 is empty }
  306. if (S1='') then
  307. begin
  308. result:=s2;
  309. exit;
  310. end;
  311. if (S2='') then
  312. begin
  313. result:=s1;
  314. exit;
  315. end;
  316. Location:=Length(S1);
  317. Size:=length(S2);
  318. SetLength(result,Size+Location);
  319. pc:=pwidechar(result);
  320. Move(S1[1],pc^,Location*sizeof(WideChar));
  321. inc(pc,location);
  322. Move(S2[1],pc^,(Size+1)*sizeof(WideChar));
  323. end;
  324. function fpc_WideStr_Concat_multi (const sarr:array of Widestring): widestring; compilerproc;
  325. Var
  326. i : Longint;
  327. p : pointer;
  328. pc : pwidechar;
  329. Size,NewSize : SizeInt;
  330. begin
  331. { First calculate size of the result so we can do
  332. a single call to SetLength() }
  333. NewSize:=0;
  334. for i:=low(sarr) to high(sarr) do
  335. inc(Newsize,length(sarr[i]));
  336. SetLength(result,NewSize);
  337. pc:=pwidechar(result);
  338. for i:=low(sarr) to high(sarr) do
  339. begin
  340. p:=pointer(sarr[i]);
  341. if assigned(p) then
  342. begin
  343. Size:=length(widestring(p));
  344. Move(pwidechar(p)^,pc^,(Size+1)*sizeof(WideChar));
  345. inc(pc,size);
  346. end;
  347. end;
  348. end;
  349. {$else STR_CONCAT_PROCS}
  350. procedure fpc_WideStr_Concat (var DestS:Widestring;const S1,S2 : WideString); compilerproc;
  351. Var
  352. Size,Location : SizeInt;
  353. same : boolean;
  354. begin
  355. { only assign if s1 or s2 is empty }
  356. if (S1='') then
  357. begin
  358. DestS:=s2;
  359. exit;
  360. end;
  361. if (S2='') then
  362. begin
  363. DestS:=s1;
  364. exit;
  365. end;
  366. Location:=Length(S1);
  367. Size:=length(S2);
  368. { Use Pointer() typecasts to prevent extra conversion code }
  369. if Pointer(DestS)=Pointer(S1) then
  370. begin
  371. same:=Pointer(S1)=Pointer(S2);
  372. SetLength(DestS,Size+Location);
  373. if same then
  374. Move(Pointer(DestS)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size)*sizeof(WideChar))
  375. else
  376. Move(Pointer(S2)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  377. end
  378. else if Pointer(DestS)=Pointer(S2) then
  379. begin
  380. SetLength(DestS,Size+Location);
  381. Move(Pointer(DestS)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  382. Move(Pointer(S1)^,Pointer(DestS)^,Location*sizeof(WideChar));
  383. end
  384. else
  385. begin
  386. DestS:='';
  387. SetLength(DestS,Size+Location);
  388. Move(Pointer(S1)^,Pointer(DestS)^,Location*sizeof(WideChar));
  389. Move(Pointer(S2)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  390. end;
  391. end;
  392. procedure fpc_WideStr_Concat_multi (var DestS:Widestring;const sarr:array of Widestring); compilerproc;
  393. Var
  394. i : Longint;
  395. p,pc : pointer;
  396. Size,NewLen : SizeInt;
  397. DestTmp : Widestring;
  398. begin
  399. if high(sarr)=0 then
  400. begin
  401. DestS:='';
  402. exit;
  403. end;
  404. { First calculate size of the result so we can do
  405. a single call to SetLength() }
  406. NewLen:=0;
  407. for i:=low(sarr) to high(sarr) do
  408. inc(NewLen,length(sarr[i]));
  409. SetLength(DestTmp,NewLen);
  410. pc:=pwidechar(DestTmp);
  411. for i:=low(sarr) to high(sarr) do
  412. begin
  413. p:=pointer(sarr[i]);
  414. if assigned(p) then
  415. begin
  416. Size:=length(widestring(p));
  417. Move(p^,pc^,(Size+1)*sizeof(WideChar));
  418. inc(pc,size*sizeof(WideChar));
  419. end;
  420. end;
  421. DestS:=DestTmp;
  422. end;
  423. {$endif STR_CONCAT_PROCS}
  424. Function fpc_Char_To_WideStr(const c : Char): WideString; compilerproc;
  425. {
  426. Converts a Char to a WideString;
  427. }
  428. begin
  429. Setlength(fpc_Char_To_WideStr,1);
  430. fpc_Char_To_WideStr[1]:=c;
  431. { Terminating Zero }
  432. PWideChar(Pointer(fpc_Char_To_WideStr)+sizeof(WideChar))^:=#0;
  433. end;
  434. Function fpc_WChar_To_WideStr(const c : WideChar): WideString; compilerproc;
  435. {
  436. Converts a WideChar to a WideString;
  437. }
  438. begin
  439. Setlength (fpc_WChar_To_WideStr,1);
  440. fpc_WChar_To_WideStr[1]:= c;
  441. end;
  442. Function fpc_WChar_To_AnsiStr(const c : WideChar): AnsiString; compilerproc;
  443. {
  444. Converts a WideChar to a AnsiString;
  445. }
  446. begin
  447. widestringmanager.Wide2AnsiMoveProc(@c, fpc_WChar_To_AnsiStr, 1);
  448. end;
  449. Function fpc_UChar_To_WideStr(const c : WideChar): WideString; compilerproc;
  450. {
  451. Converts a WideChar to a WideString;
  452. }
  453. begin
  454. Setlength (fpc_UChar_To_WideStr,1);
  455. fpc_UChar_To_WideStr[1]:= c;
  456. end;
  457. Function fpc_PChar_To_WideStr(const p : pchar): WideString; compilerproc;
  458. Var
  459. L : SizeInt;
  460. begin
  461. if (not assigned(p)) or (p[0]=#0) Then
  462. begin
  463. fpc_pchar_to_widestr := '';
  464. exit;
  465. end;
  466. l:=IndexChar(p^,-1,#0);
  467. widestringmanager.Ansi2WideMoveProc(P,fpc_PChar_To_WideStr,l);
  468. end;
  469. Function fpc_CharArray_To_WideStr(const arr: array of char; zerobased: boolean = true): WideString; compilerproc;
  470. var
  471. i : SizeInt;
  472. begin
  473. if (zerobased) then
  474. begin
  475. if (arr[0]=#0) Then
  476. begin
  477. fpc_chararray_to_widestr := '';
  478. exit;
  479. end;
  480. i:=IndexChar(arr,high(arr)+1,#0);
  481. if i = -1 then
  482. i := high(arr)+1;
  483. end
  484. else
  485. i := high(arr)+1;
  486. SetLength(fpc_CharArray_To_WideStr,i);
  487. widestringmanager.Ansi2WideMoveProc (pchar(@arr),fpc_CharArray_To_WideStr,i);
  488. end;
  489. {$ifndef FPC_STRTOCHARARRAYPROC}
  490. { inside the compiler, the resulttype is modified to that of the actual }
  491. { chararray we're converting to (JM) }
  492. function fpc_widestr_to_chararray(arraysize: SizeInt; const src: WideString): fpc_big_chararray;[public,alias: 'FPC_WIDESTR_TO_CHARARRAY']; compilerproc;
  493. var
  494. len: SizeInt;
  495. temp: ansistring;
  496. begin
  497. len := length(src);
  498. { make sure we don't dereference src if it can be nil (JM) }
  499. if len > 0 then
  500. widestringmanager.wide2ansimoveproc(pwidechar(@src[1]),temp,len);
  501. len := length(temp);
  502. if len > arraysize then
  503. len := arraysize;
  504. {$r-}
  505. move(temp[1],fpc_widestr_to_chararray[0],len);
  506. fillchar(fpc_widestr_to_chararray[len],arraysize-len,0);
  507. {$ifdef RangeCheckWasOn}
  508. {$r+}
  509. {$endif}
  510. end;
  511. { inside the compiler, the resulttype is modified to that of the actual }
  512. { widechararray we're converting to (JM) }
  513. function fpc_widestr_to_widechararray(arraysize: SizeInt; const src: WideString): fpc_big_widechararray;[public,alias: 'FPC_WIDESTR_TO_WIDECHARARRAY']; compilerproc;
  514. var
  515. len: SizeInt;
  516. begin
  517. len := length(src);
  518. if len > arraysize then
  519. len := arraysize;
  520. {$r-}
  521. { make sure we don't try to access element 1 of the ansistring if it's nil }
  522. if len > 0 then
  523. move(src[1],fpc_widestr_to_widechararray[0],len*SizeOf(WideChar));
  524. fillchar(fpc_widestr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  525. {$ifdef RangeCheckWasOn}
  526. {$r+}
  527. {$endif}
  528. end;
  529. { inside the compiler, the resulttype is modified to that of the actual }
  530. { chararray we're converting to (JM) }
  531. function fpc_ansistr_to_widechararray(arraysize: SizeInt; const src: AnsiString): fpc_big_widechararray;[public,alias: 'FPC_ANSISTR_TO_WIDECHARARRAY']; compilerproc;
  532. var
  533. len: SizeInt;
  534. temp: widestring;
  535. begin
  536. len := length(src);
  537. { make sure we don't dereference src if it can be nil (JM) }
  538. if len > 0 then
  539. widestringmanager.ansi2widemoveproc(pchar(@src[1]),temp,len);
  540. len := length(temp);
  541. if len > arraysize then
  542. len := arraysize;
  543. {$r-}
  544. move(temp[1],fpc_ansistr_to_widechararray[0],len*sizeof(widechar));
  545. fillchar(fpc_ansistr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  546. {$ifdef RangeCheckWasOn}
  547. {$r+}
  548. {$endif}
  549. end;
  550. function fpc_shortstr_to_widechararray(arraysize: SizeInt; const src: ShortString): fpc_big_widechararray;[public,alias: 'FPC_SHORTSTR_TO_WIDECHARARRAY']; compilerproc;
  551. var
  552. len: longint;
  553. temp : widestring;
  554. begin
  555. len := length(src);
  556. { make sure we don't access char 1 if length is 0 (JM) }
  557. if len > 0 then
  558. widestringmanager.ansi2widemoveproc(pchar(@src[1]),temp,len);
  559. len := length(temp);
  560. if len > arraysize then
  561. len := arraysize;
  562. {$r-}
  563. move(temp[1],fpc_shortstr_to_widechararray[0],len*sizeof(widechar));
  564. fillchar(fpc_shortstr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  565. {$ifdef RangeCheckWasOn}
  566. {$r+}
  567. {$endif}
  568. end;
  569. {$else ndef FPC_STRTOCHARARRAYPROC}
  570. procedure fpc_widestr_to_chararray(out res: array of char; const src: WideString); compilerproc;
  571. var
  572. len: SizeInt;
  573. temp: ansistring;
  574. begin
  575. len := length(src);
  576. { make sure we don't dereference src if it can be nil (JM) }
  577. if len > 0 then
  578. widestringmanager.wide2ansimoveproc(pwidechar(@src[1]),temp,len);
  579. len := length(temp);
  580. if len > length(res) then
  581. len := length(res);
  582. {$r-}
  583. move(temp[1],res[0],len);
  584. fillchar(res[len],length(res)-len,0);
  585. {$ifdef RangeCheckWasOn}
  586. {$r+}
  587. {$endif}
  588. end;
  589. procedure fpc_widestr_to_widechararray(out res: array of widechar; const src: WideString); compilerproc;
  590. var
  591. len: SizeInt;
  592. begin
  593. len := length(src);
  594. if len > length(res) then
  595. len := length(res);
  596. {$r-}
  597. { make sure we don't try to access element 1 of the ansistring if it's nil }
  598. if len > 0 then
  599. move(src[1],res[0],len*SizeOf(WideChar));
  600. fillchar(res[len],(length(res)-len)*SizeOf(WideChar),0);
  601. {$ifdef RangeCheckWasOn}
  602. {$r+}
  603. {$endif}
  604. end;
  605. {$endif ndef FPC_STRTOCHARARRAYPROC}
  606. Function fpc_WideStr_Compare(const S1,S2 : WideString): SizeInt;[Public,Alias : 'FPC_WIDESTR_COMPARE']; compilerproc;
  607. {
  608. Compares 2 WideStrings;
  609. The result is
  610. <0 if S1<S2
  611. 0 if S1=S2
  612. >0 if S1>S2
  613. }
  614. Var
  615. MaxI,Temp : SizeInt;
  616. begin
  617. if pointer(S1)=pointer(S2) then
  618. begin
  619. fpc_WideStr_Compare:=0;
  620. exit;
  621. end;
  622. Maxi:=Length(S1);
  623. temp:=Length(S2);
  624. If MaxI>Temp then
  625. MaxI:=Temp;
  626. Temp:=CompareWord(S1[1],S2[1],MaxI);
  627. if temp=0 then
  628. temp:=Length(S1)-Length(S2);
  629. fpc_WideStr_Compare:=Temp;
  630. end;
  631. Function fpc_WideStr_Compare_Equal(const S1,S2 : WideString): SizeInt;[Public,Alias : 'FPC_WIDESTR_COMPARE_EQUAL']; compilerproc;
  632. {
  633. Compares 2 WideStrings for equality only;
  634. The result is
  635. 0 if S1=S2
  636. <>0 if S1<>S2
  637. }
  638. Var
  639. MaxI : SizeInt;
  640. begin
  641. if pointer(S1)=pointer(S2) then
  642. exit(0);
  643. Maxi:=Length(S1);
  644. If MaxI<>Length(S2) then
  645. exit(-1)
  646. else
  647. exit(CompareWord(S1[1],S2[1],MaxI));
  648. end;
  649. Procedure fpc_WideStr_CheckZero(p : pointer);[Public,Alias : 'FPC_WIDESTR_CHECKZERO']; compilerproc;
  650. begin
  651. if p=nil then
  652. HandleErrorFrame(201,get_frame);
  653. end;
  654. Procedure fpc_WideStr_CheckRange(len,index : SizeInt);[Public,Alias : 'FPC_WIDESTR_RANGECHECK']; compilerproc;
  655. begin
  656. if (index>len div 2) or (Index<1) then
  657. HandleErrorFrame(201,get_frame);
  658. end;
  659. Procedure fpc_WideStr_SetLength(Var S : WideString; l : SizeInt);[Public,Alias : 'FPC_WIDESTR_SETLENGTH']; compilerproc;
  660. {
  661. Sets The length of string S to L.
  662. Makes sure S is unique, and contains enough room.
  663. }
  664. Var
  665. Temp : Pointer;
  666. movelen: SizeInt;
  667. begin
  668. if (l>0) then
  669. begin
  670. if Pointer(S)=nil then
  671. begin
  672. { Need a complete new string...}
  673. Pointer(s):=NewWideString(l);
  674. end
  675. { windows doesn't support reallocing widestrings, this code
  676. is anyways subject to be removed because widestrings shouldn't be
  677. ref. counted anymore (FK) }
  678. else
  679. if
  680. {$ifdef MSWINDOWS}
  681. not winwidestringalloc and
  682. {$endif MSWINDOWS}
  683. True
  684. then
  685. begin
  686. Dec(Pointer(S),WideFirstOff);
  687. if SizeUInt(L*sizeof(WideChar)+WideRecLen)>MemSize(Pointer(S)) then
  688. reallocmem(pointer(S), L*sizeof(WideChar)+WideRecLen);
  689. Inc(Pointer(S), WideFirstOff);
  690. end
  691. else
  692. begin
  693. { Reallocation is needed... }
  694. Temp:=Pointer(NewWideString(L));
  695. if Length(S)>0 then
  696. begin
  697. if l < succ(length(s)) then
  698. movelen := l
  699. { also move terminating null }
  700. else
  701. movelen := succ(length(s));
  702. Move(Pointer(S)^,Temp^,movelen * Sizeof(WideChar));
  703. end;
  704. fpc_widestr_decr_ref(Pointer(S));
  705. Pointer(S):=Temp;
  706. end;
  707. { Force nil termination in case it gets shorter }
  708. PWord(Pointer(S)+l*sizeof(WideChar))^:=0;
  709. {$ifdef MSWINDOWS}
  710. if not winwidestringalloc then
  711. {$endif MSWINDOWS}
  712. PWideRec(Pointer(S)-WideFirstOff)^.Len:=l*sizeof(WideChar);
  713. end
  714. else
  715. begin
  716. { Length=0 }
  717. if Pointer(S)<>nil then
  718. fpc_widestr_decr_ref (Pointer(S));
  719. Pointer(S):=Nil;
  720. end;
  721. end;
  722. {*****************************************************************************
  723. Public functions, In interface.
  724. *****************************************************************************}
  725. Function fpc_widestr_Unique(Var S : Pointer): Pointer; [Public,Alias : 'FPC_WIDESTR_UNIQUE']; compilerproc;
  726. begin
  727. pointer(result) := pointer(s);
  728. end;
  729. Function Fpc_WideStr_Copy (Const S : WideString; Index,Size : SizeInt) : WideString;compilerproc;
  730. var
  731. ResultAddress : Pointer;
  732. begin
  733. ResultAddress:=Nil;
  734. dec(index);
  735. if Index < 0 then
  736. Index := 0;
  737. { Check Size. Accounts for Zero-length S, the double check is needed because
  738. Size can be maxint and will get <0 when adding index }
  739. if (Size>Length(S)) or
  740. (Index+Size>Length(S)) then
  741. Size:=Length(S)-Index;
  742. If Size>0 then
  743. begin
  744. If Index<0 Then
  745. Index:=0;
  746. ResultAddress:=Pointer(NewWideString (Size));
  747. if ResultAddress<>Nil then
  748. begin
  749. Move (PWideChar(S)[Index],ResultAddress^,Size*sizeof(WideChar));
  750. PWideRec(ResultAddress-WideFirstOff)^.Len:=Size*sizeof(WideChar);
  751. PWideChar(ResultAddress+Size*sizeof(WideChar))^:=#0;
  752. end;
  753. end;
  754. fpc_widestr_decr_ref(Pointer(fpc_widestr_copy));
  755. Pointer(fpc_widestr_Copy):=ResultAddress;
  756. end;
  757. Function Pos (Const Substr : WideString; Const Source : WideString) : SizeInt;
  758. var
  759. i,MaxLen : SizeInt;
  760. pc : pwidechar;
  761. begin
  762. Pos:=0;
  763. if Length(SubStr)>0 then
  764. begin
  765. MaxLen:=Length(source)-Length(SubStr);
  766. i:=0;
  767. pc:=@source[1];
  768. while (i<=MaxLen) do
  769. begin
  770. inc(i);
  771. if (SubStr[1]=pc^) and
  772. (CompareWord(Substr[1],pc^,Length(SubStr))=0) then
  773. begin
  774. Pos:=i;
  775. exit;
  776. end;
  777. inc(pc);
  778. end;
  779. end;
  780. end;
  781. { Faster version for a widechar alone }
  782. Function Pos (c : WideChar; Const s : WideString) : SizeInt;
  783. var
  784. i: SizeInt;
  785. pc : pwidechar;
  786. begin
  787. pc:=@s[1];
  788. for i:=1 to length(s) do
  789. begin
  790. if pc^=c then
  791. begin
  792. pos:=i;
  793. exit;
  794. end;
  795. inc(pc);
  796. end;
  797. pos:=0;
  798. end;
  799. Function Pos (c : WideChar; Const s : AnsiString) : SizeInt;
  800. begin
  801. result:=Pos(c,WideString(s));
  802. end;
  803. Function Pos (c : AnsiString; Const s : WideString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  804. begin
  805. result:=Pos(WideString(c),s);
  806. end;
  807. Function Pos (c : ShortString; Const s : WideString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  808. begin
  809. result:=Pos(WideString(c),s);
  810. end;
  811. Function Pos (c : WideString; Const s : AnsiString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  812. begin
  813. result:=Pos(c,WideString(s));
  814. end;
  815. { Faster version for a char alone. Must be implemented because }
  816. { pos(c: char; const s: shortstring) also exists, so otherwise }
  817. { using pos(char,pchar) will always call the shortstring version }
  818. { (exact match for first argument), also with $h+ (JM) }
  819. Function Pos (c : Char; Const s : WideString) : SizeInt;
  820. var
  821. i: SizeInt;
  822. wc : widechar;
  823. pc : pwidechar;
  824. begin
  825. wc:=c;
  826. pc:=@s[1];
  827. for i:=1 to length(s) do
  828. begin
  829. if pc^=wc then
  830. begin
  831. pos:=i;
  832. exit;
  833. end;
  834. inc(pc);
  835. end;
  836. pos:=0;
  837. end;
  838. Procedure Delete (Var S : WideString; Index,Size: SizeInt);
  839. Var
  840. LS : SizeInt;
  841. begin
  842. If Length(S)=0 then
  843. exit;
  844. if index<=0 then
  845. exit;
  846. LS:=PWideRec(Pointer(S)-WideFirstOff)^.Len div sizeof(WideChar);
  847. if (Index<=LS) and (Size>0) then
  848. begin
  849. UniqueString (S);
  850. if Size+Index>LS then
  851. Size:=LS-Index+1;
  852. if Index+Size<=LS then
  853. begin
  854. Dec(Index);
  855. Move(PWideChar(S)[Index+Size],PWideChar(S)[Index],(LS-Index-Size+1)*sizeof(WideChar));
  856. end;
  857. Setlength(s,LS-Size);
  858. end;
  859. end;
  860. Procedure Insert (Const Source : WideString; Var S : WideString; Index : SizeInt);
  861. var
  862. Temp : WideString;
  863. LS : SizeInt;
  864. begin
  865. If Length(Source)=0 then
  866. exit;
  867. if index <= 0 then
  868. index := 1;
  869. Ls:=Length(S);
  870. if index > LS then
  871. index := LS+1;
  872. Dec(Index);
  873. Pointer(Temp) := NewWideString(Length(Source)+LS);
  874. SetLength(Temp,Length(Source)+LS);
  875. If Index>0 then
  876. move (PWideChar(S)^,PWideChar(Temp)^,Index*sizeof(WideChar));
  877. Move (PWideChar(Source)^,PWideChar(Temp)[Index],Length(Source)*sizeof(WideChar));
  878. If (LS-Index)>0 then
  879. Move(PWideChar(S)[Index],PWideChar(temp)[Length(Source)+index],(LS-Index)*sizeof(WideChar));
  880. S:=Temp;
  881. end;
  882. function UpCase(const s : WideString) : WideString;
  883. begin
  884. result:=widestringmanager.UpperWideStringProc(s);
  885. end;
  886. Procedure SetString (Out S : WideString; Buf : PWideChar; Len : SizeInt);
  887. begin
  888. SetLength(S,Len);
  889. If (Buf<>Nil) and (Len>0) then
  890. Move (Buf[0],S[1],Len*sizeof(WideChar));
  891. end;
  892. Procedure SetString (Out S : WideString; Buf : PChar; Len : SizeInt);
  893. begin
  894. SetLength(S,Len);
  895. If (Buf<>Nil) and (Len>0) then
  896. widestringmanager.Ansi2WideMoveProc(Buf,S,Len);
  897. end;
  898. {$ifndef FPUNONE}
  899. Function fpc_Val_Real_WideStr(Const S : WideString; out Code : ValSInt): ValReal; [public, alias:'FPC_VAL_REAL_WIDESTR']; compilerproc;
  900. Var
  901. SS : String;
  902. begin
  903. fpc_Val_Real_WideStr := 0;
  904. if length(S) > 255 then
  905. code := 256
  906. else
  907. begin
  908. SS := S;
  909. Val(SS,fpc_Val_Real_WideStr,code);
  910. end;
  911. end;
  912. {$endif}
  913. function fpc_val_enum_widestr(str2ordindex:pointer;const s:widestring;out code:valsint):longint;compilerproc;
  914. var ss:shortstring;
  915. begin
  916. if length(s)>255 then
  917. code:=256
  918. else
  919. begin
  920. ss:=s;
  921. val(ss,fpc_val_enum_widestr,code);
  922. end;
  923. end;
  924. Function fpc_Val_Currency_WideStr(Const S : WideString; out Code : ValSInt): Currency; [public, alias:'FPC_VAL_CURRENCY_WIDESTR']; compilerproc;
  925. Var
  926. SS : String;
  927. begin
  928. if length(S) > 255 then
  929. begin
  930. fpc_Val_Currency_WideStr:=0;
  931. code := 256;
  932. end
  933. else
  934. begin
  935. SS := S;
  936. Val(SS,fpc_Val_Currency_WideStr,code);
  937. end;
  938. end;
  939. Function fpc_Val_UInt_WideStr (Const S : WideString; out Code : ValSInt): ValUInt; [public, alias:'FPC_VAL_UINT_WIDESTR']; compilerproc;
  940. Var
  941. SS : ShortString;
  942. begin
  943. fpc_Val_UInt_WideStr := 0;
  944. if length(S) > 255 then
  945. code := 256
  946. else
  947. begin
  948. SS := S;
  949. Val(SS,fpc_Val_UInt_WideStr,code);
  950. end;
  951. end;
  952. Function fpc_Val_SInt_WideStr (DestSize: SizeInt; Const S : WideString; out Code : ValSInt): ValSInt; [public, alias:'FPC_VAL_SINT_WIDESTR']; compilerproc;
  953. Var
  954. SS : ShortString;
  955. begin
  956. fpc_Val_SInt_WideStr:=0;
  957. if length(S)>255 then
  958. code:=256
  959. else
  960. begin
  961. SS := S;
  962. fpc_Val_SInt_WideStr := int_Val_SInt_ShortStr(DestSize,SS,Code);
  963. end;
  964. end;
  965. {$ifndef CPU64}
  966. Function fpc_Val_qword_WideStr (Const S : WideString; out Code : ValSInt): qword; [public, alias:'FPC_VAL_QWORD_WIDESTR']; compilerproc;
  967. Var
  968. SS : ShortString;
  969. begin
  970. fpc_Val_qword_WideStr:=0;
  971. if length(S)>255 then
  972. code:=256
  973. else
  974. begin
  975. SS := S;
  976. Val(SS,fpc_Val_qword_WideStr,Code);
  977. end;
  978. end;
  979. Function fpc_Val_int64_WideStr (Const S : WideString; out Code : ValSInt): Int64; [public, alias:'FPC_VAL_INT64_WIDESTR']; compilerproc;
  980. Var
  981. SS : ShortString;
  982. begin
  983. fpc_Val_int64_WideStr:=0;
  984. if length(S)>255 then
  985. code:=256
  986. else
  987. begin
  988. SS := S;
  989. Val(SS,fpc_Val_int64_WideStr,Code);
  990. end;
  991. end;
  992. {$endif CPU64}
  993. {$ifndef FPUNONE}
  994. procedure fpc_WideStr_Float(d : ValReal;len,fr,rt : SizeInt;out s : WideString);compilerproc;
  995. var
  996. ss : shortstring;
  997. begin
  998. str_real(len,fr,d,treal_type(rt),ss);
  999. s:=ss;
  1000. end;
  1001. {$endif}
  1002. procedure fpc_widestr_enum(ordinal,len:sizeint;typinfo,ord2strindex:pointer;out s:widestring);compilerproc;
  1003. var ss:shortstring;
  1004. begin
  1005. fpc_shortstr_enum(ordinal,len,typinfo,ord2strindex,ss);
  1006. s:=ss;
  1007. end;
  1008. {$ifdef FPC_HAS_STR_CURRENCY}
  1009. procedure fpc_WideStr_Currency(c : Currency;len,fr : SizeInt;out s : WideString);compilerproc;
  1010. var
  1011. ss : shortstring;
  1012. begin
  1013. str(c:len:fr,ss);
  1014. s:=ss;
  1015. end;
  1016. {$endif FPC_HAS_STR_CURRENCY}
  1017. Procedure fpc_WideStr_SInt(v : ValSint; Len : SizeInt; out S : WideString);compilerproc;
  1018. Var
  1019. SS : ShortString;
  1020. begin
  1021. Str (v:Len,SS);
  1022. S:=SS;
  1023. end;
  1024. Procedure fpc_WideStr_UInt(v : ValUInt;Len : SizeInt; out S : WideString);compilerproc;
  1025. Var
  1026. SS : ShortString;
  1027. begin
  1028. str(v:Len,SS);
  1029. S:=SS;
  1030. end;
  1031. {$ifndef CPU64}
  1032. Procedure fpc_WideStr_Int64(v : Int64; Len : SizeInt; out S : WideString);compilerproc;
  1033. Var
  1034. SS : ShortString;
  1035. begin
  1036. Str (v:Len,SS);
  1037. S:=SS;
  1038. end;
  1039. Procedure fpc_WideStr_Qword(v : Qword;Len : SizeInt; out S : WideString);compilerproc;
  1040. Var
  1041. SS : ShortString;
  1042. begin
  1043. str(v:Len,SS);
  1044. S:=SS;
  1045. end;
  1046. {$endif CPU64}
  1047. { converts an utf-16 code point or surrogate pair to utf-32 }
  1048. function utf16toutf32(const S: WideString; const index: SizeInt; out len: longint): UCS4Char; [public, alias: 'FPC_WIDETOUTF32'];
  1049. var
  1050. w: widechar;
  1051. begin
  1052. { UTF-16 points in the range #$0-#$D7FF and #$E000-#$FFFF }
  1053. { are the same in UTF-32 }
  1054. w:=s[index];
  1055. if (w<=#$d7ff) or
  1056. (w>=#$e000) then
  1057. begin
  1058. result:=UCS4Char(w);
  1059. len:=1;
  1060. end
  1061. { valid surrogate pair? }
  1062. else if (w<=#$dbff) and
  1063. { w>=#$d7ff check not needed, checked above }
  1064. (index<length(s)) and
  1065. (s[index+1]>=#$dc00) and
  1066. (s[index+1]<=#$dfff) then
  1067. { convert the surrogate pair to UTF-32 }
  1068. begin
  1069. result:=(UCS4Char(w)-$d800) shl 10 + (UCS4Char(s[index+1])-$dc00) + $10000;
  1070. len:=2;
  1071. end
  1072. else
  1073. { invalid surrogate -> do nothing }
  1074. begin
  1075. result:=UCS4Char(w);
  1076. len:=1;
  1077. end;
  1078. end;
  1079. function UnicodeToUtf8(Dest: PChar; Source: PWideChar; MaxBytes: SizeInt): SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  1080. begin
  1081. if assigned(Source) then
  1082. Result:=UnicodeToUtf8(Dest,MaxBytes,Source,IndexWord(Source^,-1,0))
  1083. else
  1084. Result:=0;
  1085. end;
  1086. function UnicodeToUtf8(Dest: PChar; MaxDestBytes: SizeUInt; Source: PWideChar; SourceChars: SizeUInt): SizeUInt;
  1087. var
  1088. i,j : SizeUInt;
  1089. w : word;
  1090. lw : longword;
  1091. len : longint;
  1092. begin
  1093. result:=0;
  1094. if source=nil then
  1095. exit;
  1096. i:=0;
  1097. j:=0;
  1098. if assigned(Dest) then
  1099. begin
  1100. while (i<SourceChars) and (j<MaxDestBytes) do
  1101. begin
  1102. w:=word(Source[i]);
  1103. case w of
  1104. 0..$7f:
  1105. begin
  1106. Dest[j]:=char(w);
  1107. inc(j);
  1108. end;
  1109. $80..$7ff:
  1110. begin
  1111. if j+1>=MaxDestBytes then
  1112. break;
  1113. Dest[j]:=char($c0 or (w shr 6));
  1114. Dest[j+1]:=char($80 or (w and $3f));
  1115. inc(j,2);
  1116. end;
  1117. $800..$d7ff,$e000..$ffff:
  1118. begin
  1119. if j+2>=MaxDestBytes then
  1120. break;
  1121. Dest[j]:=char($e0 or (w shr 12));
  1122. Dest[j+1]:=char($80 or ((w shr 6) and $3f));
  1123. Dest[j+2]:=char($80 or (w and $3f));
  1124. inc(j,3);
  1125. end;
  1126. $d800..$dbff:
  1127. {High Surrogates}
  1128. begin
  1129. if j+3>=MaxDestBytes then
  1130. break;
  1131. if (i<sourcechars-1) and
  1132. (word(Source[i+1]) >= $dc00) and
  1133. (word(Source[i+1]) <= $dfff) then
  1134. begin
  1135. lw:=longword(utf16toutf32(Source[i] + Source[i+1], 1, len));
  1136. Dest[j]:=char($f0 or (lw shr 18));
  1137. Dest[j+1]:=char($80 or ((lw shr 12) and $3f));
  1138. Dest[j+2]:=char($80 or ((lw shr 6) and $3f));
  1139. Dest[j+3]:=char($80 or (lw and $3f));
  1140. inc(j,4);
  1141. inc(i);
  1142. end;
  1143. end;
  1144. end;
  1145. inc(i);
  1146. end;
  1147. if j>SizeUInt(MaxDestBytes-1) then
  1148. j:=MaxDestBytes-1;
  1149. Dest[j]:=#0;
  1150. end
  1151. else
  1152. begin
  1153. while i<SourceChars do
  1154. begin
  1155. case word(Source[i]) of
  1156. $0..$7f:
  1157. inc(j);
  1158. $80..$7ff:
  1159. inc(j,2);
  1160. $800..$d7ff,$e000..$ffff:
  1161. inc(j,3);
  1162. $d800..$dbff:
  1163. begin
  1164. if (i<sourcechars-1) and
  1165. (word(Source[i+1]) >= $dc00) and
  1166. (word(Source[i+1]) <= $dfff) then
  1167. begin
  1168. inc(j,4);
  1169. inc(i);
  1170. end;
  1171. end;
  1172. end;
  1173. inc(i);
  1174. end;
  1175. end;
  1176. result:=j+1;
  1177. end;
  1178. function Utf8ToUnicode(Dest: PWideChar; Source: PChar; MaxChars: SizeInt): SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  1179. begin
  1180. if assigned(Source) then
  1181. Result:=Utf8ToUnicode(Dest,MaxChars,Source,strlen(Source))
  1182. else
  1183. Result:=0;
  1184. end;
  1185. function UTF8ToUnicode(Dest: PWideChar; MaxDestChars: SizeUInt; Source: PChar; SourceBytes: SizeUInt): SizeUInt;
  1186. const
  1187. UNICODE_INVALID=63;
  1188. var
  1189. InputUTF8: SizeUInt;
  1190. IBYTE: BYTE;
  1191. OutputUnicode: SizeUInt;
  1192. PRECHAR: SizeUInt;
  1193. TempBYTE: BYTE;
  1194. CharLen: SizeUint;
  1195. LookAhead: SizeUInt;
  1196. UC: SizeUInt;
  1197. begin
  1198. if not assigned(Source) then
  1199. begin
  1200. result:=0;
  1201. exit;
  1202. end;
  1203. result:=SizeUInt(-1);
  1204. InputUTF8:=0;
  1205. OutputUnicode:=0;
  1206. PreChar:=0;
  1207. if Assigned(Dest) Then
  1208. begin
  1209. while (OutputUnicode<MaxDestChars) and (InputUTF8<SourceBytes) do
  1210. begin
  1211. IBYTE:=byte(Source[InputUTF8]);
  1212. if (IBYTE and $80) = 0 then
  1213. begin
  1214. //One character US-ASCII, convert it to unicode
  1215. if IBYTE = 10 then
  1216. begin
  1217. If (PreChar<>13) and FALSE then
  1218. begin
  1219. //Expand to crlf, conform UTF-8.
  1220. //This procedure will break the memory alocation by
  1221. //FPC for the widestring, so never use it. Condition never true due the "and FALSE".
  1222. if OutputUnicode+1<MaxDestChars then
  1223. begin
  1224. Dest[OutputUnicode]:=WideChar(13);
  1225. inc(OutputUnicode);
  1226. Dest[OutputUnicode]:=WideChar(10);
  1227. inc(OutputUnicode);
  1228. PreChar:=10;
  1229. end
  1230. else
  1231. begin
  1232. Dest[OutputUnicode]:=WideChar(13);
  1233. inc(OutputUnicode);
  1234. end;
  1235. end
  1236. else
  1237. begin
  1238. Dest[OutputUnicode]:=WideChar(IBYTE);
  1239. inc(OutputUnicode);
  1240. PreChar:=IBYTE;
  1241. end;
  1242. end
  1243. else
  1244. begin
  1245. Dest[OutputUnicode]:=WideChar(IBYTE);
  1246. inc(OutputUnicode);
  1247. PreChar:=IBYTE;
  1248. end;
  1249. inc(InputUTF8);
  1250. end
  1251. else
  1252. begin
  1253. TempByte:=IBYTE;
  1254. CharLen:=0;
  1255. while (TempBYTE and $80)<>0 do
  1256. begin
  1257. TempBYTE:=(TempBYTE shl 1) and $FE;
  1258. inc(CharLen);
  1259. end;
  1260. //Test for the "CharLen" conforms UTF-8 string
  1261. //This means the 10xxxxxx pattern.
  1262. if SizeUInt(InputUTF8+CharLen-1)>SourceBytes then
  1263. begin
  1264. //Insuficient chars in string to decode
  1265. //UTF-8 array. Fallback to single char.
  1266. CharLen:= 1;
  1267. end;
  1268. for LookAhead := 1 to CharLen-1 do
  1269. begin
  1270. if ((byte(Source[InputUTF8+LookAhead]) and $80)<>$80) or
  1271. ((byte(Source[InputUTF8+LookAhead]) and $40)<>$00) then
  1272. begin
  1273. //Invalid UTF-8 sequence, fallback.
  1274. CharLen:= LookAhead;
  1275. break;
  1276. end;
  1277. end;
  1278. UC:=$FFFF;
  1279. case CharLen of
  1280. 1: begin
  1281. //Not valid UTF-8 sequence
  1282. UC:=UNICODE_INVALID;
  1283. end;
  1284. 2: begin
  1285. //Two bytes UTF, convert it
  1286. UC:=(byte(Source[InputUTF8]) and $1F) shl 6;
  1287. UC:=UC or (byte(Source[InputUTF8+1]) and $3F);
  1288. if UC <= $7F then
  1289. begin
  1290. //Invalid UTF sequence.
  1291. UC:=UNICODE_INVALID;
  1292. end;
  1293. end;
  1294. 3: begin
  1295. //Three bytes, convert it to unicode
  1296. UC:= (byte(Source[InputUTF8]) and $0F) shl 12;
  1297. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 6);
  1298. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F));
  1299. if (UC <= $7FF) or (UC >= $FFFE) or ((UC >= $D800) and (UC <= $DFFF)) then
  1300. begin
  1301. //Invalid UTF-8 sequence
  1302. UC:= UNICODE_INVALID;
  1303. End;
  1304. end;
  1305. 4: begin
  1306. //Four bytes, convert it to two unicode characters
  1307. UC:= (byte(Source[InputUTF8]) and $07) shl 18;
  1308. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 12);
  1309. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F) shl 6);
  1310. UC:= UC or ((byte(Source[InputUTF8+3]) and $3F));
  1311. if (UC < $10000) or (UC > $10FFFF) then
  1312. begin
  1313. UC:= UNICODE_INVALID;
  1314. end
  1315. else
  1316. begin
  1317. { only store pair if room }
  1318. dec(UC,$10000);
  1319. if (OutputUnicode<MaxDestChars-1) then
  1320. begin
  1321. Dest[OutputUnicode]:=WideChar(UC shr 10 + $D800);
  1322. inc(OutputUnicode);
  1323. UC:=(UC and $3ff) + $DC00;
  1324. end
  1325. else
  1326. begin
  1327. InputUTF8:= InputUTF8 + CharLen;
  1328. { don't store anything }
  1329. CharLen:=0;
  1330. end;
  1331. end;
  1332. end;
  1333. 5,6,7: begin
  1334. //Invalid UTF8 to unicode conversion,
  1335. //mask it as invalid UNICODE too.
  1336. UC:=UNICODE_INVALID;
  1337. end;
  1338. end;
  1339. if CharLen > 0 then
  1340. begin
  1341. PreChar:=UC;
  1342. Dest[OutputUnicode]:=WideChar(UC);
  1343. inc(OutputUnicode);
  1344. end;
  1345. InputUTF8:= InputUTF8 + CharLen;
  1346. end;
  1347. end;
  1348. Result:=OutputUnicode+1;
  1349. end
  1350. else
  1351. begin
  1352. while (InputUTF8<SourceBytes) do
  1353. begin
  1354. IBYTE:=byte(Source[InputUTF8]);
  1355. if (IBYTE and $80) = 0 then
  1356. begin
  1357. //One character US-ASCII, convert it to unicode
  1358. if IBYTE = 10 then
  1359. begin
  1360. if (PreChar<>13) and FALSE then
  1361. begin
  1362. //Expand to crlf, conform UTF-8.
  1363. //This procedure will break the memory alocation by
  1364. //FPC for the widestring, so never use it. Condition never true due the "and FALSE".
  1365. inc(OutputUnicode,2);
  1366. PreChar:=10;
  1367. end
  1368. else
  1369. begin
  1370. inc(OutputUnicode);
  1371. PreChar:=IBYTE;
  1372. end;
  1373. end
  1374. else
  1375. begin
  1376. inc(OutputUnicode);
  1377. PreChar:=IBYTE;
  1378. end;
  1379. inc(InputUTF8);
  1380. end
  1381. else
  1382. begin
  1383. TempByte:=IBYTE;
  1384. CharLen:=0;
  1385. while (TempBYTE and $80)<>0 do
  1386. begin
  1387. TempBYTE:=(TempBYTE shl 1) and $FE;
  1388. inc(CharLen);
  1389. end;
  1390. //Test for the "CharLen" conforms UTF-8 string
  1391. //This means the 10xxxxxx pattern.
  1392. if SizeUInt(InputUTF8+CharLen-1)>SourceBytes then
  1393. begin
  1394. //Insuficient chars in string to decode
  1395. //UTF-8 array. Fallback to single char.
  1396. CharLen:= 1;
  1397. end;
  1398. for LookAhead := 1 to CharLen-1 do
  1399. begin
  1400. if ((byte(Source[InputUTF8+LookAhead]) and $80)<>$80) or
  1401. ((byte(Source[InputUTF8+LookAhead]) and $40)<>$00) then
  1402. begin
  1403. //Invalid UTF-8 sequence, fallback.
  1404. CharLen:= LookAhead;
  1405. break;
  1406. end;
  1407. end;
  1408. UC:=$FFFF;
  1409. case CharLen of
  1410. 1: begin
  1411. //Not valid UTF-8 sequence
  1412. UC:=UNICODE_INVALID;
  1413. end;
  1414. 2: begin
  1415. //Two bytes UTF, convert it
  1416. UC:=(byte(Source[InputUTF8]) and $1F) shl 6;
  1417. UC:=UC or (byte(Source[InputUTF8+1]) and $3F);
  1418. if UC <= $7F then
  1419. begin
  1420. //Invalid UTF sequence.
  1421. UC:=UNICODE_INVALID;
  1422. end;
  1423. end;
  1424. 3: begin
  1425. //Three bytes, convert it to unicode
  1426. UC:= (byte(Source[InputUTF8]) and $0F) shl 12;
  1427. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 6);
  1428. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F));
  1429. If (UC <= $7FF) or (UC >= $FFFE) or ((UC >= $D800) and (UC <= $DFFF)) then
  1430. begin
  1431. //Invalid UTF-8 sequence
  1432. UC:= UNICODE_INVALID;
  1433. end;
  1434. end;
  1435. 4: begin
  1436. //Four bytes, convert it to two unicode characters
  1437. UC:= (byte(Source[InputUTF8]) and $07) shl 18;
  1438. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 12);
  1439. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F) shl 6);
  1440. UC:= UC or ((byte(Source[InputUTF8+3]) and $3F));
  1441. if (UC < $10000) or (UC > $10FFFF) then
  1442. UC:= UNICODE_INVALID
  1443. else
  1444. { extra character character }
  1445. inc(OutputUnicode);
  1446. end;
  1447. 5,6,7: begin
  1448. //Invalid UTF8 to unicode conversion,
  1449. //mask it as invalid UNICODE too.
  1450. UC:=UNICODE_INVALID;
  1451. end;
  1452. end;
  1453. if CharLen > 0 then
  1454. begin
  1455. PreChar:=UC;
  1456. inc(OutputUnicode);
  1457. end;
  1458. InputUTF8:= InputUTF8 + CharLen;
  1459. end;
  1460. end;
  1461. Result:=OutputUnicode+1;
  1462. end;
  1463. end;
  1464. function UTF8Encode(const s : WideString) : UTF8String;
  1465. var
  1466. i : SizeInt;
  1467. hs : UTF8String;
  1468. begin
  1469. result:='';
  1470. if s='' then
  1471. exit;
  1472. SetLength(hs,length(s)*3);
  1473. i:=UnicodeToUtf8(pchar(hs),length(hs)+1,PWideChar(s),length(s));
  1474. if i>0 then
  1475. begin
  1476. SetLength(hs,i-1);
  1477. result:=hs;
  1478. end;
  1479. end;
  1480. const
  1481. SNoWidestrings = 'This binary has no widestrings support compiled in.';
  1482. SRecompileWithWidestrings = 'Recompile the application with a widestrings-manager in the program uses clause.';
  1483. procedure unimplementedwidestring;
  1484. begin
  1485. {$ifdef FPC_HAS_FEATURE_CONSOLEIO}
  1486. If IsConsole then
  1487. begin
  1488. Writeln(StdErr,SNoWidestrings);
  1489. Writeln(StdErr,SRecompileWithWidestrings);
  1490. end;
  1491. {$endif FPC_HAS_FEATURE_CONSOLEIO}
  1492. HandleErrorFrame(233,get_frame);
  1493. end;
  1494. {$warnings off}
  1495. function GenericWideCase(const s : WideString) : WideString;
  1496. begin
  1497. unimplementedwidestring;
  1498. end;
  1499. function CompareWideString(const s1, s2 : WideString) : PtrInt;
  1500. begin
  1501. unimplementedwidestring;
  1502. end;
  1503. function CompareTextWideString(const s1, s2 : WideString): PtrInt;
  1504. begin
  1505. unimplementedwidestring;
  1506. end;
  1507. {$warnings on}
  1508. function CharLengthPChar(const Str: PChar): PtrInt;forward;
  1509. procedure initwidestringmanager;
  1510. begin
  1511. fillchar(widestringmanager,sizeof(widestringmanager),0);
  1512. {$ifndef HAS_WIDESTRINGMANAGER}
  1513. widestringmanager.Wide2AnsiMoveProc:=@defaultWide2AnsiMove;
  1514. widestringmanager.Ansi2WideMoveProc:=@defaultAnsi2WideMove;
  1515. widestringmanager.UpperWideStringProc:=@GenericWideCase;
  1516. widestringmanager.LowerWideStringProc:=@GenericWideCase;
  1517. {$endif HAS_WIDESTRINGMANAGER}
  1518. widestringmanager.CompareWideStringProc:=@CompareWideString;
  1519. widestringmanager.CompareTextWideStringProc:=@CompareTextWideString;
  1520. widestringmanager.CharLengthPCharProc:=@CharLengthPChar;
  1521. end;