wstrings.inc 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689
  1. {
  2. This file is part of the Free Pascal run time library.
  3. Copyright (c) 1999-2005 by Florian Klaempfl,
  4. member of the Free Pascal development team.
  5. This file implements support routines for WideStrings with FPC
  6. See the file COPYING.FPC, included in this distribution,
  7. for details about the copyright.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11. **********************************************************************}
  12. {
  13. This file contains the implementation of the WideString type,
  14. and all things that are needed for it.
  15. WideString is defined as a 'silent' pwidechar :
  16. a pwidechar that points to :
  17. @-8 : SizeInt for reference count;
  18. @-4 : SizeInt for size; size=number of bytes, not the number of chars. Divide or multiply
  19. with sizeof(WideChar) to convert. This is needed to be compatible with Delphi and
  20. Windows COM BSTR.
  21. @ : String + Terminating #0;
  22. Pwidechar(Widestring) is a valid typecast.
  23. So WS[i] is converted to the address @WS+i-1.
  24. Constants should be assigned a reference count of -1
  25. Meaning that they can't be disposed of.
  26. }
  27. Type
  28. PWideRec = ^TWideRec;
  29. TWideRec = Packed Record
  30. Len : DWord;
  31. First : WideChar;
  32. end;
  33. Const
  34. WideRecLen = SizeOf(TWideRec);
  35. WideFirstOff = SizeOf(TWideRec)-sizeof(WideChar);
  36. {
  37. Default WideChar <-> Char conversion is to only convert the
  38. lower 127 chars, all others are translated to spaces.
  39. These routines can be overwritten for the Current Locale
  40. }
  41. procedure DefaultWide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
  42. var
  43. i : SizeInt;
  44. destp: PChar;
  45. begin
  46. setlength(dest,len);
  47. destp := PChar(Pointer(dest));
  48. for i:=1 to len do
  49. begin
  50. if word(source^)<256 then
  51. destp^:=char(word(source^))
  52. else
  53. destp^:='?';
  54. inc(source);
  55. inc(destp);
  56. end;
  57. end;
  58. procedure DefaultAnsi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
  59. var
  60. i : SizeInt;
  61. begin
  62. setlength(dest,len);
  63. for i:=1 to len do
  64. begin
  65. dest[i]:=widechar(byte(source^));
  66. inc(source);
  67. end;
  68. end;
  69. (*
  70. Procedure UniqueWideString(Var S : WideString); [Public,Alias : 'FPC_WIDESTR_UNIQUE'];
  71. {
  72. Make sure reference count of S is 1,
  73. using copy-on-write semantics.
  74. }
  75. begin
  76. end;
  77. *)
  78. {****************************************************************************
  79. Internal functions, not in interface.
  80. ****************************************************************************}
  81. procedure WideStringError;
  82. begin
  83. HandleErrorFrame(204,get_frame);
  84. end;
  85. {$ifdef WideStrDebug}
  86. Procedure DumpWideRec(S : Pointer);
  87. begin
  88. If S=Nil then
  89. Writeln ('String is nil')
  90. Else
  91. Begin
  92. With PWideRec(S-WideFirstOff)^ do
  93. begin
  94. Write ('(Len:',len);
  95. Writeln (' Ref: ',ref,')');
  96. end;
  97. end;
  98. end;
  99. {$endif}
  100. Function NewWideString(Len : SizeInt) : Pointer;
  101. {
  102. Allocate a new WideString on the heap.
  103. initialize it to zero length and reference count 1.
  104. }
  105. Var
  106. P : Pointer;
  107. begin
  108. {$ifdef MSWINDOWS}
  109. if winwidestringalloc then
  110. begin
  111. P:=SysAllocStringLen(nil,Len);
  112. if P=nil then
  113. WideStringError;
  114. end
  115. else
  116. {$endif MSWINDOWS}
  117. begin
  118. GetMem(P,Len*sizeof(WideChar)+WideRecLen);
  119. If P<>Nil then
  120. begin
  121. PWideRec(P)^.Len:=Len*2; { Initial length }
  122. PWideRec(P)^.First:=#0; { Terminating #0 }
  123. inc(p,WideFirstOff); { Points to string now }
  124. end
  125. else
  126. WideStringError;
  127. end;
  128. NewWideString:=P;
  129. end;
  130. Procedure DisposeWideString(Var S : Pointer);
  131. {
  132. Deallocates a WideString From the heap.
  133. }
  134. begin
  135. If S=Nil then
  136. exit;
  137. {$ifndef MSWINDOWS}
  138. Dec (S,WideFirstOff);
  139. Freemem(S);
  140. {$else MSWINDOWS}
  141. if winwidestringalloc then
  142. SysFreeString(S)
  143. else
  144. begin
  145. Dec (S,WideFirstOff);
  146. Freemem(S);
  147. end;
  148. {$endif MSWINDOWS}
  149. S:=Nil;
  150. end;
  151. Procedure fpc_WideStr_Decr_Ref (Var S : Pointer);[Public,Alias:'FPC_WIDESTR_DECR_REF']; compilerproc;
  152. {
  153. Decreases the ReferenceCount of a non constant widestring;
  154. If the reference count is zero, deallocate the string;
  155. }
  156. Begin
  157. DisposeWideString(S); { does test for nil }
  158. end;
  159. { alias for internal use }
  160. Procedure fpc_WideStr_Decr_Ref (Var S : Pointer);[external name 'FPC_WIDESTR_DECR_REF'];
  161. Procedure fpc_WideStr_Incr_Ref(Var S : Pointer);[Public,Alias:'FPC_WIDESTR_INCR_REF']; compilerproc;
  162. var
  163. p : pointer;
  164. Begin
  165. If S=Nil then
  166. exit;
  167. p:=NewWidestring(length(WideString(S)));
  168. move(s^,p^,(length(WideString(s))+1)*sizeof(widechar)); // double #0 too
  169. s:=p;
  170. end;
  171. { alias for internal use }
  172. Procedure fpc_WideStr_Incr_Ref (Var S : Pointer);[external name 'FPC_WIDESTR_INCR_REF'];
  173. {$ifndef FPC_STRTOSHORTSTRINGPROC}
  174. function fpc_WideStr_To_ShortStr (high_of_res: SizeInt;const S2 : WideString): shortstring;[Public, alias: 'FPC_WIDESTR_TO_SHORTSTR']; compilerproc;
  175. {
  176. Converts a WideString to a ShortString;
  177. }
  178. Var
  179. Size : SizeInt;
  180. temp : ansistring;
  181. begin
  182. result:='';
  183. Size:=Length(S2);
  184. if Size>0 then
  185. begin
  186. If Size>high_of_res then
  187. Size:=high_of_res;
  188. widestringmanager.Wide2AnsiMoveProc(PWideChar(S2),temp,Size);
  189. result:=temp;
  190. end;
  191. end;
  192. {$else FPC_STRTOSHORTSTRINGPROC}
  193. procedure fpc_WideStr_To_ShortStr (out res: ShortString;const S2 : WideString); [Public, alias: 'FPC_WIDESTR_TO_SHORTSTR'];compilerproc;
  194. {
  195. Converts a WideString to a ShortString;
  196. }
  197. Var
  198. Size : SizeInt;
  199. temp : ansistring;
  200. begin
  201. res:='';
  202. Size:=Length(S2);
  203. if Size>0 then
  204. begin
  205. If Size>high(res) then
  206. Size:=high(res);
  207. widestringmanager.Wide2AnsiMoveProc(PWideChar(S2),temp,Size);
  208. res:=temp;
  209. end;
  210. end;
  211. {$endif FPC_STRTOSHORTSTRINGPROC}
  212. Function fpc_ShortStr_To_WideStr (Const S2 : ShortString): WideString;compilerproc;
  213. {
  214. Converts a ShortString to a WideString;
  215. }
  216. Var
  217. Size : SizeInt;
  218. begin
  219. result:='';
  220. Size:=Length(S2);
  221. if Size>0 then
  222. begin
  223. widestringmanager.Ansi2WideMoveProc(PChar(@S2[1]),result,Size);
  224. { Terminating Zero }
  225. PWideChar(Pointer(fpc_ShortStr_To_WideStr)+Size*sizeof(WideChar))^:=#0;
  226. end;
  227. end;
  228. Function fpc_WideStr_To_AnsiStr (const S2 : WideString): AnsiString; compilerproc;
  229. {
  230. Converts a WideString to an AnsiString
  231. }
  232. Var
  233. Size : SizeInt;
  234. begin
  235. result:='';
  236. Size:=Length(S2);
  237. if Size>0 then
  238. widestringmanager.Wide2AnsiMoveProc(PWideChar(Pointer(S2)),result,Size);
  239. end;
  240. Function fpc_AnsiStr_To_WideStr (Const S2 : AnsiString): WideString; compilerproc;
  241. {
  242. Converts an AnsiString to a WideString;
  243. }
  244. Var
  245. Size : SizeInt;
  246. begin
  247. result:='';
  248. Size:=Length(S2);
  249. if Size>0 then
  250. widestringmanager.Ansi2WideMoveProc(PChar(S2),result,Size);
  251. end;
  252. Function fpc_PWideChar_To_WideStr(const p : pwidechar): widestring; compilerproc;
  253. var
  254. Size : SizeInt;
  255. begin
  256. result:='';
  257. if p=nil then
  258. exit;
  259. Size := IndexWord(p^, -1, 0);
  260. Setlength(result,Size);
  261. if Size>0 then
  262. begin
  263. Move(p^,PWideChar(Pointer(result))^,Size*sizeof(WideChar));
  264. { Terminating Zero }
  265. PWideChar(Pointer(result)+Size*sizeof(WideChar))^:=#0;
  266. end;
  267. end;
  268. { checked against the ansistring routine, 2001-05-27 (FK) }
  269. Procedure fpc_WideStr_Assign (Var S1 : Pointer;S2 : Pointer);[Public,Alias:'FPC_WIDESTR_ASSIGN']; compilerproc;
  270. {
  271. Assigns S2 to S1 (S1:=S2), taking in account reference counts.
  272. }
  273. begin
  274. if S1=S2 then exit;
  275. if S2<>nil then
  276. begin
  277. {$ifdef MSWINDOWS}
  278. if winwidestringalloc then
  279. begin
  280. if SysReAllocStringLen(S1, S2, Length(WideString(S2))) = 0 then
  281. WideStringError;
  282. end
  283. else
  284. {$endif MSWINDOWS}
  285. begin
  286. SetLength(WideString(S1),length(WideString(S2)));
  287. move(s2^,s1^,(length(WideString(s1))+1)*sizeof(widechar));
  288. end;
  289. end
  290. else
  291. begin
  292. { Free S1 }
  293. fpc_widestr_decr_ref (S1);
  294. S1:=nil;
  295. end;
  296. end;
  297. { alias for internal use }
  298. Procedure fpc_WideStr_Assign (Var S1 : Pointer;S2 : Pointer);[external name 'FPC_WIDESTR_ASSIGN'];
  299. {$ifndef STR_CONCAT_PROCS}
  300. function fpc_WideStr_Concat (const S1,S2 : WideString): WideString; compilerproc;
  301. Var
  302. Size,Location : SizeInt;
  303. pc : pwidechar;
  304. begin
  305. { only assign if s1 or s2 is empty }
  306. if (S1='') then
  307. begin
  308. result:=s2;
  309. exit;
  310. end;
  311. if (S2='') then
  312. begin
  313. result:=s1;
  314. exit;
  315. end;
  316. Location:=Length(S1);
  317. Size:=length(S2);
  318. SetLength(result,Size+Location);
  319. pc:=pwidechar(result);
  320. Move(S1[1],pc^,Location*sizeof(WideChar));
  321. inc(pc,location);
  322. Move(S2[1],pc^,(Size+1)*sizeof(WideChar));
  323. end;
  324. function fpc_WideStr_Concat_multi (const sarr:array of Widestring): widestring; compilerproc;
  325. Var
  326. i : Longint;
  327. p : pointer;
  328. pc : pwidechar;
  329. Size,NewSize : SizeInt;
  330. begin
  331. { First calculate size of the result so we can do
  332. a single call to SetLength() }
  333. NewSize:=0;
  334. for i:=low(sarr) to high(sarr) do
  335. inc(Newsize,length(sarr[i]));
  336. SetLength(result,NewSize);
  337. pc:=pwidechar(result);
  338. for i:=low(sarr) to high(sarr) do
  339. begin
  340. p:=pointer(sarr[i]);
  341. if assigned(p) then
  342. begin
  343. Size:=length(widestring(p));
  344. Move(pwidechar(p)^,pc^,(Size+1)*sizeof(WideChar));
  345. inc(pc,size);
  346. end;
  347. end;
  348. end;
  349. {$else STR_CONCAT_PROCS}
  350. procedure fpc_WideStr_Concat (var DestS:Widestring;const S1,S2 : WideString); compilerproc;
  351. Var
  352. Size,Location : SizeInt;
  353. same : boolean;
  354. begin
  355. { only assign if s1 or s2 is empty }
  356. if (S1='') then
  357. begin
  358. DestS:=s2;
  359. exit;
  360. end;
  361. if (S2='') then
  362. begin
  363. DestS:=s1;
  364. exit;
  365. end;
  366. Location:=Length(S1);
  367. Size:=length(S2);
  368. { Use Pointer() typecasts to prevent extra conversion code }
  369. if Pointer(DestS)=Pointer(S1) then
  370. begin
  371. same:=Pointer(S1)=Pointer(S2);
  372. SetLength(DestS,Size+Location);
  373. if same then
  374. Move(Pointer(DestS)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size)*sizeof(WideChar))
  375. else
  376. Move(Pointer(S2)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  377. end
  378. else if Pointer(DestS)=Pointer(S2) then
  379. begin
  380. SetLength(DestS,Size+Location);
  381. Move(Pointer(DestS)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  382. Move(Pointer(S1)^,Pointer(DestS)^,Location*sizeof(WideChar));
  383. end
  384. else
  385. begin
  386. DestS:='';
  387. SetLength(DestS,Size+Location);
  388. Move(Pointer(S1)^,Pointer(DestS)^,Location*sizeof(WideChar));
  389. Move(Pointer(S2)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  390. end;
  391. end;
  392. procedure fpc_WideStr_Concat_multi (var DestS:Widestring;const sarr:array of Widestring); compilerproc;
  393. Var
  394. i : Longint;
  395. p,pc : pointer;
  396. Size,NewLen : SizeInt;
  397. DestTmp : Widestring;
  398. begin
  399. if high(sarr)=0 then
  400. begin
  401. DestS:='';
  402. exit;
  403. end;
  404. { First calculate size of the result so we can do
  405. a single call to SetLength() }
  406. NewLen:=0;
  407. for i:=low(sarr) to high(sarr) do
  408. inc(NewLen,length(sarr[i]));
  409. SetLength(DestTmp,NewLen);
  410. pc:=pwidechar(DestTmp);
  411. for i:=low(sarr) to high(sarr) do
  412. begin
  413. p:=pointer(sarr[i]);
  414. if assigned(p) then
  415. begin
  416. Size:=length(widestring(p));
  417. Move(p^,pc^,(Size+1)*sizeof(WideChar));
  418. inc(pc,size*sizeof(WideChar));
  419. end;
  420. end;
  421. DestS:=DestTmp;
  422. end;
  423. {$endif STR_CONCAT_PROCS}
  424. Function fpc_Char_To_WideStr(const c : Char): WideString; compilerproc;
  425. {
  426. Converts a Char to a WideString;
  427. }
  428. begin
  429. Setlength(fpc_Char_To_WideStr,1);
  430. fpc_Char_To_WideStr[1]:=c;
  431. { Terminating Zero }
  432. PWideChar(Pointer(fpc_Char_To_WideStr)+sizeof(WideChar))^:=#0;
  433. end;
  434. Function fpc_WChar_To_WideStr(const c : WideChar): WideString; compilerproc;
  435. {
  436. Converts a WideChar to a WideString;
  437. }
  438. begin
  439. Setlength (fpc_WChar_To_WideStr,1);
  440. fpc_WChar_To_WideStr[1]:= c;
  441. end;
  442. Function fpc_WChar_To_AnsiStr(const c : WideChar): AnsiString; compilerproc;
  443. {
  444. Converts a WideChar to a AnsiString;
  445. }
  446. begin
  447. widestringmanager.Wide2AnsiMoveProc(@c, fpc_WChar_To_AnsiStr, 1);
  448. end;
  449. Function fpc_UChar_To_WideStr(const c : WideChar): WideString; compilerproc;
  450. {
  451. Converts a WideChar to a WideString;
  452. }
  453. begin
  454. Setlength (fpc_UChar_To_WideStr,1);
  455. fpc_UChar_To_WideStr[1]:= c;
  456. end;
  457. Function fpc_PChar_To_WideStr(const p : pchar): WideString; compilerproc;
  458. Var
  459. L : SizeInt;
  460. begin
  461. if (not assigned(p)) or (p[0]=#0) Then
  462. begin
  463. fpc_pchar_to_widestr := '';
  464. exit;
  465. end;
  466. l:=IndexChar(p^,-1,#0);
  467. widestringmanager.Ansi2WideMoveProc(P,fpc_PChar_To_WideStr,l);
  468. end;
  469. Function fpc_CharArray_To_WideStr(const arr: array of char; zerobased: boolean = true): WideString; compilerproc;
  470. var
  471. i : SizeInt;
  472. begin
  473. if (zerobased) then
  474. begin
  475. if (arr[0]=#0) Then
  476. begin
  477. fpc_chararray_to_widestr := '';
  478. exit;
  479. end;
  480. i:=IndexChar(arr,high(arr)+1,#0);
  481. if i = -1 then
  482. i := high(arr)+1;
  483. end
  484. else
  485. i := high(arr)+1;
  486. SetLength(fpc_CharArray_To_WideStr,i);
  487. widestringmanager.Ansi2WideMoveProc (pchar(@arr),fpc_CharArray_To_WideStr,i);
  488. end;
  489. {$ifndef FPC_STRTOCHARARRAYPROC}
  490. { inside the compiler, the resulttype is modified to that of the actual }
  491. { chararray we're converting to (JM) }
  492. function fpc_widestr_to_chararray(arraysize: SizeInt; const src: WideString): fpc_big_chararray;[public,alias: 'FPC_WIDESTR_TO_CHARARRAY']; compilerproc;
  493. var
  494. len: SizeInt;
  495. temp: ansistring;
  496. begin
  497. len := length(src);
  498. { make sure we don't dereference src if it can be nil (JM) }
  499. if len > 0 then
  500. widestringmanager.wide2ansimoveproc(pwidechar(@src[1]),temp,len);
  501. len := length(temp);
  502. if len > arraysize then
  503. len := arraysize;
  504. {$r-}
  505. move(temp[1],fpc_widestr_to_chararray[0],len);
  506. fillchar(fpc_widestr_to_chararray[len],arraysize-len,0);
  507. {$ifdef RangeCheckWasOn}
  508. {$r+}
  509. {$endif}
  510. end;
  511. { inside the compiler, the resulttype is modified to that of the actual }
  512. { widechararray we're converting to (JM) }
  513. function fpc_widestr_to_widechararray(arraysize: SizeInt; const src: WideString): fpc_big_widechararray;[public,alias: 'FPC_WIDESTR_TO_WIDECHARARRAY']; compilerproc;
  514. var
  515. len: SizeInt;
  516. begin
  517. len := length(src);
  518. if len > arraysize then
  519. len := arraysize;
  520. {$r-}
  521. { make sure we don't try to access element 1 of the ansistring if it's nil }
  522. if len > 0 then
  523. move(src[1],fpc_widestr_to_widechararray[0],len*SizeOf(WideChar));
  524. fillchar(fpc_widestr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  525. {$ifdef RangeCheckWasOn}
  526. {$r+}
  527. {$endif}
  528. end;
  529. { inside the compiler, the resulttype is modified to that of the actual }
  530. { chararray we're converting to (JM) }
  531. function fpc_ansistr_to_widechararray(arraysize: SizeInt; const src: AnsiString): fpc_big_widechararray;[public,alias: 'FPC_ANSISTR_TO_WIDECHARARRAY']; compilerproc;
  532. var
  533. len: SizeInt;
  534. temp: widestring;
  535. begin
  536. len := length(src);
  537. { make sure we don't dereference src if it can be nil (JM) }
  538. if len > 0 then
  539. widestringmanager.ansi2widemoveproc(pchar(@src[1]),temp,len);
  540. len := length(temp);
  541. if len > arraysize then
  542. len := arraysize;
  543. {$r-}
  544. move(temp[1],fpc_ansistr_to_widechararray[0],len*sizeof(widechar));
  545. fillchar(fpc_ansistr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  546. {$ifdef RangeCheckWasOn}
  547. {$r+}
  548. {$endif}
  549. end;
  550. function fpc_shortstr_to_widechararray(arraysize: SizeInt; const src: ShortString): fpc_big_widechararray;[public,alias: 'FPC_SHORTSTR_TO_WIDECHARARRAY']; compilerproc;
  551. var
  552. len: longint;
  553. temp : widestring;
  554. begin
  555. len := length(src);
  556. { make sure we don't access char 1 if length is 0 (JM) }
  557. if len > 0 then
  558. widestringmanager.ansi2widemoveproc(pchar(@src[1]),temp,len);
  559. len := length(temp);
  560. if len > arraysize then
  561. len := arraysize;
  562. {$r-}
  563. move(temp[1],fpc_shortstr_to_widechararray[0],len*sizeof(widechar));
  564. fillchar(fpc_shortstr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  565. {$ifdef RangeCheckWasOn}
  566. {$r+}
  567. {$endif}
  568. end;
  569. {$else ndef FPC_STRTOCHARARRAYPROC}
  570. procedure fpc_widestr_to_chararray(out res: array of char; const src: WideString); compilerproc;
  571. var
  572. len: SizeInt;
  573. temp: ansistring;
  574. begin
  575. len := length(src);
  576. { make sure we don't dereference src if it can be nil (JM) }
  577. if len > 0 then
  578. widestringmanager.wide2ansimoveproc(pwidechar(@src[1]),temp,len);
  579. len := length(temp);
  580. if len > length(res) then
  581. len := length(res);
  582. {$r-}
  583. move(temp[1],res[0],len);
  584. fillchar(res[len],length(res)-len,0);
  585. {$ifdef RangeCheckWasOn}
  586. {$r+}
  587. {$endif}
  588. end;
  589. procedure fpc_widestr_to_widechararray(out res: array of widechar; const src: WideString); compilerproc;
  590. var
  591. len: SizeInt;
  592. begin
  593. len := length(src);
  594. if len > length(res) then
  595. len := length(res);
  596. {$r-}
  597. { make sure we don't try to access element 1 of the ansistring if it's nil }
  598. if len > 0 then
  599. move(src[1],res[0],len*SizeOf(WideChar));
  600. fillchar(res[len],(length(res)-len)*SizeOf(WideChar),0);
  601. {$ifdef RangeCheckWasOn}
  602. {$r+}
  603. {$endif}
  604. end;
  605. {$endif ndef FPC_STRTOCHARARRAYPROC}
  606. Function fpc_WideStr_Compare(const S1,S2 : WideString): SizeInt;[Public,Alias : 'FPC_WIDESTR_COMPARE']; compilerproc;
  607. {
  608. Compares 2 WideStrings;
  609. The result is
  610. <0 if S1<S2
  611. 0 if S1=S2
  612. >0 if S1>S2
  613. }
  614. Var
  615. MaxI,Temp : SizeInt;
  616. begin
  617. if pointer(S1)=pointer(S2) then
  618. begin
  619. fpc_WideStr_Compare:=0;
  620. exit;
  621. end;
  622. Maxi:=Length(S1);
  623. temp:=Length(S2);
  624. If MaxI>Temp then
  625. MaxI:=Temp;
  626. Temp:=CompareWord(S1[1],S2[1],MaxI);
  627. if temp=0 then
  628. temp:=Length(S1)-Length(S2);
  629. fpc_WideStr_Compare:=Temp;
  630. end;
  631. Function fpc_WideStr_Compare_Equal(const S1,S2 : WideString): SizeInt;[Public,Alias : 'FPC_WIDESTR_COMPARE_EQUAL']; compilerproc;
  632. {
  633. Compares 2 WideStrings for equality only;
  634. The result is
  635. 0 if S1=S2
  636. <>0 if S1<>S2
  637. }
  638. Var
  639. MaxI : SizeInt;
  640. begin
  641. if pointer(S1)=pointer(S2) then
  642. exit(0);
  643. Maxi:=Length(S1);
  644. If MaxI<>Length(S2) then
  645. exit(-1)
  646. else
  647. exit(CompareWord(S1[1],S2[1],MaxI));
  648. end;
  649. Procedure fpc_WideStr_CheckZero(p : pointer);[Public,Alias : 'FPC_WIDESTR_CHECKZERO']; compilerproc;
  650. begin
  651. if p=nil then
  652. HandleErrorFrame(201,get_frame);
  653. end;
  654. Procedure fpc_WideStr_CheckRange(len,index : SizeInt);[Public,Alias : 'FPC_WIDESTR_RANGECHECK']; compilerproc;
  655. begin
  656. if (index>len div 2) or (Index<1) then
  657. HandleErrorFrame(201,get_frame);
  658. end;
  659. Procedure fpc_WideStr_SetLength(Var S : WideString; l : SizeInt);[Public,Alias : 'FPC_WIDESTR_SETLENGTH']; compilerproc;
  660. {
  661. Sets The length of string S to L.
  662. Makes sure S is unique, and contains enough room.
  663. }
  664. Var
  665. Temp : Pointer;
  666. movelen: SizeInt;
  667. begin
  668. if (l>0) then
  669. begin
  670. if Pointer(S)=nil then
  671. begin
  672. { Need a complete new string...}
  673. Pointer(s):=NewWideString(l);
  674. end
  675. { windows doesn't support reallocing widestrings, this code
  676. is anyways subject to be removed because widestrings shouldn't be
  677. ref. counted anymore (FK) }
  678. else
  679. if
  680. {$ifdef MSWINDOWS}
  681. not winwidestringalloc and
  682. {$endif MSWINDOWS}
  683. True
  684. then
  685. begin
  686. Dec(Pointer(S),WideFirstOff);
  687. if SizeUInt(L*sizeof(WideChar)+WideRecLen)>MemSize(Pointer(S)) then
  688. reallocmem(pointer(S), L*sizeof(WideChar)+WideRecLen);
  689. Inc(Pointer(S), WideFirstOff);
  690. end
  691. else
  692. begin
  693. { Reallocation is needed... }
  694. Temp:=Pointer(NewWideString(L));
  695. if Length(S)>0 then
  696. begin
  697. if l < succ(length(s)) then
  698. movelen := l
  699. { also move terminating null }
  700. else
  701. movelen := succ(length(s));
  702. Move(Pointer(S)^,Temp^,movelen * Sizeof(WideChar));
  703. end;
  704. fpc_widestr_decr_ref(Pointer(S));
  705. Pointer(S):=Temp;
  706. end;
  707. { Force nil termination in case it gets shorter }
  708. PWord(Pointer(S)+l*sizeof(WideChar))^:=0;
  709. {$ifdef MSWINDOWS}
  710. if not winwidestringalloc then
  711. {$endif MSWINDOWS}
  712. PWideRec(Pointer(S)-WideFirstOff)^.Len:=l*sizeof(WideChar);
  713. end
  714. else
  715. begin
  716. { Length=0 }
  717. if Pointer(S)<>nil then
  718. fpc_widestr_decr_ref (Pointer(S));
  719. Pointer(S):=Nil;
  720. end;
  721. end;
  722. {*****************************************************************************
  723. Public functions, In interface.
  724. *****************************************************************************}
  725. Function fpc_widestr_Unique(Var S : Pointer): Pointer; [Public,Alias : 'FPC_WIDESTR_UNIQUE']; compilerproc;
  726. begin
  727. pointer(result) := pointer(s);
  728. end;
  729. Function Fpc_WideStr_Copy (Const S : WideString; Index,Size : SizeInt) : WideString;compilerproc;
  730. var
  731. ResultAddress : Pointer;
  732. begin
  733. ResultAddress:=Nil;
  734. dec(index);
  735. if Index < 0 then
  736. Index := 0;
  737. { Check Size. Accounts for Zero-length S, the double check is needed because
  738. Size can be maxint and will get <0 when adding index }
  739. if (Size>Length(S)) or
  740. (Index+Size>Length(S)) then
  741. Size:=Length(S)-Index;
  742. If Size>0 then
  743. begin
  744. If Index<0 Then
  745. Index:=0;
  746. ResultAddress:=Pointer(NewWideString (Size));
  747. if ResultAddress<>Nil then
  748. begin
  749. Move (PWideChar(S)[Index],ResultAddress^,Size*sizeof(WideChar));
  750. PWideRec(ResultAddress-WideFirstOff)^.Len:=Size*sizeof(WideChar);
  751. PWideChar(ResultAddress+Size*sizeof(WideChar))^:=#0;
  752. end;
  753. end;
  754. fpc_widestr_decr_ref(Pointer(fpc_widestr_copy));
  755. Pointer(fpc_widestr_Copy):=ResultAddress;
  756. end;
  757. Function Pos (Const Substr : WideString; Const Source : WideString) : SizeInt;
  758. var
  759. i,MaxLen : SizeInt;
  760. pc : pwidechar;
  761. begin
  762. Pos:=0;
  763. if Length(SubStr)>0 then
  764. begin
  765. MaxLen:=Length(source)-Length(SubStr);
  766. i:=0;
  767. pc:=@source[1];
  768. while (i<=MaxLen) do
  769. begin
  770. inc(i);
  771. if (SubStr[1]=pc^) and
  772. (CompareWord(Substr[1],pc^,Length(SubStr))=0) then
  773. begin
  774. Pos:=i;
  775. exit;
  776. end;
  777. inc(pc);
  778. end;
  779. end;
  780. end;
  781. { Faster version for a widechar alone }
  782. Function Pos (c : WideChar; Const s : WideString) : SizeInt;
  783. var
  784. i: SizeInt;
  785. pc : pwidechar;
  786. begin
  787. pc:=@s[1];
  788. for i:=1 to length(s) do
  789. begin
  790. if pc^=c then
  791. begin
  792. pos:=i;
  793. exit;
  794. end;
  795. inc(pc);
  796. end;
  797. pos:=0;
  798. end;
  799. Function Pos (c : WideChar; Const s : AnsiString) : SizeInt;
  800. begin
  801. result:=Pos(c,WideString(s));
  802. end;
  803. Function Pos (c : AnsiString; Const s : WideString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  804. begin
  805. result:=Pos(WideString(c),s);
  806. end;
  807. Function Pos (c : ShortString; Const s : WideString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  808. begin
  809. result:=Pos(WideString(c),s);
  810. end;
  811. Function Pos (c : WideString; Const s : AnsiString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  812. begin
  813. result:=Pos(c,WideString(s));
  814. end;
  815. { Faster version for a char alone. Must be implemented because }
  816. { pos(c: char; const s: shortstring) also exists, so otherwise }
  817. { using pos(char,pchar) will always call the shortstring version }
  818. { (exact match for first argument), also with $h+ (JM) }
  819. Function Pos (c : Char; Const s : WideString) : SizeInt;
  820. var
  821. i: SizeInt;
  822. wc : widechar;
  823. pc : pwidechar;
  824. begin
  825. wc:=c;
  826. pc:=@s[1];
  827. for i:=1 to length(s) do
  828. begin
  829. if pc^=wc then
  830. begin
  831. pos:=i;
  832. exit;
  833. end;
  834. inc(pc);
  835. end;
  836. pos:=0;
  837. end;
  838. Procedure Delete (Var S : WideString; Index,Size: SizeInt);
  839. Var
  840. LS : SizeInt;
  841. begin
  842. LS:=Length(S);
  843. if (Index>LS) or (Index<=0) or (Size<=0) then
  844. exit;
  845. UniqueString (S);
  846. { (Size+Index) will overflow if Size=MaxInt. }
  847. if Size>LS-Index then
  848. Size:=LS-Index+1;
  849. if Size<=LS-Index then
  850. begin
  851. Dec(Index);
  852. Move(PWideChar(S)[Index+Size],PWideChar(S)[Index],(LS-Index-Size+1)*sizeof(WideChar));
  853. end;
  854. Setlength(s,LS-Size);
  855. end;
  856. Procedure Insert (Const Source : WideString; Var S : WideString; Index : SizeInt);
  857. var
  858. Temp : WideString;
  859. LS : SizeInt;
  860. begin
  861. If Length(Source)=0 then
  862. exit;
  863. if index <= 0 then
  864. index := 1;
  865. Ls:=Length(S);
  866. if index > LS then
  867. index := LS+1;
  868. Dec(Index);
  869. Pointer(Temp) := NewWideString(Length(Source)+LS);
  870. SetLength(Temp,Length(Source)+LS);
  871. If Index>0 then
  872. move (PWideChar(S)^,PWideChar(Temp)^,Index*sizeof(WideChar));
  873. Move (PWideChar(Source)^,PWideChar(Temp)[Index],Length(Source)*sizeof(WideChar));
  874. If (LS-Index)>0 then
  875. Move(PWideChar(S)[Index],PWideChar(temp)[Length(Source)+index],(LS-Index)*sizeof(WideChar));
  876. S:=Temp;
  877. end;
  878. function UpCase(const s : WideString) : WideString;
  879. begin
  880. result:=widestringmanager.UpperWideStringProc(s);
  881. end;
  882. Procedure SetString (Out S : WideString; Buf : PWideChar; Len : SizeInt);
  883. begin
  884. SetLength(S,Len);
  885. If (Buf<>Nil) and (Len>0) then
  886. Move (Buf[0],S[1],Len*sizeof(WideChar));
  887. end;
  888. Procedure SetString (Out S : WideString; Buf : PChar; Len : SizeInt);
  889. begin
  890. SetLength(S,Len);
  891. If (Buf<>Nil) and (Len>0) then
  892. widestringmanager.Ansi2WideMoveProc(Buf,S,Len);
  893. end;
  894. {$ifndef FPUNONE}
  895. Function fpc_Val_Real_WideStr(Const S : WideString; out Code : ValSInt): ValReal; [public, alias:'FPC_VAL_REAL_WIDESTR']; compilerproc;
  896. Var
  897. SS : String;
  898. begin
  899. fpc_Val_Real_WideStr := 0;
  900. if length(S) > 255 then
  901. code := 256
  902. else
  903. begin
  904. SS := S;
  905. Val(SS,fpc_Val_Real_WideStr,code);
  906. end;
  907. end;
  908. {$endif}
  909. function fpc_val_enum_widestr(str2ordindex:pointer;const s:widestring;out code:valsint):longint;compilerproc;
  910. var ss:shortstring;
  911. begin
  912. if length(s)>255 then
  913. code:=256
  914. else
  915. begin
  916. ss:=s;
  917. val(ss,fpc_val_enum_widestr,code);
  918. end;
  919. end;
  920. Function fpc_Val_Currency_WideStr(Const S : WideString; out Code : ValSInt): Currency; [public, alias:'FPC_VAL_CURRENCY_WIDESTR']; compilerproc;
  921. Var
  922. SS : String;
  923. begin
  924. if length(S) > 255 then
  925. begin
  926. fpc_Val_Currency_WideStr:=0;
  927. code := 256;
  928. end
  929. else
  930. begin
  931. SS := S;
  932. Val(SS,fpc_Val_Currency_WideStr,code);
  933. end;
  934. end;
  935. Function fpc_Val_UInt_WideStr (Const S : WideString; out Code : ValSInt): ValUInt; [public, alias:'FPC_VAL_UINT_WIDESTR']; compilerproc;
  936. Var
  937. SS : ShortString;
  938. begin
  939. fpc_Val_UInt_WideStr := 0;
  940. if length(S) > 255 then
  941. code := 256
  942. else
  943. begin
  944. SS := S;
  945. Val(SS,fpc_Val_UInt_WideStr,code);
  946. end;
  947. end;
  948. Function fpc_Val_SInt_WideStr (DestSize: SizeInt; Const S : WideString; out Code : ValSInt): ValSInt; [public, alias:'FPC_VAL_SINT_WIDESTR']; compilerproc;
  949. Var
  950. SS : ShortString;
  951. begin
  952. fpc_Val_SInt_WideStr:=0;
  953. if length(S)>255 then
  954. code:=256
  955. else
  956. begin
  957. SS := S;
  958. fpc_Val_SInt_WideStr := int_Val_SInt_ShortStr(DestSize,SS,Code);
  959. end;
  960. end;
  961. {$ifndef CPU64}
  962. Function fpc_Val_qword_WideStr (Const S : WideString; out Code : ValSInt): qword; [public, alias:'FPC_VAL_QWORD_WIDESTR']; compilerproc;
  963. Var
  964. SS : ShortString;
  965. begin
  966. fpc_Val_qword_WideStr:=0;
  967. if length(S)>255 then
  968. code:=256
  969. else
  970. begin
  971. SS := S;
  972. Val(SS,fpc_Val_qword_WideStr,Code);
  973. end;
  974. end;
  975. Function fpc_Val_int64_WideStr (Const S : WideString; out Code : ValSInt): Int64; [public, alias:'FPC_VAL_INT64_WIDESTR']; compilerproc;
  976. Var
  977. SS : ShortString;
  978. begin
  979. fpc_Val_int64_WideStr:=0;
  980. if length(S)>255 then
  981. code:=256
  982. else
  983. begin
  984. SS := S;
  985. Val(SS,fpc_Val_int64_WideStr,Code);
  986. end;
  987. end;
  988. {$endif CPU64}
  989. {$ifndef FPUNONE}
  990. procedure fpc_WideStr_Float(d : ValReal;len,fr,rt : SizeInt;out s : WideString);compilerproc;
  991. var
  992. ss : shortstring;
  993. begin
  994. str_real(len,fr,d,treal_type(rt),ss);
  995. s:=ss;
  996. end;
  997. {$endif}
  998. procedure fpc_widestr_enum(ordinal,len:sizeint;typinfo,ord2strindex:pointer;out s:widestring);compilerproc;
  999. var ss:shortstring;
  1000. begin
  1001. fpc_shortstr_enum(ordinal,len,typinfo,ord2strindex,ss);
  1002. s:=ss;
  1003. end;
  1004. {$ifdef FPC_HAS_STR_CURRENCY}
  1005. procedure fpc_WideStr_Currency(c : Currency;len,fr : SizeInt;out s : WideString);compilerproc;
  1006. var
  1007. ss : shortstring;
  1008. begin
  1009. str(c:len:fr,ss);
  1010. s:=ss;
  1011. end;
  1012. {$endif FPC_HAS_STR_CURRENCY}
  1013. Procedure fpc_WideStr_SInt(v : ValSint; Len : SizeInt; out S : WideString);compilerproc;
  1014. Var
  1015. SS : ShortString;
  1016. begin
  1017. Str (v:Len,SS);
  1018. S:=SS;
  1019. end;
  1020. Procedure fpc_WideStr_UInt(v : ValUInt;Len : SizeInt; out S : WideString);compilerproc;
  1021. Var
  1022. SS : ShortString;
  1023. begin
  1024. str(v:Len,SS);
  1025. S:=SS;
  1026. end;
  1027. {$ifndef CPU64}
  1028. Procedure fpc_WideStr_Int64(v : Int64; Len : SizeInt; out S : WideString);compilerproc;
  1029. Var
  1030. SS : ShortString;
  1031. begin
  1032. Str (v:Len,SS);
  1033. S:=SS;
  1034. end;
  1035. Procedure fpc_WideStr_Qword(v : Qword;Len : SizeInt; out S : WideString);compilerproc;
  1036. Var
  1037. SS : ShortString;
  1038. begin
  1039. str(v:Len,SS);
  1040. S:=SS;
  1041. end;
  1042. {$endif CPU64}
  1043. { converts an utf-16 code point or surrogate pair to utf-32 }
  1044. function utf16toutf32(const S: WideString; const index: SizeInt; out len: longint): UCS4Char; [public, alias: 'FPC_WIDETOUTF32'];
  1045. var
  1046. w: widechar;
  1047. begin
  1048. { UTF-16 points in the range #$0-#$D7FF and #$E000-#$FFFF }
  1049. { are the same in UTF-32 }
  1050. w:=s[index];
  1051. if (w<=#$d7ff) or
  1052. (w>=#$e000) then
  1053. begin
  1054. result:=UCS4Char(w);
  1055. len:=1;
  1056. end
  1057. { valid surrogate pair? }
  1058. else if (w<=#$dbff) and
  1059. { w>=#$d7ff check not needed, checked above }
  1060. (index<length(s)) and
  1061. (s[index+1]>=#$dc00) and
  1062. (s[index+1]<=#$dfff) then
  1063. { convert the surrogate pair to UTF-32 }
  1064. begin
  1065. result:=(UCS4Char(w)-$d800) shl 10 + (UCS4Char(s[index+1])-$dc00) + $10000;
  1066. len:=2;
  1067. end
  1068. else
  1069. { invalid surrogate -> do nothing }
  1070. begin
  1071. result:=UCS4Char(w);
  1072. len:=1;
  1073. end;
  1074. end;
  1075. function UnicodeToUtf8(Dest: PChar; Source: PWideChar; MaxBytes: SizeInt): SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  1076. begin
  1077. if assigned(Source) then
  1078. Result:=UnicodeToUtf8(Dest,MaxBytes,Source,IndexWord(Source^,-1,0))
  1079. else
  1080. Result:=0;
  1081. end;
  1082. function UnicodeToUtf8(Dest: PChar; MaxDestBytes: SizeUInt; Source: PWideChar; SourceChars: SizeUInt): SizeUInt;
  1083. var
  1084. i,j : SizeUInt;
  1085. w : word;
  1086. lw : longword;
  1087. len : longint;
  1088. begin
  1089. result:=0;
  1090. if source=nil then
  1091. exit;
  1092. i:=0;
  1093. j:=0;
  1094. if assigned(Dest) then
  1095. begin
  1096. while (i<SourceChars) and (j<MaxDestBytes) do
  1097. begin
  1098. w:=word(Source[i]);
  1099. case w of
  1100. 0..$7f:
  1101. begin
  1102. Dest[j]:=char(w);
  1103. inc(j);
  1104. end;
  1105. $80..$7ff:
  1106. begin
  1107. if j+1>=MaxDestBytes then
  1108. break;
  1109. Dest[j]:=char($c0 or (w shr 6));
  1110. Dest[j+1]:=char($80 or (w and $3f));
  1111. inc(j,2);
  1112. end;
  1113. $800..$d7ff,$e000..$ffff:
  1114. begin
  1115. if j+2>=MaxDestBytes then
  1116. break;
  1117. Dest[j]:=char($e0 or (w shr 12));
  1118. Dest[j+1]:=char($80 or ((w shr 6) and $3f));
  1119. Dest[j+2]:=char($80 or (w and $3f));
  1120. inc(j,3);
  1121. end;
  1122. $d800..$dbff:
  1123. {High Surrogates}
  1124. begin
  1125. if j+3>=MaxDestBytes then
  1126. break;
  1127. if (i<sourcechars-1) and
  1128. (word(Source[i+1]) >= $dc00) and
  1129. (word(Source[i+1]) <= $dfff) then
  1130. begin
  1131. lw:=longword(utf16toutf32(Source[i] + Source[i+1], 1, len));
  1132. Dest[j]:=char($f0 or (lw shr 18));
  1133. Dest[j+1]:=char($80 or ((lw shr 12) and $3f));
  1134. Dest[j+2]:=char($80 or ((lw shr 6) and $3f));
  1135. Dest[j+3]:=char($80 or (lw and $3f));
  1136. inc(j,4);
  1137. inc(i);
  1138. end;
  1139. end;
  1140. end;
  1141. inc(i);
  1142. end;
  1143. if j>SizeUInt(MaxDestBytes-1) then
  1144. j:=MaxDestBytes-1;
  1145. Dest[j]:=#0;
  1146. end
  1147. else
  1148. begin
  1149. while i<SourceChars do
  1150. begin
  1151. case word(Source[i]) of
  1152. $0..$7f:
  1153. inc(j);
  1154. $80..$7ff:
  1155. inc(j,2);
  1156. $800..$d7ff,$e000..$ffff:
  1157. inc(j,3);
  1158. $d800..$dbff:
  1159. begin
  1160. if (i<sourcechars-1) and
  1161. (word(Source[i+1]) >= $dc00) and
  1162. (word(Source[i+1]) <= $dfff) then
  1163. begin
  1164. inc(j,4);
  1165. inc(i);
  1166. end;
  1167. end;
  1168. end;
  1169. inc(i);
  1170. end;
  1171. end;
  1172. result:=j+1;
  1173. end;
  1174. function Utf8ToUnicode(Dest: PWideChar; Source: PChar; MaxChars: SizeInt): SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  1175. begin
  1176. if assigned(Source) then
  1177. Result:=Utf8ToUnicode(Dest,MaxChars,Source,strlen(Source))
  1178. else
  1179. Result:=0;
  1180. end;
  1181. function UTF8ToUnicode(Dest: PWideChar; MaxDestChars: SizeUInt; Source: PChar; SourceBytes: SizeUInt): SizeUInt;
  1182. const
  1183. UNICODE_INVALID=63;
  1184. var
  1185. InputUTF8: SizeUInt;
  1186. IBYTE: BYTE;
  1187. OutputUnicode: SizeUInt;
  1188. PRECHAR: SizeUInt;
  1189. TempBYTE: BYTE;
  1190. CharLen: SizeUint;
  1191. LookAhead: SizeUInt;
  1192. UC: SizeUInt;
  1193. begin
  1194. if not assigned(Source) then
  1195. begin
  1196. result:=0;
  1197. exit;
  1198. end;
  1199. result:=SizeUInt(-1);
  1200. InputUTF8:=0;
  1201. OutputUnicode:=0;
  1202. PreChar:=0;
  1203. if Assigned(Dest) Then
  1204. begin
  1205. while (OutputUnicode<MaxDestChars) and (InputUTF8<SourceBytes) do
  1206. begin
  1207. IBYTE:=byte(Source[InputUTF8]);
  1208. if (IBYTE and $80) = 0 then
  1209. begin
  1210. //One character US-ASCII, convert it to unicode
  1211. if IBYTE = 10 then
  1212. begin
  1213. If (PreChar<>13) and FALSE then
  1214. begin
  1215. //Expand to crlf, conform UTF-8.
  1216. //This procedure will break the memory alocation by
  1217. //FPC for the widestring, so never use it. Condition never true due the "and FALSE".
  1218. if OutputUnicode+1<MaxDestChars then
  1219. begin
  1220. Dest[OutputUnicode]:=WideChar(13);
  1221. inc(OutputUnicode);
  1222. Dest[OutputUnicode]:=WideChar(10);
  1223. inc(OutputUnicode);
  1224. PreChar:=10;
  1225. end
  1226. else
  1227. begin
  1228. Dest[OutputUnicode]:=WideChar(13);
  1229. inc(OutputUnicode);
  1230. end;
  1231. end
  1232. else
  1233. begin
  1234. Dest[OutputUnicode]:=WideChar(IBYTE);
  1235. inc(OutputUnicode);
  1236. PreChar:=IBYTE;
  1237. end;
  1238. end
  1239. else
  1240. begin
  1241. Dest[OutputUnicode]:=WideChar(IBYTE);
  1242. inc(OutputUnicode);
  1243. PreChar:=IBYTE;
  1244. end;
  1245. inc(InputUTF8);
  1246. end
  1247. else
  1248. begin
  1249. TempByte:=IBYTE;
  1250. CharLen:=0;
  1251. while (TempBYTE and $80)<>0 do
  1252. begin
  1253. TempBYTE:=(TempBYTE shl 1) and $FE;
  1254. inc(CharLen);
  1255. end;
  1256. //Test for the "CharLen" conforms UTF-8 string
  1257. //This means the 10xxxxxx pattern.
  1258. if SizeUInt(InputUTF8+CharLen-1)>SourceBytes then
  1259. begin
  1260. //Insuficient chars in string to decode
  1261. //UTF-8 array. Fallback to single char.
  1262. CharLen:= 1;
  1263. end;
  1264. for LookAhead := 1 to CharLen-1 do
  1265. begin
  1266. if ((byte(Source[InputUTF8+LookAhead]) and $80)<>$80) or
  1267. ((byte(Source[InputUTF8+LookAhead]) and $40)<>$00) then
  1268. begin
  1269. //Invalid UTF-8 sequence, fallback.
  1270. CharLen:= LookAhead;
  1271. break;
  1272. end;
  1273. end;
  1274. UC:=$FFFF;
  1275. case CharLen of
  1276. 1: begin
  1277. //Not valid UTF-8 sequence
  1278. UC:=UNICODE_INVALID;
  1279. end;
  1280. 2: begin
  1281. //Two bytes UTF, convert it
  1282. UC:=(byte(Source[InputUTF8]) and $1F) shl 6;
  1283. UC:=UC or (byte(Source[InputUTF8+1]) and $3F);
  1284. if UC <= $7F then
  1285. begin
  1286. //Invalid UTF sequence.
  1287. UC:=UNICODE_INVALID;
  1288. end;
  1289. end;
  1290. 3: begin
  1291. //Three bytes, convert it to unicode
  1292. UC:= (byte(Source[InputUTF8]) and $0F) shl 12;
  1293. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 6);
  1294. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F));
  1295. if (UC <= $7FF) or (UC >= $FFFE) or ((UC >= $D800) and (UC <= $DFFF)) then
  1296. begin
  1297. //Invalid UTF-8 sequence
  1298. UC:= UNICODE_INVALID;
  1299. End;
  1300. end;
  1301. 4: begin
  1302. //Four bytes, convert it to two unicode characters
  1303. UC:= (byte(Source[InputUTF8]) and $07) shl 18;
  1304. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 12);
  1305. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F) shl 6);
  1306. UC:= UC or ((byte(Source[InputUTF8+3]) and $3F));
  1307. if (UC < $10000) or (UC > $10FFFF) then
  1308. begin
  1309. UC:= UNICODE_INVALID;
  1310. end
  1311. else
  1312. begin
  1313. { only store pair if room }
  1314. dec(UC,$10000);
  1315. if (OutputUnicode<MaxDestChars-1) then
  1316. begin
  1317. Dest[OutputUnicode]:=WideChar(UC shr 10 + $D800);
  1318. inc(OutputUnicode);
  1319. UC:=(UC and $3ff) + $DC00;
  1320. end
  1321. else
  1322. begin
  1323. InputUTF8:= InputUTF8 + CharLen;
  1324. { don't store anything }
  1325. CharLen:=0;
  1326. end;
  1327. end;
  1328. end;
  1329. 5,6,7: begin
  1330. //Invalid UTF8 to unicode conversion,
  1331. //mask it as invalid UNICODE too.
  1332. UC:=UNICODE_INVALID;
  1333. end;
  1334. end;
  1335. if CharLen > 0 then
  1336. begin
  1337. PreChar:=UC;
  1338. Dest[OutputUnicode]:=WideChar(UC);
  1339. inc(OutputUnicode);
  1340. end;
  1341. InputUTF8:= InputUTF8 + CharLen;
  1342. end;
  1343. end;
  1344. Result:=OutputUnicode+1;
  1345. end
  1346. else
  1347. begin
  1348. while (InputUTF8<SourceBytes) do
  1349. begin
  1350. IBYTE:=byte(Source[InputUTF8]);
  1351. if (IBYTE and $80) = 0 then
  1352. begin
  1353. //One character US-ASCII, convert it to unicode
  1354. if IBYTE = 10 then
  1355. begin
  1356. if (PreChar<>13) and FALSE then
  1357. begin
  1358. //Expand to crlf, conform UTF-8.
  1359. //This procedure will break the memory alocation by
  1360. //FPC for the widestring, so never use it. Condition never true due the "and FALSE".
  1361. inc(OutputUnicode,2);
  1362. PreChar:=10;
  1363. end
  1364. else
  1365. begin
  1366. inc(OutputUnicode);
  1367. PreChar:=IBYTE;
  1368. end;
  1369. end
  1370. else
  1371. begin
  1372. inc(OutputUnicode);
  1373. PreChar:=IBYTE;
  1374. end;
  1375. inc(InputUTF8);
  1376. end
  1377. else
  1378. begin
  1379. TempByte:=IBYTE;
  1380. CharLen:=0;
  1381. while (TempBYTE and $80)<>0 do
  1382. begin
  1383. TempBYTE:=(TempBYTE shl 1) and $FE;
  1384. inc(CharLen);
  1385. end;
  1386. //Test for the "CharLen" conforms UTF-8 string
  1387. //This means the 10xxxxxx pattern.
  1388. if SizeUInt(InputUTF8+CharLen-1)>SourceBytes then
  1389. begin
  1390. //Insuficient chars in string to decode
  1391. //UTF-8 array. Fallback to single char.
  1392. CharLen:= 1;
  1393. end;
  1394. for LookAhead := 1 to CharLen-1 do
  1395. begin
  1396. if ((byte(Source[InputUTF8+LookAhead]) and $80)<>$80) or
  1397. ((byte(Source[InputUTF8+LookAhead]) and $40)<>$00) then
  1398. begin
  1399. //Invalid UTF-8 sequence, fallback.
  1400. CharLen:= LookAhead;
  1401. break;
  1402. end;
  1403. end;
  1404. UC:=$FFFF;
  1405. case CharLen of
  1406. 1: begin
  1407. //Not valid UTF-8 sequence
  1408. UC:=UNICODE_INVALID;
  1409. end;
  1410. 2: begin
  1411. //Two bytes UTF, convert it
  1412. UC:=(byte(Source[InputUTF8]) and $1F) shl 6;
  1413. UC:=UC or (byte(Source[InputUTF8+1]) and $3F);
  1414. if UC <= $7F then
  1415. begin
  1416. //Invalid UTF sequence.
  1417. UC:=UNICODE_INVALID;
  1418. end;
  1419. end;
  1420. 3: begin
  1421. //Three bytes, convert it to unicode
  1422. UC:= (byte(Source[InputUTF8]) and $0F) shl 12;
  1423. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 6);
  1424. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F));
  1425. If (UC <= $7FF) or (UC >= $FFFE) or ((UC >= $D800) and (UC <= $DFFF)) then
  1426. begin
  1427. //Invalid UTF-8 sequence
  1428. UC:= UNICODE_INVALID;
  1429. end;
  1430. end;
  1431. 4: begin
  1432. //Four bytes, convert it to two unicode characters
  1433. UC:= (byte(Source[InputUTF8]) and $07) shl 18;
  1434. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 12);
  1435. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F) shl 6);
  1436. UC:= UC or ((byte(Source[InputUTF8+3]) and $3F));
  1437. if (UC < $10000) or (UC > $10FFFF) then
  1438. UC:= UNICODE_INVALID
  1439. else
  1440. { extra character character }
  1441. inc(OutputUnicode);
  1442. end;
  1443. 5,6,7: begin
  1444. //Invalid UTF8 to unicode conversion,
  1445. //mask it as invalid UNICODE too.
  1446. UC:=UNICODE_INVALID;
  1447. end;
  1448. end;
  1449. if CharLen > 0 then
  1450. begin
  1451. PreChar:=UC;
  1452. inc(OutputUnicode);
  1453. end;
  1454. InputUTF8:= InputUTF8 + CharLen;
  1455. end;
  1456. end;
  1457. Result:=OutputUnicode+1;
  1458. end;
  1459. end;
  1460. function UTF8Encode(const s : WideString) : UTF8String;
  1461. var
  1462. i : SizeInt;
  1463. hs : UTF8String;
  1464. begin
  1465. result:='';
  1466. if s='' then
  1467. exit;
  1468. SetLength(hs,length(s)*3);
  1469. i:=UnicodeToUtf8(pchar(hs),length(hs)+1,PWideChar(s),length(s));
  1470. if i>0 then
  1471. begin
  1472. SetLength(hs,i-1);
  1473. result:=hs;
  1474. end;
  1475. end;
  1476. const
  1477. SNoWidestrings = 'This binary has no widestrings support compiled in.';
  1478. SRecompileWithWidestrings = 'Recompile the application with a widestrings-manager in the program uses clause.';
  1479. procedure unimplementedwidestring;
  1480. begin
  1481. {$ifdef FPC_HAS_FEATURE_CONSOLEIO}
  1482. If IsConsole then
  1483. begin
  1484. Writeln(StdErr,SNoWidestrings);
  1485. Writeln(StdErr,SRecompileWithWidestrings);
  1486. end;
  1487. {$endif FPC_HAS_FEATURE_CONSOLEIO}
  1488. HandleErrorFrame(233,get_frame);
  1489. end;
  1490. {$warnings off}
  1491. function GenericWideCase(const s : WideString) : WideString;
  1492. begin
  1493. unimplementedwidestring;
  1494. end;
  1495. function CompareWideString(const s1, s2 : WideString) : PtrInt;
  1496. begin
  1497. unimplementedwidestring;
  1498. end;
  1499. function CompareTextWideString(const s1, s2 : WideString): PtrInt;
  1500. begin
  1501. unimplementedwidestring;
  1502. end;
  1503. {$warnings on}
  1504. function CharLengthPChar(const Str: PChar): PtrInt;forward;
  1505. procedure initwidestringmanager;
  1506. begin
  1507. fillchar(widestringmanager,sizeof(widestringmanager),0);
  1508. {$ifndef HAS_WIDESTRINGMANAGER}
  1509. widestringmanager.Wide2AnsiMoveProc:=@defaultWide2AnsiMove;
  1510. widestringmanager.Ansi2WideMoveProc:=@defaultAnsi2WideMove;
  1511. widestringmanager.UpperWideStringProc:=@GenericWideCase;
  1512. widestringmanager.LowerWideStringProc:=@GenericWideCase;
  1513. {$endif HAS_WIDESTRINGMANAGER}
  1514. widestringmanager.CompareWideStringProc:=@CompareWideString;
  1515. widestringmanager.CompareTextWideStringProc:=@CompareTextWideString;
  1516. widestringmanager.CharLengthPCharProc:=@CharLengthPChar;
  1517. end;