wstrings.inc 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694
  1. {
  2. This file is part of the Free Pascal run time library.
  3. Copyright (c) 1999-2005 by Florian Klaempfl,
  4. member of the Free Pascal development team.
  5. This file implements support routines for WideStrings with FPC
  6. See the file COPYING.FPC, included in this distribution,
  7. for details about the copyright.
  8. This program is distributed in the hope that it will be useful,
  9. but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11. **********************************************************************}
  12. {
  13. This file contains the implementation of the WideString type,
  14. and all things that are needed for it.
  15. WideString is defined as a 'silent' pwidechar :
  16. a pwidechar that points to :
  17. @-8 : SizeInt for reference count;
  18. @-4 : SizeInt for size; size=number of bytes, not the number of chars. Divide or multiply
  19. with sizeof(WideChar) to convert. This is needed to be compatible with Delphi and
  20. Windows COM BSTR.
  21. @ : String + Terminating #0;
  22. Pwidechar(Widestring) is a valid typecast.
  23. So WS[i] is converted to the address @WS+i-1.
  24. Constants should be assigned a reference count of -1
  25. Meaning that they can't be disposed of.
  26. }
  27. Type
  28. PWideRec = ^TWideRec;
  29. TWideRec = Packed Record
  30. Len : DWord;
  31. First : WideChar;
  32. end;
  33. Const
  34. WideRecLen = SizeOf(TWideRec);
  35. WideFirstOff = SizeOf(TWideRec)-sizeof(WideChar);
  36. {
  37. Default WideChar <-> Char conversion is to only convert the
  38. lower 127 chars, all others are translated to spaces.
  39. These routines can be overwritten for the Current Locale
  40. }
  41. procedure DefaultWide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
  42. var
  43. i : SizeInt;
  44. destp: PChar;
  45. begin
  46. setlength(dest,len);
  47. destp := PChar(Pointer(dest));
  48. for i:=1 to len do
  49. begin
  50. if word(source^)<256 then
  51. destp^:=char(word(source^))
  52. else
  53. destp^:='?';
  54. inc(source);
  55. inc(destp);
  56. end;
  57. end;
  58. procedure DefaultAnsi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
  59. var
  60. i : SizeInt;
  61. begin
  62. setlength(dest,len);
  63. for i:=1 to len do
  64. begin
  65. dest[i]:=widechar(byte(source^));
  66. inc(source);
  67. end;
  68. end;
  69. (*
  70. Procedure UniqueWideString(Var S : WideString); [Public,Alias : 'FPC_WIDESTR_UNIQUE'];
  71. {
  72. Make sure reference count of S is 1,
  73. using copy-on-write semantics.
  74. }
  75. begin
  76. end;
  77. *)
  78. {****************************************************************************
  79. Internal functions, not in interface.
  80. ****************************************************************************}
  81. procedure WideStringError;
  82. begin
  83. HandleErrorFrame(204,get_frame);
  84. end;
  85. {$ifdef WideStrDebug}
  86. Procedure DumpWideRec(S : Pointer);
  87. begin
  88. If S=Nil then
  89. Writeln ('String is nil')
  90. Else
  91. Begin
  92. With PWideRec(S-WideFirstOff)^ do
  93. begin
  94. Write ('(Len:',len);
  95. Writeln (' Ref: ',ref,')');
  96. end;
  97. end;
  98. end;
  99. {$endif}
  100. Function NewWideString(Len : SizeInt) : Pointer;
  101. {
  102. Allocate a new WideString on the heap.
  103. initialize it to zero length and reference count 1.
  104. }
  105. Var
  106. P : Pointer;
  107. begin
  108. {$ifdef MSWINDOWS}
  109. if winwidestringalloc then
  110. begin
  111. P:=SysAllocStringLen(nil,Len);
  112. if P=nil then
  113. WideStringError;
  114. end
  115. else
  116. {$endif MSWINDOWS}
  117. begin
  118. GetMem(P,Len*sizeof(WideChar)+WideRecLen);
  119. If P<>Nil then
  120. begin
  121. PWideRec(P)^.Len:=Len*2; { Initial length }
  122. PWideRec(P)^.First:=#0; { Terminating #0 }
  123. inc(p,WideFirstOff); { Points to string now }
  124. end
  125. else
  126. WideStringError;
  127. end;
  128. NewWideString:=P;
  129. end;
  130. Procedure DisposeWideString(Var S : Pointer);
  131. {
  132. Deallocates a WideString From the heap.
  133. }
  134. begin
  135. If S=Nil then
  136. exit;
  137. {$ifndef MSWINDOWS}
  138. Dec (S,WideFirstOff);
  139. Freemem(S);
  140. {$else MSWINDOWS}
  141. if winwidestringalloc then
  142. SysFreeString(S)
  143. else
  144. begin
  145. Dec (S,WideFirstOff);
  146. Freemem(S);
  147. end;
  148. {$endif MSWINDOWS}
  149. S:=Nil;
  150. end;
  151. Procedure fpc_WideStr_Decr_Ref (Var S : Pointer);[Public,Alias:'FPC_WIDESTR_DECR_REF']; compilerproc;
  152. {
  153. Decreases the ReferenceCount of a non constant widestring;
  154. If the reference count is zero, deallocate the string;
  155. }
  156. Begin
  157. DisposeWideString(S); { does test for nil }
  158. end;
  159. { alias for internal use }
  160. Procedure fpc_WideStr_Decr_Ref (Var S : Pointer);[external name 'FPC_WIDESTR_DECR_REF'];
  161. Procedure fpc_WideStr_Incr_Ref(Var S : Pointer);[Public,Alias:'FPC_WIDESTR_INCR_REF']; compilerproc;
  162. var
  163. p : pointer;
  164. Begin
  165. If S=Nil then
  166. exit;
  167. p:=NewWidestring(length(WideString(S)));
  168. move(s^,p^,(length(WideString(s))+1)*sizeof(widechar)); // double #0 too
  169. s:=p;
  170. end;
  171. { alias for internal use }
  172. Procedure fpc_WideStr_Incr_Ref (Var S : Pointer);[external name 'FPC_WIDESTR_INCR_REF'];
  173. {$ifndef FPC_STRTOSHORTSTRINGPROC}
  174. function fpc_WideStr_To_ShortStr (high_of_res: SizeInt;const S2 : WideString): shortstring;[Public, alias: 'FPC_WIDESTR_TO_SHORTSTR']; compilerproc;
  175. {
  176. Converts a WideString to a ShortString;
  177. }
  178. Var
  179. Size : SizeInt;
  180. temp : ansistring;
  181. begin
  182. result:='';
  183. Size:=Length(S2);
  184. if Size>0 then
  185. begin
  186. If Size>high_of_res then
  187. Size:=high_of_res;
  188. widestringmanager.Wide2AnsiMoveProc(PWideChar(S2),temp,Size);
  189. result:=temp;
  190. end;
  191. end;
  192. {$else FPC_STRTOSHORTSTRINGPROC}
  193. procedure fpc_WideStr_To_ShortStr (out res: ShortString;const S2 : WideString); [Public, alias: 'FPC_WIDESTR_TO_SHORTSTR'];compilerproc;
  194. {
  195. Converts a WideString to a ShortString;
  196. }
  197. Var
  198. Size : SizeInt;
  199. temp : ansistring;
  200. begin
  201. res:='';
  202. Size:=Length(S2);
  203. if Size>0 then
  204. begin
  205. If Size>high(res) then
  206. Size:=high(res);
  207. widestringmanager.Wide2AnsiMoveProc(PWideChar(S2),temp,Size);
  208. res:=temp;
  209. end;
  210. end;
  211. {$endif FPC_STRTOSHORTSTRINGPROC}
  212. Function fpc_ShortStr_To_WideStr (Const S2 : ShortString): WideString;compilerproc;
  213. {
  214. Converts a ShortString to a WideString;
  215. }
  216. Var
  217. Size : SizeInt;
  218. begin
  219. result:='';
  220. Size:=Length(S2);
  221. if Size>0 then
  222. begin
  223. widestringmanager.Ansi2WideMoveProc(PChar(@S2[1]),result,Size);
  224. { Terminating Zero }
  225. PWideChar(Pointer(fpc_ShortStr_To_WideStr)+Size*sizeof(WideChar))^:=#0;
  226. end;
  227. end;
  228. Function fpc_WideStr_To_AnsiStr (const S2 : WideString): AnsiString; compilerproc;
  229. {
  230. Converts a WideString to an AnsiString
  231. }
  232. Var
  233. Size : SizeInt;
  234. begin
  235. result:='';
  236. Size:=Length(S2);
  237. if Size>0 then
  238. widestringmanager.Wide2AnsiMoveProc(PWideChar(Pointer(S2)),result,Size);
  239. end;
  240. Function fpc_AnsiStr_To_WideStr (Const S2 : AnsiString): WideString; compilerproc;
  241. {
  242. Converts an AnsiString to a WideString;
  243. }
  244. Var
  245. Size : SizeInt;
  246. begin
  247. result:='';
  248. Size:=Length(S2);
  249. if Size>0 then
  250. widestringmanager.Ansi2WideMoveProc(PChar(S2),result,Size);
  251. end;
  252. Function fpc_PWideChar_To_WideStr(const p : pwidechar): widestring; compilerproc;
  253. var
  254. Size : SizeInt;
  255. begin
  256. result:='';
  257. if p=nil then
  258. exit;
  259. Size := IndexWord(p^, -1, 0);
  260. Setlength(result,Size);
  261. if Size>0 then
  262. begin
  263. Move(p^,PWideChar(Pointer(result))^,Size*sizeof(WideChar));
  264. { Terminating Zero }
  265. PWideChar(Pointer(result)+Size*sizeof(WideChar))^:=#0;
  266. end;
  267. end;
  268. { checked against the ansistring routine, 2001-05-27 (FK) }
  269. Procedure fpc_WideStr_Assign (Var S1 : Pointer;S2 : Pointer);[Public,Alias:'FPC_WIDESTR_ASSIGN']; compilerproc;
  270. {
  271. Assigns S2 to S1 (S1:=S2), taking in account reference counts.
  272. }
  273. begin
  274. if S1=S2 then exit;
  275. if S2<>nil then
  276. begin
  277. {$ifdef MSWINDOWS}
  278. if winwidestringalloc then
  279. begin
  280. if SysReAllocStringLen(S1, S2, Length(WideString(S2))) = 0 then
  281. WideStringError;
  282. end
  283. else
  284. {$endif MSWINDOWS}
  285. begin
  286. SetLength(WideString(S1),length(WideString(S2)));
  287. move(s2^,s1^,(length(WideString(s1))+1)*sizeof(widechar));
  288. end;
  289. end
  290. else
  291. begin
  292. { Free S1 }
  293. fpc_widestr_decr_ref (S1);
  294. S1:=nil;
  295. end;
  296. end;
  297. { alias for internal use }
  298. Procedure fpc_WideStr_Assign (Var S1 : Pointer;S2 : Pointer);[external name 'FPC_WIDESTR_ASSIGN'];
  299. {$ifndef STR_CONCAT_PROCS}
  300. function fpc_WideStr_Concat (const S1,S2 : WideString): WideString; compilerproc;
  301. Var
  302. Size,Location : SizeInt;
  303. pc : pwidechar;
  304. begin
  305. { only assign if s1 or s2 is empty }
  306. if (S1='') then
  307. begin
  308. result:=s2;
  309. exit;
  310. end;
  311. if (S2='') then
  312. begin
  313. result:=s1;
  314. exit;
  315. end;
  316. Location:=Length(S1);
  317. Size:=length(S2);
  318. SetLength(result,Size+Location);
  319. pc:=pwidechar(result);
  320. Move(S1[1],pc^,Location*sizeof(WideChar));
  321. inc(pc,location);
  322. Move(S2[1],pc^,(Size+1)*sizeof(WideChar));
  323. end;
  324. function fpc_WideStr_Concat_multi (const sarr:array of Widestring): widestring; compilerproc;
  325. Var
  326. i : Longint;
  327. p : pointer;
  328. pc : pwidechar;
  329. Size,NewSize : SizeInt;
  330. begin
  331. { First calculate size of the result so we can do
  332. a single call to SetLength() }
  333. NewSize:=0;
  334. for i:=low(sarr) to high(sarr) do
  335. inc(Newsize,length(sarr[i]));
  336. SetLength(result,NewSize);
  337. pc:=pwidechar(result);
  338. for i:=low(sarr) to high(sarr) do
  339. begin
  340. p:=pointer(sarr[i]);
  341. if assigned(p) then
  342. begin
  343. Size:=length(widestring(p));
  344. Move(pwidechar(p)^,pc^,(Size+1)*sizeof(WideChar));
  345. inc(pc,size);
  346. end;
  347. end;
  348. end;
  349. {$else STR_CONCAT_PROCS}
  350. procedure fpc_WideStr_Concat (var DestS:Widestring;const S1,S2 : WideString); compilerproc;
  351. Var
  352. Size,Location : SizeInt;
  353. same : boolean;
  354. begin
  355. { only assign if s1 or s2 is empty }
  356. if (S1='') then
  357. begin
  358. DestS:=s2;
  359. exit;
  360. end;
  361. if (S2='') then
  362. begin
  363. DestS:=s1;
  364. exit;
  365. end;
  366. Location:=Length(S1);
  367. Size:=length(S2);
  368. { Use Pointer() typecasts to prevent extra conversion code }
  369. if Pointer(DestS)=Pointer(S1) then
  370. begin
  371. same:=Pointer(S1)=Pointer(S2);
  372. SetLength(DestS,Size+Location);
  373. if same then
  374. Move(Pointer(DestS)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size)*sizeof(WideChar))
  375. else
  376. Move(Pointer(S2)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  377. end
  378. else if Pointer(DestS)=Pointer(S2) then
  379. begin
  380. SetLength(DestS,Size+Location);
  381. Move(Pointer(DestS)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  382. Move(Pointer(S1)^,Pointer(DestS)^,Location*sizeof(WideChar));
  383. end
  384. else
  385. begin
  386. DestS:='';
  387. SetLength(DestS,Size+Location);
  388. Move(Pointer(S1)^,Pointer(DestS)^,Location*sizeof(WideChar));
  389. Move(Pointer(S2)^,(Pointer(DestS)+Location*sizeof(WideChar))^,(Size+1)*sizeof(WideChar));
  390. end;
  391. end;
  392. procedure fpc_WideStr_Concat_multi (var DestS:Widestring;const sarr:array of Widestring); compilerproc;
  393. Var
  394. i : Longint;
  395. p,pc : pointer;
  396. Size,NewLen : SizeInt;
  397. DestTmp : Widestring;
  398. begin
  399. if high(sarr)=0 then
  400. begin
  401. DestS:='';
  402. exit;
  403. end;
  404. { First calculate size of the result so we can do
  405. a single call to SetLength() }
  406. NewLen:=0;
  407. for i:=low(sarr) to high(sarr) do
  408. inc(NewLen,length(sarr[i]));
  409. SetLength(DestTmp,NewLen);
  410. pc:=pwidechar(DestTmp);
  411. for i:=low(sarr) to high(sarr) do
  412. begin
  413. p:=pointer(sarr[i]);
  414. if assigned(p) then
  415. begin
  416. Size:=length(widestring(p));
  417. Move(p^,pc^,(Size+1)*sizeof(WideChar));
  418. inc(pc,size*sizeof(WideChar));
  419. end;
  420. end;
  421. DestS:=DestTmp;
  422. end;
  423. {$endif STR_CONCAT_PROCS}
  424. Function fpc_Char_To_WideStr(const c : Char): WideString; compilerproc;
  425. {
  426. Converts a Char to a WideString;
  427. }
  428. begin
  429. Setlength(fpc_Char_To_WideStr,1);
  430. fpc_Char_To_WideStr[1]:=c;
  431. { Terminating Zero }
  432. PWideChar(Pointer(fpc_Char_To_WideStr)+sizeof(WideChar))^:=#0;
  433. end;
  434. Function fpc_WChar_To_WideStr(const c : WideChar): WideString; compilerproc;
  435. {
  436. Converts a WideChar to a WideString;
  437. }
  438. begin
  439. Setlength (fpc_WChar_To_WideStr,1);
  440. fpc_WChar_To_WideStr[1]:= c;
  441. end;
  442. Function fpc_WChar_To_AnsiStr(const c : WideChar): AnsiString; compilerproc;
  443. {
  444. Converts a WideChar to a AnsiString;
  445. }
  446. begin
  447. widestringmanager.Wide2AnsiMoveProc(@c, fpc_WChar_To_AnsiStr, 1);
  448. end;
  449. Function fpc_UChar_To_WideStr(const c : WideChar): WideString; compilerproc;
  450. {
  451. Converts a WideChar to a WideString;
  452. }
  453. begin
  454. Setlength (fpc_UChar_To_WideStr,1);
  455. fpc_UChar_To_WideStr[1]:= c;
  456. end;
  457. Function fpc_PChar_To_WideStr(const p : pchar): WideString; compilerproc;
  458. Var
  459. L : SizeInt;
  460. begin
  461. if (not assigned(p)) or (p[0]=#0) Then
  462. begin
  463. fpc_pchar_to_widestr := '';
  464. exit;
  465. end;
  466. l:=IndexChar(p^,-1,#0);
  467. widestringmanager.Ansi2WideMoveProc(P,fpc_PChar_To_WideStr,l);
  468. end;
  469. Function fpc_CharArray_To_WideStr(const arr: array of char; zerobased: boolean = true): WideString; compilerproc;
  470. var
  471. i : SizeInt;
  472. begin
  473. if (zerobased) then
  474. begin
  475. if (arr[0]=#0) Then
  476. begin
  477. fpc_chararray_to_widestr := '';
  478. exit;
  479. end;
  480. i:=IndexChar(arr,high(arr)+1,#0);
  481. if i = -1 then
  482. i := high(arr)+1;
  483. end
  484. else
  485. i := high(arr)+1;
  486. SetLength(fpc_CharArray_To_WideStr,i);
  487. widestringmanager.Ansi2WideMoveProc (pchar(@arr),fpc_CharArray_To_WideStr,i);
  488. end;
  489. {$ifndef FPC_STRTOCHARARRAYPROC}
  490. { inside the compiler, the resulttype is modified to that of the actual }
  491. { chararray we're converting to (JM) }
  492. function fpc_widestr_to_chararray(arraysize: SizeInt; const src: WideString): fpc_big_chararray;[public,alias: 'FPC_WIDESTR_TO_CHARARRAY']; compilerproc;
  493. var
  494. len: SizeInt;
  495. temp: ansistring;
  496. begin
  497. len := length(src);
  498. { make sure we don't dereference src if it can be nil (JM) }
  499. if len > 0 then
  500. widestringmanager.wide2ansimoveproc(pwidechar(@src[1]),temp,len);
  501. len := length(temp);
  502. if len > arraysize then
  503. len := arraysize;
  504. {$r-}
  505. move(temp[1],fpc_widestr_to_chararray[0],len);
  506. fillchar(fpc_widestr_to_chararray[len],arraysize-len,0);
  507. {$ifdef RangeCheckWasOn}
  508. {$r+}
  509. {$endif}
  510. end;
  511. { inside the compiler, the resulttype is modified to that of the actual }
  512. { widechararray we're converting to (JM) }
  513. function fpc_widestr_to_widechararray(arraysize: SizeInt; const src: WideString): fpc_big_widechararray;[public,alias: 'FPC_WIDESTR_TO_WIDECHARARRAY']; compilerproc;
  514. var
  515. len: SizeInt;
  516. begin
  517. len := length(src);
  518. if len > arraysize then
  519. len := arraysize;
  520. {$r-}
  521. { make sure we don't try to access element 1 of the ansistring if it's nil }
  522. if len > 0 then
  523. move(src[1],fpc_widestr_to_widechararray[0],len*SizeOf(WideChar));
  524. fillchar(fpc_widestr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  525. {$ifdef RangeCheckWasOn}
  526. {$r+}
  527. {$endif}
  528. end;
  529. { inside the compiler, the resulttype is modified to that of the actual }
  530. { chararray we're converting to (JM) }
  531. function fpc_ansistr_to_widechararray(arraysize: SizeInt; const src: AnsiString): fpc_big_widechararray;[public,alias: 'FPC_ANSISTR_TO_WIDECHARARRAY']; compilerproc;
  532. var
  533. len: SizeInt;
  534. temp: widestring;
  535. begin
  536. len := length(src);
  537. { make sure we don't dereference src if it can be nil (JM) }
  538. if len > 0 then
  539. widestringmanager.ansi2widemoveproc(pchar(@src[1]),temp,len);
  540. len := length(temp);
  541. if len > arraysize then
  542. len := arraysize;
  543. {$r-}
  544. move(temp[1],fpc_ansistr_to_widechararray[0],len*sizeof(widechar));
  545. fillchar(fpc_ansistr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  546. {$ifdef RangeCheckWasOn}
  547. {$r+}
  548. {$endif}
  549. end;
  550. function fpc_shortstr_to_widechararray(arraysize: SizeInt; const src: ShortString): fpc_big_widechararray;[public,alias: 'FPC_SHORTSTR_TO_WIDECHARARRAY']; compilerproc;
  551. var
  552. len: longint;
  553. temp : widestring;
  554. begin
  555. len := length(src);
  556. { make sure we don't access char 1 if length is 0 (JM) }
  557. if len > 0 then
  558. widestringmanager.ansi2widemoveproc(pchar(@src[1]),temp,len);
  559. len := length(temp);
  560. if len > arraysize then
  561. len := arraysize;
  562. {$r-}
  563. move(temp[1],fpc_shortstr_to_widechararray[0],len*sizeof(widechar));
  564. fillchar(fpc_shortstr_to_widechararray[len],(arraysize-len)*SizeOf(WideChar),0);
  565. {$ifdef RangeCheckWasOn}
  566. {$r+}
  567. {$endif}
  568. end;
  569. {$else ndef FPC_STRTOCHARARRAYPROC}
  570. procedure fpc_widestr_to_chararray(out res: array of char; const src: WideString); compilerproc;
  571. var
  572. len: SizeInt;
  573. temp: ansistring;
  574. begin
  575. len := length(src);
  576. { make sure we don't dereference src if it can be nil (JM) }
  577. if len > 0 then
  578. widestringmanager.wide2ansimoveproc(pwidechar(@src[1]),temp,len);
  579. len := length(temp);
  580. if len > length(res) then
  581. len := length(res);
  582. {$r-}
  583. move(temp[1],res[0],len);
  584. fillchar(res[len],length(res)-len,0);
  585. {$ifdef RangeCheckWasOn}
  586. {$r+}
  587. {$endif}
  588. end;
  589. procedure fpc_widestr_to_widechararray(out res: array of widechar; const src: WideString); compilerproc;
  590. var
  591. len: SizeInt;
  592. begin
  593. len := length(src);
  594. if len > length(res) then
  595. len := length(res);
  596. {$r-}
  597. { make sure we don't try to access element 1 of the ansistring if it's nil }
  598. if len > 0 then
  599. move(src[1],res[0],len*SizeOf(WideChar));
  600. fillchar(res[len],(length(res)-len)*SizeOf(WideChar),0);
  601. {$ifdef RangeCheckWasOn}
  602. {$r+}
  603. {$endif}
  604. end;
  605. {$endif ndef FPC_STRTOCHARARRAYPROC}
  606. Function fpc_WideStr_Compare(const S1,S2 : WideString): SizeInt;[Public,Alias : 'FPC_WIDESTR_COMPARE']; compilerproc;
  607. {
  608. Compares 2 WideStrings;
  609. The result is
  610. <0 if S1<S2
  611. 0 if S1=S2
  612. >0 if S1>S2
  613. }
  614. Var
  615. MaxI,Temp : SizeInt;
  616. begin
  617. if pointer(S1)=pointer(S2) then
  618. begin
  619. fpc_WideStr_Compare:=0;
  620. exit;
  621. end;
  622. Maxi:=Length(S1);
  623. temp:=Length(S2);
  624. If MaxI>Temp then
  625. MaxI:=Temp;
  626. Temp:=CompareWord(S1[1],S2[1],MaxI);
  627. if temp=0 then
  628. temp:=Length(S1)-Length(S2);
  629. fpc_WideStr_Compare:=Temp;
  630. end;
  631. Function fpc_WideStr_Compare_Equal(const S1,S2 : WideString): SizeInt;[Public,Alias : 'FPC_WIDESTR_COMPARE_EQUAL']; compilerproc;
  632. {
  633. Compares 2 WideStrings for equality only;
  634. The result is
  635. 0 if S1=S2
  636. <>0 if S1<>S2
  637. }
  638. Var
  639. MaxI : SizeInt;
  640. begin
  641. if pointer(S1)=pointer(S2) then
  642. exit(0);
  643. Maxi:=Length(S1);
  644. If MaxI<>Length(S2) then
  645. exit(-1)
  646. else
  647. exit(CompareWord(S1[1],S2[1],MaxI));
  648. end;
  649. Procedure fpc_WideStr_CheckRange(p: Pointer; index: SizeInt);[Public,Alias : 'FPC_WIDESTR_RANGECHECK']; compilerproc;
  650. begin
  651. if (p=nil) or (index>PWideRec(p-WideFirstOff)^.len div 2) or (Index<1) then
  652. HandleErrorFrame(201,get_frame);
  653. end;
  654. Procedure fpc_WideStr_SetLength(Var S : WideString; l : SizeInt);[Public,Alias : 'FPC_WIDESTR_SETLENGTH']; compilerproc;
  655. {
  656. Sets The length of string S to L.
  657. Makes sure S is unique, and contains enough room.
  658. }
  659. Var
  660. Temp : Pointer;
  661. movelen: SizeInt;
  662. begin
  663. if (l>0) then
  664. begin
  665. if Pointer(S)=nil then
  666. begin
  667. { Need a complete new string...}
  668. Pointer(s):=NewWideString(l);
  669. end
  670. { windows doesn't support reallocing widestrings, this code
  671. is anyways subject to be removed because widestrings shouldn't be
  672. ref. counted anymore (FK) }
  673. else
  674. if
  675. {$ifdef MSWINDOWS}
  676. not winwidestringalloc and
  677. {$endif MSWINDOWS}
  678. True
  679. then
  680. begin
  681. Dec(Pointer(S),WideFirstOff);
  682. if SizeUInt(L*sizeof(WideChar)+WideRecLen)>MemSize(Pointer(S)) then
  683. reallocmem(pointer(S), L*sizeof(WideChar)+WideRecLen);
  684. Inc(Pointer(S), WideFirstOff);
  685. end
  686. else
  687. begin
  688. { Reallocation is needed... }
  689. Temp:=Pointer(NewWideString(L));
  690. if Length(S)>0 then
  691. begin
  692. if l < succ(length(s)) then
  693. movelen := l
  694. { also move terminating null }
  695. else
  696. movelen := succ(length(s));
  697. Move(Pointer(S)^,Temp^,movelen * Sizeof(WideChar));
  698. end;
  699. fpc_widestr_decr_ref(Pointer(S));
  700. Pointer(S):=Temp;
  701. end;
  702. { Force nil termination in case it gets shorter }
  703. PWord(Pointer(S)+l*sizeof(WideChar))^:=0;
  704. {$ifdef MSWINDOWS}
  705. if not winwidestringalloc then
  706. {$endif MSWINDOWS}
  707. PWideRec(Pointer(S)-WideFirstOff)^.Len:=l*sizeof(WideChar);
  708. end
  709. else
  710. begin
  711. { Length=0 }
  712. if Pointer(S)<>nil then
  713. fpc_widestr_decr_ref (Pointer(S));
  714. Pointer(S):=Nil;
  715. end;
  716. end;
  717. {*****************************************************************************
  718. Public functions, In interface.
  719. *****************************************************************************}
  720. Function fpc_widestr_Unique(Var S : Pointer): Pointer; [Public,Alias : 'FPC_WIDESTR_UNIQUE']; compilerproc;
  721. begin
  722. pointer(result) := pointer(s);
  723. end;
  724. Function Fpc_WideStr_Copy (Const S : WideString; Index,Size : SizeInt) : WideString;compilerproc;
  725. var
  726. ResultAddress : Pointer;
  727. begin
  728. ResultAddress:=Nil;
  729. dec(index);
  730. if Index < 0 then
  731. Index := 0;
  732. { Check Size. Accounts for Zero-length S, the double check is needed because
  733. Size can be maxint and will get <0 when adding index }
  734. if (Size>Length(S)) or
  735. (Index+Size>Length(S)) then
  736. Size:=Length(S)-Index;
  737. If Size>0 then
  738. begin
  739. If Index<0 Then
  740. Index:=0;
  741. ResultAddress:=Pointer(NewWideString (Size));
  742. if ResultAddress<>Nil then
  743. begin
  744. Move (PWideChar(S)[Index],ResultAddress^,Size*sizeof(WideChar));
  745. PWideRec(ResultAddress-WideFirstOff)^.Len:=Size*sizeof(WideChar);
  746. PWideChar(ResultAddress+Size*sizeof(WideChar))^:=#0;
  747. end;
  748. end;
  749. fpc_widestr_decr_ref(Pointer(fpc_widestr_copy));
  750. Pointer(fpc_widestr_Copy):=ResultAddress;
  751. end;
  752. Function Pos (Const Substr : WideString; Const Source : WideString) : SizeInt;
  753. var
  754. i,MaxLen : SizeInt;
  755. pc : pwidechar;
  756. begin
  757. Pos:=0;
  758. if Length(SubStr)>0 then
  759. begin
  760. MaxLen:=Length(source)-Length(SubStr);
  761. i:=0;
  762. pc:=@source[1];
  763. while (i<=MaxLen) do
  764. begin
  765. inc(i);
  766. if (SubStr[1]=pc^) and
  767. (CompareWord(Substr[1],pc^,Length(SubStr))=0) then
  768. begin
  769. Pos:=i;
  770. exit;
  771. end;
  772. inc(pc);
  773. end;
  774. end;
  775. end;
  776. { Faster version for a widechar alone }
  777. Function Pos (c : WideChar; Const s : WideString) : SizeInt;
  778. var
  779. i: SizeInt;
  780. pc : pwidechar;
  781. begin
  782. pc:=@s[1];
  783. for i:=1 to length(s) do
  784. begin
  785. if pc^=c then
  786. begin
  787. pos:=i;
  788. exit;
  789. end;
  790. inc(pc);
  791. end;
  792. pos:=0;
  793. end;
  794. Function Pos (c : WideChar; Const s : AnsiString) : SizeInt;
  795. begin
  796. result:=Pos(c,WideString(s));
  797. end;
  798. Function Pos (c : AnsiString; Const s : WideString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  799. begin
  800. result:=Pos(WideString(c),s);
  801. end;
  802. Function Pos (c : ShortString; Const s : WideString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  803. begin
  804. result:=Pos(WideString(c),s);
  805. end;
  806. Function Pos (c : WideString; Const s : AnsiString) : SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  807. begin
  808. result:=Pos(c,WideString(s));
  809. end;
  810. { Faster version for a char alone. Must be implemented because }
  811. { pos(c: char; const s: shortstring) also exists, so otherwise }
  812. { using pos(char,pchar) will always call the shortstring version }
  813. { (exact match for first argument), also with $h+ (JM) }
  814. Function Pos (c : Char; Const s : WideString) : SizeInt;
  815. var
  816. i: SizeInt;
  817. wc : widechar;
  818. pc : pwidechar;
  819. begin
  820. wc:=c;
  821. pc:=@s[1];
  822. for i:=1 to length(s) do
  823. begin
  824. if pc^=wc then
  825. begin
  826. pos:=i;
  827. exit;
  828. end;
  829. inc(pc);
  830. end;
  831. pos:=0;
  832. end;
  833. Procedure Delete (Var S : WideString; Index,Size: SizeInt);
  834. Var
  835. LS : SizeInt;
  836. begin
  837. LS:=Length(S);
  838. if (Index>LS) or (Index<=0) or (Size<=0) then
  839. exit;
  840. UniqueString (S);
  841. { (Size+Index) will overflow if Size=MaxInt. }
  842. if Size>LS-Index then
  843. Size:=LS-Index+1;
  844. if Size<=LS-Index then
  845. begin
  846. Dec(Index);
  847. Move(PWideChar(S)[Index+Size],PWideChar(S)[Index],(LS-Index-Size+1)*sizeof(WideChar));
  848. end;
  849. Setlength(s,LS-Size);
  850. end;
  851. Procedure Insert (Const Source : WideString; Var S : WideString; Index : SizeInt);
  852. var
  853. Temp : WideString;
  854. LS : SizeInt;
  855. begin
  856. If Length(Source)=0 then
  857. exit;
  858. if index <= 0 then
  859. index := 1;
  860. Ls:=Length(S);
  861. if index > LS then
  862. index := LS+1;
  863. Dec(Index);
  864. Pointer(Temp) := NewWideString(Length(Source)+LS);
  865. SetLength(Temp,Length(Source)+LS);
  866. If Index>0 then
  867. move (PWideChar(S)^,PWideChar(Temp)^,Index*sizeof(WideChar));
  868. Move (PWideChar(Source)^,PWideChar(Temp)[Index],Length(Source)*sizeof(WideChar));
  869. If (LS-Index)>0 then
  870. Move(PWideChar(S)[Index],PWideChar(temp)[Length(Source)+index],(LS-Index)*sizeof(WideChar));
  871. S:=Temp;
  872. end;
  873. function UpCase(const s : WideString) : WideString;
  874. begin
  875. result:=widestringmanager.UpperWideStringProc(s);
  876. end;
  877. Procedure SetString (Out S : WideString; Buf : PWideChar; Len : SizeInt);
  878. begin
  879. SetLength(S,Len);
  880. If (Buf<>Nil) and (Len>0) then
  881. Move (Buf[0],S[1],Len*sizeof(WideChar));
  882. end;
  883. Procedure SetString (Out S : WideString; Buf : PChar; Len : SizeInt);
  884. begin
  885. SetLength(S,Len);
  886. If (Buf<>Nil) and (Len>0) then
  887. widestringmanager.Ansi2WideMoveProc(Buf,S,Len);
  888. end;
  889. {$ifndef FPUNONE}
  890. Function fpc_Val_Real_WideStr(Const S : WideString; out Code : ValSInt): ValReal; [public, alias:'FPC_VAL_REAL_WIDESTR']; compilerproc;
  891. Var
  892. SS : String;
  893. begin
  894. fpc_Val_Real_WideStr := 0;
  895. if length(S) > 255 then
  896. code := 256
  897. else
  898. begin
  899. SS := S;
  900. Val(SS,fpc_Val_Real_WideStr,code);
  901. end;
  902. end;
  903. {$endif}
  904. function fpc_val_enum_widestr(str2ordindex:pointer;const s:widestring;out code:valsint):longint;compilerproc;
  905. var ss:shortstring;
  906. begin
  907. if length(s)>255 then
  908. code:=256
  909. else
  910. begin
  911. ss:=s;
  912. val(ss,fpc_val_enum_widestr,code);
  913. end;
  914. end;
  915. Function fpc_Val_Currency_WideStr(Const S : WideString; out Code : ValSInt): Currency; [public, alias:'FPC_VAL_CURRENCY_WIDESTR']; compilerproc;
  916. Var
  917. SS : String;
  918. begin
  919. if length(S) > 255 then
  920. begin
  921. fpc_Val_Currency_WideStr:=0;
  922. code := 256;
  923. end
  924. else
  925. begin
  926. SS := S;
  927. Val(SS,fpc_Val_Currency_WideStr,code);
  928. end;
  929. end;
  930. Function fpc_Val_UInt_WideStr (Const S : WideString; out Code : ValSInt): ValUInt; [public, alias:'FPC_VAL_UINT_WIDESTR']; compilerproc;
  931. Var
  932. SS : ShortString;
  933. begin
  934. fpc_Val_UInt_WideStr := 0;
  935. if length(S) > 255 then
  936. code := 256
  937. else
  938. begin
  939. SS := S;
  940. Val(SS,fpc_Val_UInt_WideStr,code);
  941. end;
  942. end;
  943. Function fpc_Val_SInt_WideStr (DestSize: SizeInt; Const S : WideString; out Code : ValSInt): ValSInt; [public, alias:'FPC_VAL_SINT_WIDESTR']; compilerproc;
  944. Var
  945. SS : ShortString;
  946. begin
  947. fpc_Val_SInt_WideStr:=0;
  948. if length(S)>255 then
  949. code:=256
  950. else
  951. begin
  952. SS := S;
  953. fpc_Val_SInt_WideStr := int_Val_SInt_ShortStr(DestSize,SS,Code);
  954. end;
  955. end;
  956. {$ifndef CPU64}
  957. Function fpc_Val_qword_WideStr (Const S : WideString; out Code : ValSInt): qword; [public, alias:'FPC_VAL_QWORD_WIDESTR']; compilerproc;
  958. Var
  959. SS : ShortString;
  960. begin
  961. fpc_Val_qword_WideStr:=0;
  962. if length(S)>255 then
  963. code:=256
  964. else
  965. begin
  966. SS := S;
  967. Val(SS,fpc_Val_qword_WideStr,Code);
  968. end;
  969. end;
  970. Function fpc_Val_int64_WideStr (Const S : WideString; out Code : ValSInt): Int64; [public, alias:'FPC_VAL_INT64_WIDESTR']; compilerproc;
  971. Var
  972. SS : ShortString;
  973. begin
  974. fpc_Val_int64_WideStr:=0;
  975. if length(S)>255 then
  976. code:=256
  977. else
  978. begin
  979. SS := S;
  980. Val(SS,fpc_Val_int64_WideStr,Code);
  981. end;
  982. end;
  983. {$endif CPU64}
  984. {$ifndef FPUNONE}
  985. procedure fpc_WideStr_Float(d : ValReal;len,fr,rt : SizeInt;out s : WideString);compilerproc;
  986. var
  987. ss : shortstring;
  988. begin
  989. str_real(len,fr,d,treal_type(rt),ss);
  990. s:=ss;
  991. end;
  992. {$endif}
  993. procedure fpc_widestr_enum(ordinal,len:sizeint;typinfo,ord2strindex:pointer;out s:widestring);compilerproc;
  994. var ss:shortstring;
  995. begin
  996. fpc_shortstr_enum(ordinal,len,typinfo,ord2strindex,ss);
  997. s:=ss;
  998. end;
  999. procedure fpc_widestr_bool(b : boolean;len:sizeint;out s:widestring);compilerproc;
  1000. var ss:shortstring;
  1001. begin
  1002. fpc_shortstr_bool(b,len,ss);
  1003. s:=ss;
  1004. end;
  1005. {$ifdef FPC_HAS_STR_CURRENCY}
  1006. procedure fpc_WideStr_Currency(c : Currency;len,fr : SizeInt;out s : WideString);compilerproc;
  1007. var
  1008. ss : shortstring;
  1009. begin
  1010. str(c:len:fr,ss);
  1011. s:=ss;
  1012. end;
  1013. {$endif FPC_HAS_STR_CURRENCY}
  1014. Procedure fpc_WideStr_SInt(v : ValSint; Len : SizeInt; out S : WideString);compilerproc;
  1015. Var
  1016. SS : ShortString;
  1017. begin
  1018. Str (v:Len,SS);
  1019. S:=SS;
  1020. end;
  1021. Procedure fpc_WideStr_UInt(v : ValUInt;Len : SizeInt; out S : WideString);compilerproc;
  1022. Var
  1023. SS : ShortString;
  1024. begin
  1025. str(v:Len,SS);
  1026. S:=SS;
  1027. end;
  1028. {$ifndef CPU64}
  1029. Procedure fpc_WideStr_Int64(v : Int64; Len : SizeInt; out S : WideString);compilerproc;
  1030. Var
  1031. SS : ShortString;
  1032. begin
  1033. Str (v:Len,SS);
  1034. S:=SS;
  1035. end;
  1036. Procedure fpc_WideStr_Qword(v : Qword;Len : SizeInt; out S : WideString);compilerproc;
  1037. Var
  1038. SS : ShortString;
  1039. begin
  1040. str(v:Len,SS);
  1041. S:=SS;
  1042. end;
  1043. {$endif CPU64}
  1044. { converts an utf-16 code point or surrogate pair to utf-32 }
  1045. function utf16toutf32(const S: WideString; const index: SizeInt; out len: longint): UCS4Char; [public, alias: 'FPC_WIDETOUTF32'];
  1046. var
  1047. w: widechar;
  1048. begin
  1049. { UTF-16 points in the range #$0-#$D7FF and #$E000-#$FFFF }
  1050. { are the same in UTF-32 }
  1051. w:=s[index];
  1052. if (w<=#$d7ff) or
  1053. (w>=#$e000) then
  1054. begin
  1055. result:=UCS4Char(w);
  1056. len:=1;
  1057. end
  1058. { valid surrogate pair? }
  1059. else if (w<=#$dbff) and
  1060. { w>=#$d7ff check not needed, checked above }
  1061. (index<length(s)) and
  1062. (s[index+1]>=#$dc00) and
  1063. (s[index+1]<=#$dfff) then
  1064. { convert the surrogate pair to UTF-32 }
  1065. begin
  1066. result:=(UCS4Char(w)-$d800) shl 10 + (UCS4Char(s[index+1])-$dc00) + $10000;
  1067. len:=2;
  1068. end
  1069. else
  1070. { invalid surrogate -> do nothing }
  1071. begin
  1072. result:=UCS4Char(w);
  1073. len:=1;
  1074. end;
  1075. end;
  1076. function UnicodeToUtf8(Dest: PChar; Source: PWideChar; MaxBytes: SizeInt): SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  1077. begin
  1078. if assigned(Source) then
  1079. Result:=UnicodeToUtf8(Dest,MaxBytes,Source,IndexWord(Source^,-1,0))
  1080. else
  1081. Result:=0;
  1082. end;
  1083. function UnicodeToUtf8(Dest: PChar; MaxDestBytes: SizeUInt; Source: PWideChar; SourceChars: SizeUInt): SizeUInt;
  1084. var
  1085. i,j : SizeUInt;
  1086. w : word;
  1087. lw : longword;
  1088. len : longint;
  1089. begin
  1090. result:=0;
  1091. if source=nil then
  1092. exit;
  1093. i:=0;
  1094. j:=0;
  1095. if assigned(Dest) then
  1096. begin
  1097. while (i<SourceChars) and (j<MaxDestBytes) do
  1098. begin
  1099. w:=word(Source[i]);
  1100. case w of
  1101. 0..$7f:
  1102. begin
  1103. Dest[j]:=char(w);
  1104. inc(j);
  1105. end;
  1106. $80..$7ff:
  1107. begin
  1108. if j+1>=MaxDestBytes then
  1109. break;
  1110. Dest[j]:=char($c0 or (w shr 6));
  1111. Dest[j+1]:=char($80 or (w and $3f));
  1112. inc(j,2);
  1113. end;
  1114. $800..$d7ff,$e000..$ffff:
  1115. begin
  1116. if j+2>=MaxDestBytes then
  1117. break;
  1118. Dest[j]:=char($e0 or (w shr 12));
  1119. Dest[j+1]:=char($80 or ((w shr 6) and $3f));
  1120. Dest[j+2]:=char($80 or (w and $3f));
  1121. inc(j,3);
  1122. end;
  1123. $d800..$dbff:
  1124. {High Surrogates}
  1125. begin
  1126. if j+3>=MaxDestBytes then
  1127. break;
  1128. if (i<sourcechars-1) and
  1129. (word(Source[i+1]) >= $dc00) and
  1130. (word(Source[i+1]) <= $dfff) then
  1131. begin
  1132. lw:=longword(utf16toutf32(Source[i] + Source[i+1], 1, len));
  1133. Dest[j]:=char($f0 or (lw shr 18));
  1134. Dest[j+1]:=char($80 or ((lw shr 12) and $3f));
  1135. Dest[j+2]:=char($80 or ((lw shr 6) and $3f));
  1136. Dest[j+3]:=char($80 or (lw and $3f));
  1137. inc(j,4);
  1138. inc(i);
  1139. end;
  1140. end;
  1141. end;
  1142. inc(i);
  1143. end;
  1144. if j>SizeUInt(MaxDestBytes-1) then
  1145. j:=MaxDestBytes-1;
  1146. Dest[j]:=#0;
  1147. end
  1148. else
  1149. begin
  1150. while i<SourceChars do
  1151. begin
  1152. case word(Source[i]) of
  1153. $0..$7f:
  1154. inc(j);
  1155. $80..$7ff:
  1156. inc(j,2);
  1157. $800..$d7ff,$e000..$ffff:
  1158. inc(j,3);
  1159. $d800..$dbff:
  1160. begin
  1161. if (i<sourcechars-1) and
  1162. (word(Source[i+1]) >= $dc00) and
  1163. (word(Source[i+1]) <= $dfff) then
  1164. begin
  1165. inc(j,4);
  1166. inc(i);
  1167. end;
  1168. end;
  1169. end;
  1170. inc(i);
  1171. end;
  1172. end;
  1173. result:=j+1;
  1174. end;
  1175. function Utf8ToUnicode(Dest: PWideChar; Source: PChar; MaxChars: SizeInt): SizeInt;{$ifdef SYSTEMINLINE}inline;{$endif}
  1176. begin
  1177. if assigned(Source) then
  1178. Result:=Utf8ToUnicode(Dest,MaxChars,Source,strlen(Source))
  1179. else
  1180. Result:=0;
  1181. end;
  1182. function UTF8ToUnicode(Dest: PWideChar; MaxDestChars: SizeUInt; Source: PChar; SourceBytes: SizeUInt): SizeUInt;
  1183. const
  1184. UNICODE_INVALID=63;
  1185. var
  1186. InputUTF8: SizeUInt;
  1187. IBYTE: BYTE;
  1188. OutputUnicode: SizeUInt;
  1189. PRECHAR: SizeUInt;
  1190. TempBYTE: BYTE;
  1191. CharLen: SizeUint;
  1192. LookAhead: SizeUInt;
  1193. UC: SizeUInt;
  1194. begin
  1195. if not assigned(Source) then
  1196. begin
  1197. result:=0;
  1198. exit;
  1199. end;
  1200. result:=SizeUInt(-1);
  1201. InputUTF8:=0;
  1202. OutputUnicode:=0;
  1203. PreChar:=0;
  1204. if Assigned(Dest) Then
  1205. begin
  1206. while (OutputUnicode<MaxDestChars) and (InputUTF8<SourceBytes) do
  1207. begin
  1208. IBYTE:=byte(Source[InputUTF8]);
  1209. if (IBYTE and $80) = 0 then
  1210. begin
  1211. //One character US-ASCII, convert it to unicode
  1212. if IBYTE = 10 then
  1213. begin
  1214. If (PreChar<>13) and FALSE then
  1215. begin
  1216. //Expand to crlf, conform UTF-8.
  1217. //This procedure will break the memory alocation by
  1218. //FPC for the widestring, so never use it. Condition never true due the "and FALSE".
  1219. if OutputUnicode+1<MaxDestChars then
  1220. begin
  1221. Dest[OutputUnicode]:=WideChar(13);
  1222. inc(OutputUnicode);
  1223. Dest[OutputUnicode]:=WideChar(10);
  1224. inc(OutputUnicode);
  1225. PreChar:=10;
  1226. end
  1227. else
  1228. begin
  1229. Dest[OutputUnicode]:=WideChar(13);
  1230. inc(OutputUnicode);
  1231. end;
  1232. end
  1233. else
  1234. begin
  1235. Dest[OutputUnicode]:=WideChar(IBYTE);
  1236. inc(OutputUnicode);
  1237. PreChar:=IBYTE;
  1238. end;
  1239. end
  1240. else
  1241. begin
  1242. Dest[OutputUnicode]:=WideChar(IBYTE);
  1243. inc(OutputUnicode);
  1244. PreChar:=IBYTE;
  1245. end;
  1246. inc(InputUTF8);
  1247. end
  1248. else
  1249. begin
  1250. TempByte:=IBYTE;
  1251. CharLen:=0;
  1252. while (TempBYTE and $80)<>0 do
  1253. begin
  1254. TempBYTE:=(TempBYTE shl 1) and $FE;
  1255. inc(CharLen);
  1256. end;
  1257. //Test for the "CharLen" conforms UTF-8 string
  1258. //This means the 10xxxxxx pattern.
  1259. if SizeUInt(InputUTF8+CharLen-1)>SourceBytes then
  1260. begin
  1261. //Insuficient chars in string to decode
  1262. //UTF-8 array. Fallback to single char.
  1263. CharLen:= 1;
  1264. end;
  1265. for LookAhead := 1 to CharLen-1 do
  1266. begin
  1267. if ((byte(Source[InputUTF8+LookAhead]) and $80)<>$80) or
  1268. ((byte(Source[InputUTF8+LookAhead]) and $40)<>$00) then
  1269. begin
  1270. //Invalid UTF-8 sequence, fallback.
  1271. CharLen:= LookAhead;
  1272. break;
  1273. end;
  1274. end;
  1275. UC:=$FFFF;
  1276. case CharLen of
  1277. 1: begin
  1278. //Not valid UTF-8 sequence
  1279. UC:=UNICODE_INVALID;
  1280. end;
  1281. 2: begin
  1282. //Two bytes UTF, convert it
  1283. UC:=(byte(Source[InputUTF8]) and $1F) shl 6;
  1284. UC:=UC or (byte(Source[InputUTF8+1]) and $3F);
  1285. if UC <= $7F then
  1286. begin
  1287. //Invalid UTF sequence.
  1288. UC:=UNICODE_INVALID;
  1289. end;
  1290. end;
  1291. 3: begin
  1292. //Three bytes, convert it to unicode
  1293. UC:= (byte(Source[InputUTF8]) and $0F) shl 12;
  1294. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 6);
  1295. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F));
  1296. if (UC <= $7FF) or (UC >= $FFFE) or ((UC >= $D800) and (UC <= $DFFF)) then
  1297. begin
  1298. //Invalid UTF-8 sequence
  1299. UC:= UNICODE_INVALID;
  1300. End;
  1301. end;
  1302. 4: begin
  1303. //Four bytes, convert it to two unicode characters
  1304. UC:= (byte(Source[InputUTF8]) and $07) shl 18;
  1305. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 12);
  1306. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F) shl 6);
  1307. UC:= UC or ((byte(Source[InputUTF8+3]) and $3F));
  1308. if (UC < $10000) or (UC > $10FFFF) then
  1309. begin
  1310. UC:= UNICODE_INVALID;
  1311. end
  1312. else
  1313. begin
  1314. { only store pair if room }
  1315. dec(UC,$10000);
  1316. if (OutputUnicode<MaxDestChars-1) then
  1317. begin
  1318. Dest[OutputUnicode]:=WideChar(UC shr 10 + $D800);
  1319. inc(OutputUnicode);
  1320. UC:=(UC and $3ff) + $DC00;
  1321. end
  1322. else
  1323. begin
  1324. InputUTF8:= InputUTF8 + CharLen;
  1325. { don't store anything }
  1326. CharLen:=0;
  1327. end;
  1328. end;
  1329. end;
  1330. 5,6,7: begin
  1331. //Invalid UTF8 to unicode conversion,
  1332. //mask it as invalid UNICODE too.
  1333. UC:=UNICODE_INVALID;
  1334. end;
  1335. end;
  1336. if CharLen > 0 then
  1337. begin
  1338. PreChar:=UC;
  1339. Dest[OutputUnicode]:=WideChar(UC);
  1340. inc(OutputUnicode);
  1341. end;
  1342. InputUTF8:= InputUTF8 + CharLen;
  1343. end;
  1344. end;
  1345. Result:=OutputUnicode+1;
  1346. end
  1347. else
  1348. begin
  1349. while (InputUTF8<SourceBytes) do
  1350. begin
  1351. IBYTE:=byte(Source[InputUTF8]);
  1352. if (IBYTE and $80) = 0 then
  1353. begin
  1354. //One character US-ASCII, convert it to unicode
  1355. if IBYTE = 10 then
  1356. begin
  1357. if (PreChar<>13) and FALSE then
  1358. begin
  1359. //Expand to crlf, conform UTF-8.
  1360. //This procedure will break the memory alocation by
  1361. //FPC for the widestring, so never use it. Condition never true due the "and FALSE".
  1362. inc(OutputUnicode,2);
  1363. PreChar:=10;
  1364. end
  1365. else
  1366. begin
  1367. inc(OutputUnicode);
  1368. PreChar:=IBYTE;
  1369. end;
  1370. end
  1371. else
  1372. begin
  1373. inc(OutputUnicode);
  1374. PreChar:=IBYTE;
  1375. end;
  1376. inc(InputUTF8);
  1377. end
  1378. else
  1379. begin
  1380. TempByte:=IBYTE;
  1381. CharLen:=0;
  1382. while (TempBYTE and $80)<>0 do
  1383. begin
  1384. TempBYTE:=(TempBYTE shl 1) and $FE;
  1385. inc(CharLen);
  1386. end;
  1387. //Test for the "CharLen" conforms UTF-8 string
  1388. //This means the 10xxxxxx pattern.
  1389. if SizeUInt(InputUTF8+CharLen-1)>SourceBytes then
  1390. begin
  1391. //Insuficient chars in string to decode
  1392. //UTF-8 array. Fallback to single char.
  1393. CharLen:= 1;
  1394. end;
  1395. for LookAhead := 1 to CharLen-1 do
  1396. begin
  1397. if ((byte(Source[InputUTF8+LookAhead]) and $80)<>$80) or
  1398. ((byte(Source[InputUTF8+LookAhead]) and $40)<>$00) then
  1399. begin
  1400. //Invalid UTF-8 sequence, fallback.
  1401. CharLen:= LookAhead;
  1402. break;
  1403. end;
  1404. end;
  1405. UC:=$FFFF;
  1406. case CharLen of
  1407. 1: begin
  1408. //Not valid UTF-8 sequence
  1409. UC:=UNICODE_INVALID;
  1410. end;
  1411. 2: begin
  1412. //Two bytes UTF, convert it
  1413. UC:=(byte(Source[InputUTF8]) and $1F) shl 6;
  1414. UC:=UC or (byte(Source[InputUTF8+1]) and $3F);
  1415. if UC <= $7F then
  1416. begin
  1417. //Invalid UTF sequence.
  1418. UC:=UNICODE_INVALID;
  1419. end;
  1420. end;
  1421. 3: begin
  1422. //Three bytes, convert it to unicode
  1423. UC:= (byte(Source[InputUTF8]) and $0F) shl 12;
  1424. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 6);
  1425. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F));
  1426. If (UC <= $7FF) or (UC >= $FFFE) or ((UC >= $D800) and (UC <= $DFFF)) then
  1427. begin
  1428. //Invalid UTF-8 sequence
  1429. UC:= UNICODE_INVALID;
  1430. end;
  1431. end;
  1432. 4: begin
  1433. //Four bytes, convert it to two unicode characters
  1434. UC:= (byte(Source[InputUTF8]) and $07) shl 18;
  1435. UC:= UC or ((byte(Source[InputUTF8+1]) and $3F) shl 12);
  1436. UC:= UC or ((byte(Source[InputUTF8+2]) and $3F) shl 6);
  1437. UC:= UC or ((byte(Source[InputUTF8+3]) and $3F));
  1438. if (UC < $10000) or (UC > $10FFFF) then
  1439. UC:= UNICODE_INVALID
  1440. else
  1441. { extra character character }
  1442. inc(OutputUnicode);
  1443. end;
  1444. 5,6,7: begin
  1445. //Invalid UTF8 to unicode conversion,
  1446. //mask it as invalid UNICODE too.
  1447. UC:=UNICODE_INVALID;
  1448. end;
  1449. end;
  1450. if CharLen > 0 then
  1451. begin
  1452. PreChar:=UC;
  1453. inc(OutputUnicode);
  1454. end;
  1455. InputUTF8:= InputUTF8 + CharLen;
  1456. end;
  1457. end;
  1458. Result:=OutputUnicode+1;
  1459. end;
  1460. end;
  1461. function UTF8Encode(const s : WideString) : UTF8String;
  1462. var
  1463. i : SizeInt;
  1464. hs : UTF8String;
  1465. begin
  1466. result:='';
  1467. if s='' then
  1468. exit;
  1469. SetLength(hs,length(s)*3);
  1470. i:=UnicodeToUtf8(pchar(hs),length(hs)+1,PWideChar(s),length(s));
  1471. if i>0 then
  1472. begin
  1473. SetLength(hs,i-1);
  1474. result:=hs;
  1475. end;
  1476. end;
  1477. const
  1478. SNoWidestrings = 'This binary has no widestrings support compiled in.';
  1479. SRecompileWithWidestrings = 'Recompile the application with a widestrings-manager in the program uses clause.';
  1480. procedure unimplementedwidestring;
  1481. begin
  1482. {$ifdef FPC_HAS_FEATURE_CONSOLEIO}
  1483. If IsConsole then
  1484. begin
  1485. Writeln(StdErr,SNoWidestrings);
  1486. Writeln(StdErr,SRecompileWithWidestrings);
  1487. end;
  1488. {$endif FPC_HAS_FEATURE_CONSOLEIO}
  1489. HandleErrorFrame(233,get_frame);
  1490. end;
  1491. {$warnings off}
  1492. function GenericWideCase(const s : WideString) : WideString;
  1493. begin
  1494. unimplementedwidestring;
  1495. end;
  1496. function CompareWideString(const s1, s2 : WideString) : PtrInt;
  1497. begin
  1498. unimplementedwidestring;
  1499. end;
  1500. function CompareTextWideString(const s1, s2 : WideString): PtrInt;
  1501. begin
  1502. unimplementedwidestring;
  1503. end;
  1504. {$warnings on}
  1505. function DefaultCharLengthPChar(const Str: PChar): PtrInt;forward;
  1506. function DefaultCodePointLength(const Str: PChar; MaxLookAead: PtrInt): Ptrint;forward;
  1507. procedure initwidestringmanager;
  1508. begin
  1509. fillchar(widestringmanager,sizeof(widestringmanager),0);
  1510. {$ifndef HAS_WIDESTRINGMANAGER}
  1511. widestringmanager.Wide2AnsiMoveProc:=@defaultWide2AnsiMove;
  1512. widestringmanager.Ansi2WideMoveProc:=@defaultAnsi2WideMove;
  1513. widestringmanager.UpperWideStringProc:=@GenericWideCase;
  1514. widestringmanager.LowerWideStringProc:=@GenericWideCase;
  1515. {$endif HAS_WIDESTRINGMANAGER}
  1516. widestringmanager.CompareWideStringProc:=@CompareWideString;
  1517. widestringmanager.CompareTextWideStringProc:=@CompareTextWideString;
  1518. widestringmanager.CharLengthPCharProc:=@DefaultCharLengthPChar;
  1519. widestringmanager.CodePointLengthProc:=@DefaultCodePointLength;
  1520. end;