* made cwstring thread safe without locks + test (twide4): widestring

    manager now has two extra parameterless procedures (ThreadInitProc
    and ThreadFiniProc) which are called whenever a thread begins/ends,
    and cwstring uses these to create separate iconv handles for
    each thread (via threadvars)
  * renamed UCS4 to UCS-4BE/LE, because UCS4 is not recognised by most
    systems
  * clean up all iconv handles on exit, and check whether they are
    valid before doing so

git-svn-id: trunk@7949 -

.gitattributes

@@ -7016,6 +7016,7 @@ tests/test/tw6727.pp svneol=native#text/plain
 tests/test/twide1.pp svneol=native#text/plain
 tests/test/twide2.pp svneol=native#text/plain
 tests/test/twide3.pp svneol=native#text/plain
+tests/test/twide4.pp svneol=native#text/plain
 tests/test/twrstr1.pp svneol=native#text/plain
 tests/test/twrstr2.pp svneol=native#text/plain
 tests/test/twrstr3.pp svneol=native#text/plain

rtl/inc/thread.inc

@@ -30,6 +30,10 @@ Var
 {$endif HAS_MEMORYMANAGER}
         if MemoryManager.InitThread <> nil then
           MemoryManager.InitThread();
+{$ifdef FPC_HAS_FEATURE_WIDESTRINGS}
+        if assigned(widestringmanager.ThreadInitProc) then
+          widestringmanager.ThreadInitProc;
+{$endif FPC_HAS_FEATURE_WIDESTRINGS}
         { ExceptAddrStack and ExceptObjectStack are threadvars       }
         { so every thread has its on exception handling capabilities }
         SysInitExceptions;
@@ -45,6 +49,10 @@ Var
 
     procedure DoneThread;
       begin
+{$ifdef FPC_HAS_FEATURE_WIDESTRINGS}
+        if assigned(widestringmanager.ThreadFiniProc) then
+          widestringmanager.ThreadFiniProc;
+{$endif FPC_HAS_FEATURE_WIDESTRINGS}
 {$ifndef HAS_MEMORYMANAGER}
         FinalizeHeap;
 {$endif HAS_MEMORYMANAGER}

rtl/inc/wstringh.inc

@@ -71,6 +71,8 @@ Type
     StrLICompAnsiStringProc : function(S1, S2: PChar; MaxLen: PtrUInt): PtrInt;
     StrLowerAnsiStringProc : function(Str: PChar): PChar;
     StrUpperAnsiStringProc : function(Str: PChar): PChar;
+    ThreadInitProc : procedure;
+    ThreadFiniProc : procedure;
   end;
 
 

rtl/unix/cwstring.pp

@@ -44,16 +44,7 @@ Const
     libiconvname='iconv';
 {$endif}
 
-{ Case-mapping "arrays" }
-var
-  AnsiUpperChars: AnsiString; // 1..255
-  AnsiLowerChars: AnsiString; // 1..255
-  WideUpperChars: WideString; // 1..65535
-  WideLowerChars: WideString; // 1..65535
-
-{ the following declarations are from the libc unit for linux so they
-  might be very linux centric
-  maybe this needs to be splitted in an os depend way later }
+{ helper functions from libc }
 function towlower(__wc:wint_t):wint_t;cdecl;external libiconvname name 'towlower';
 function towupper(__wc:wint_t):wint_t;cdecl;external libiconvname name 'towupper';
 function wcscoll (__s1:pwchar_t; __s2:pwchar_t):cint;cdecl;external libiconvname name 'wcscoll';
@@ -92,9 +83,11 @@ const
 
 { unicode encoding name }
 {$ifdef FPC_LITTLE_ENDIAN}
-  unicode_encoding = 'UTF-16LE';
+  unicode_encoding2 = 'UTF-16LE';
+  unicode_encoding4 = 'UCS-4LE'; 
 {$else  FPC_LITTLE_ENDIAN}
-  unicode_encoding = 'UTF-16BE';
+  unicode_encoding2 = 'UTF-16BE';
+  unicode_encoding4 = 'UCS-4BE';
 {$endif  FPC_LITTLE_ENDIAN}
 
 type
@@ -113,33 +106,11 @@ function iconv(__cd:iconv_t; __inbuf:ppchar; __inbytesleft:psize_t; __outbuf:ppc
 function iconv_close(__cd:iconv_t):cint;cdecl;external libiconvname name 'libiconv_close';
 {$endif}
 
-var
+threadvar
   iconv_ansi2ucs4,
   iconv_ucs42ansi,
   iconv_ansi2wide,
   iconv_wide2ansi : iconv_t;
-  
-  lock_ansi2ucs4 : integer = -1;
-  lock_ucs42ansi : integer = -1;
-  lock_ansi2wide : integer = -1;
-  lock_wide2ansi : integer = -1;
-
-  iconv_lock : TRTLcriticalsection;
-
-{
-procedure lockiconv(var lockcount: integer);
-begin
- while interlockedincrement(lockcount) <> 0 do begin
-  interlockeddecrement(lockcount);
-  sleep(0);
- end;
-end;
-
-procedure unlockiconv(var lockcount: integer);
-begin
- interlockeddecrement(lockcount);
-end;
-}
  
 procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
   var
@@ -161,7 +132,6 @@ procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
     srcpos:=source;
     destpos:=pchar(dest);
     outleft:=outlength;
-    entercriticalsection(iconv_lock);
     while iconv(iconv_wide2ansi,ppchar(@srcpos),@srclen,@destpos,@outleft)=size_t(-1) do
       begin
         case fpgetCerrno of
@@ -187,11 +157,9 @@ procedure Wide2AnsiMove(source:pwidechar;var dest:ansistring;len:SizeInt);
               destpos:=pchar(dest)+outoffset;
             end;
           else
-            leavecriticalsection(iconv_lock);
             runerror(231);
         end;
       end;
-    leavecriticalsection(iconv_lock);
     // truncate string
     setlength(dest,length(dest)-outleft);
   end;
@@ -216,7 +184,6 @@ procedure Ansi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
     srcpos:=source;
     destpos:=pchar(dest);
     outleft:=outlength*2;
-    entercriticalsection(iconv_lock);
     while iconv(iconv_ansi2wide,@srcpos,psize(@len),@destpos,@outleft)=size_t(-1) do
       begin
         case fpgetCerrno of
@@ -242,11 +209,9 @@ procedure Ansi2WideMove(source:pchar;var dest:widestring;len:SizeInt);
               destpos:=pchar(dest)+outoffset;
             end;
           else
-            leavecriticalsection(iconv_lock);
             runerror(231);
         end;
       end;
-    leavecriticalsection(iconv_lock);
     // truncate string
     setlength(dest,length(dest)-outleft div 2);
   end;
@@ -291,7 +256,6 @@ procedure Ansi2UCS4Move(source:pchar;var dest:UCS4String;len:SizeInt);
     srcpos:=source;
     destpos:=pchar(dest);
     outleft:=outlength*4;
-    entercriticalsection(iconv_lock);
     while iconv(iconv_ansi2ucs4,@srcpos,psize(@len),@destpos,@outleft)=size_t(-1) do
       begin
         case fpgetCerrno of
@@ -306,11 +270,9 @@ procedure Ansi2UCS4Move(source:pchar;var dest:UCS4String;len:SizeInt);
               destpos:=pchar(dest)+outoffset;
             end;
           else
-            leavecriticalsection(iconv_lock);
             runerror(231);
         end;
       end;
-    leavecriticalsection(iconv_lock);
     // truncate string
     setlength(dest,length(dest)-outleft div 4);
   end;
@@ -338,6 +300,28 @@ function StrCompAnsi(s1,s2 : PChar): PtrInt;
   end;
 
 
+procedure InitThread;
+begin
+  iconv_wide2ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding2);
+  iconv_ansi2wide:=iconv_open(unicode_encoding2,nl_langinfo(CODESET));
+  iconv_ucs42ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding4);
+  iconv_ansi2ucs4:=iconv_open(unicode_encoding4,nl_langinfo(CODESET));
+end;
+
+
+procedure FiniThread;
+begin
+  if (iconv_wide2ansi <> iconv_t(-1)) then
+    iconv_close(iconv_wide2ansi);
+  if (iconv_ansi2wide <> iconv_t(-1)) then
+    iconv_close(iconv_ansi2wide);
+  if (iconv_ucs42ansi <> iconv_t(-1)) then
+    iconv_close(iconv_ucs42ansi);
+  if (iconv_ansi2ucs4 <> iconv_t(-1)) then
+    iconv_close(iconv_ansi2ucs4);
+end;
+
+
 Procedure SetCWideStringManager;
 Var
   CWideStringManager : TWideStringManager;
@@ -369,6 +353,8 @@ begin
       StrLowerAnsiStringProc
       StrUpperAnsiStringProc
       }
+      ThreadInitProc:=@InitThread;
+      ThreadFiniProc:=@FiniThread;
     end;
   SetWideStringManager(CWideStringManager);
 end;
@@ -376,19 +362,15 @@ end;
 
 initialization
   SetCWideStringManager;
-  initcriticalsection(iconv_lock);
 
   { you have to call setlocale(LC_ALL,'') to initialise the langinfo stuff  }
   { with the information from the environment variables according to POSIX  }
   { (some OSes do this automatically, but e.g. Darwin and Solaris don't)    }
   setlocale(LC_ALL,'');
 
-  { init conversion tables }
-  iconv_wide2ansi:=iconv_open(nl_langinfo(CODESET),unicode_encoding);
-  iconv_ansi2wide:=iconv_open(unicode_encoding,nl_langinfo(CODESET));
-  iconv_ucs42ansi:=iconv_open(nl_langinfo(CODESET),'UCS4');
-  iconv_ansi2ucs4:=iconv_open('UCS4',nl_langinfo(CODESET));
+  { init conversion tables for main program }
+  InitThread;
 finalization
-  donecriticalsection(iconv_lock);
-  iconv_close(iconv_ansi2wide);
+  { fini conversion tables for main program }
+  FiniThread;
 end.

tests/test/twide4.pp

@@ -0,0 +1,92 @@
+
+{$ifdef fpc}
+{$mode objfpc}
+{$endif fpc}
+
+uses
+{$ifdef unix}
+  cthreads, cwstring,
+{$endif}
+  Classes, SysUtils;
+
+type
+  tc = class(tthread)
+    orgstr: ansistring;
+    cnvstr: widestring;
+    constructor create(const s: ansistring; const w: widestring);
+    procedure execute; override;
+  end;
+
+const
+  // string with an invalid utf-8 code sequence
+  str1 = #$c1#$34'Życie'#$c1#$34' jest jak papier '#$c1#$34'toaletowy'#$c1#$34' : długie, szare i '#$c1#$34'do'#$c1#$34' dupy';
+  str2 = 'Życie '#$c1#$34'jest'#$c1#$34' jak papier toaletowy : '#$c1#$34'długie'#$c1#$34', szare i do '#$c1#$34'dupy'#$c1#$34'222222222222222222222222222222222222222222222222';
+  str3 = 'Życie jest '#$c1#$34'jak'#$c1#$34' papier 333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333 toaletowy : długie, '#$c1#$34'szare'#$c1#$34' i do dupy';
+  str4 = 'Życie jest 4444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444 jak '#$c1#$34'papier'#$c1#$34' toaletowy : długie, szare '#$c1#$34'i'#$c1#$34' do dupy';
+  count = 20000;
+
+var
+  wstr: widestring;
+//  cnvstr: ansistring;
+  error: boolean;
+
+
+constructor tc.create(const s: ansistring; const w: widestring);
+begin
+  orgstr:=s;
+  cnvstr:=w;
+  inherited create(true);
+end;
+
+
+procedure tc.execute;
+var
+  i: longint;
+  w: widestring;
+begin
+  for i := 1 to count do
+    begin
+      w:=orgstr;
+      if (w<>cnvstr) then
+        error:=true;
+    end;
+end;
+
+var
+  a: array[1..4] of tc;
+  w1,w2,w3,w4: widestring;
+  cnvstr: ansistring;
+begin
+  error:=false;
+  cnvstr:=str1;
+  w1:=cnvstr;
+  cnvstr:=str2;
+  w2:=cnvstr;
+  cnvstr:=str3;
+  w3:=cnvstr;
+  cnvstr:=str4;
+  w4:=cnvstr;
+  writeln(w1);
+  writeln(w2);
+  writeln(w3);
+  writeln(w4);
+  a[1]:=tc.create(str1,w1);
+  a[2]:=tc.create(str2,w2);
+  a[3]:=tc.create(str3,w3);
+  a[4]:=tc.create(str4,w4);
+  a[1].resume;
+  a[2].resume;
+  a[3].resume;
+  a[4].resume;
+  a[1].waitfor;
+  a[2].waitfor;
+  a[3].waitfor;
+  a[4].waitfor;
+  a[1].free;
+  a[2].free;
+  a[3].free;
+  a[4].free;
+  
+  if error then
+    halt(1);  
+end.