Add support for UTF8 .isl files without requiring a special extension. Do keep support for non UTF8 .isl files. Also get rid of SCompilerLanguageNameNotAscii, don't need it anymore (regardless of the former).

Projects/CompMsgs.pas

@@ -325,9 +325,6 @@ const
   SCompilerUnknownLanguage = 'Unknown language name "%s"';
   SCompilerCantSpecifyLanguage = 'A language name may not be specified in a messages file';
   SCompilerCantSpecifyLangOption = 'Language option "%s" cannot be applied to all languages';
-  SCompilerLanguageNameNotAscii = 'LanguageName should not contain non-ASCII characters; ' +
-    'such characters will be interpreted as being from the ISO-8859-1 character set by a non Unicode Setup. ' +
-    'Use "<nnnn>" to embed Unicode characters, where "nnnn" is the 4-digit hexadecimal Unicode character code.';
 
   { [Files] }
   SCompilerFilesTmpBadFlag = 'Parameter "Flags" cannot have the "%s" flag on ' +

Projects/Compile.pas

@@ -2073,9 +2073,6 @@ var
   Lines: TLowFragStringList;
   F: TTextFileReader;
   L: String;
-{$IFDEF UNICODE}
-  S: RawByteString;
-{$ENDIF}
 begin
   Data := CompilerData;
   Filename := AFilename;
@@ -2098,16 +2095,9 @@ begin
   try
     F := TTextFileReader.Create(Filename, fdOpenExisting, faRead, fsRead);
     try
+      F.CodePage := Data.AnsiConvertCodePage;
       while not F.Eof do begin
-{$IFDEF UNICODE}
-        if Data.AnsiConvertCodePage <> 0 then begin
-          { Read the ANSI line, then convert it to Unicode. }
-          S := F.ReadAnsiLine;
-          SetCodePage(S, Data.AnsiConvertCodePage, False);
-          L := String(S);
-        end else
-{$ENDIF}
-          L := F.ReadLine;
+         L := F.ReadLine;
         for I := 1 to Length(L) do
           if L[I] = #0 then
             raise Exception.CreateFmt(SCompilerIllegalNullChar, [Lines.Count + 1]);
@@ -4564,7 +4554,7 @@ end;
 procedure TSetupCompiler.EnumLangOptions(const Line: PChar; const Ext, Ext2: Integer);
 
   procedure ApplyToLangEntry(const KeyName, Value: String;
-    var LangOptions: TSetupLanguageEntry; const AffectsMultipleLangs, AnsiLanguageFile: Boolean);
+    var LangOptions: TSetupLanguageEntry; const AffectsMultipleLangs: Boolean);
   var
     I: Integer;
     Directive: TLangOptionsSectionDirectives;
@@ -4585,14 +4575,12 @@ procedure TSetupCompiler.EnumLangOptions(const Line: PChar; const Ext, Ext2: Int
 
     function ConvertLanguageName(N: String): String;
     var
-      AsciiWarningShown: Boolean;
       I, J, L: Integer;
       W: Word;
     begin
       N := Trim(N);
       if N = '' then
         Invalid;
-      AsciiWarningShown := False;
       Result := '';
       I := 1;
       while I <= Length(N) do begin
@@ -4607,10 +4595,6 @@ procedure TSetupCompiler.EnumLangOptions(const Line: PChar; const Ext, Ext2: Int
           Inc(I, 6);
         end
         else begin
-          if AnsiLanguageFile and (N[I] > #126) and not AsciiWarningShown then begin
-            WarningsList.Add(SCompilerLanguageNameNotAscii);
-            AsciiWarningShown := True;
-          end;
           W := Ord(N[I]);
           Inc(I);
         end;
@@ -4678,17 +4662,15 @@ procedure TSetupCompiler.EnumLangOptions(const Line: PChar; const Ext, Ext2: Int
 var
   KeyName, Value: String;
   I, LangIndex: Integer;
-  AnsiLanguageFile: Boolean;
 begin
   SeparateDirective(Line, KeyName, Value);
   LangIndex := ExtractLangIndex(Self, KeyName, Ext, False);
-  AnsiLanguageFile := Boolean(Ext2);
   if LangIndex = -1 then begin
     for I := 0 to LanguageEntries.Count-1 do
       ApplyToLangEntry(KeyName, Value, PSetupLanguageEntry(LanguageEntries[I])^,
-        LanguageEntries.Count > 1, AnsiLanguageFile);
+        LanguageEntries.Count > 1);
   end else
-    ApplyToLangEntry(KeyName, Value, PSetupLanguageEntry(LanguageEntries[LangIndex])^, False, AnsiLanguageFile);
+    ApplyToLangEntry(KeyName, Value, PSetupLanguageEntry(LanguageEntries[LangIndex])^, False);
 end;
 
 procedure TSetupCompiler.EnumTypes(const Line: PChar; const Ext, Ext2: Integer);
@@ -7267,7 +7249,7 @@ begin
     if Filename = '' then
       Break;
     Filename := PathExpand(PrependSourceDirName(Filename));
-    AnsiLanguageFile := CompareText(PathExtractExt(Filename), '.islu') <> 0;
+    AnsiLanguageFile := not TFile.IsUTF8File(Filename);
     AddStatus(Format(SCompilerStatusReadingInFile, [Filename]));
     EnumIniSection(EnumLangOptionsPre, 'LangOptions', ALangIndex, 0, False, True, Filename, AnsiLanguageFile, True);
     CallIdleProc;
@@ -7287,7 +7269,7 @@ begin
     if Filename = '' then
       Break;
     Filename := PathExpand(PrependSourceDirName(Filename));
-    AnsiLanguageFile := CompareText(PathExtractExt(Filename), '.islu') <> 0;
+    AnsiLanguageFile := not TFile.IsUTF8File(Filename);
     AddStatus(Format(SCompilerStatusReadingInFile, [Filename]));
     EnumIniSection(EnumLangOptions, 'LangOptions', ALangIndex, Integer(AnsiLanguageFile), False, True, Filename, AnsiLanguageFile, False);
     CallIdleProc;

Projects/FileClass.pas

@@ -72,6 +72,7 @@ type
     procedure Seek64(Offset: Integer64); override;
     procedure SeekToEnd;
     procedure Truncate;
+    class function IsUTF8File(const Filename: String): Boolean;
     procedure WriteBuffer(const Buffer; Count: Cardinal); override;
     property Handle: THandle read FHandle;
   end;
@@ -104,7 +105,7 @@ type
     FBuffer: array[0..4095] of AnsiChar;
 {$IFDEF UNICODE}
     FSawFirstLine: Boolean;
-    FUTF8: Boolean;
+    FCodePage: Cardinal;
 {$ENDIF}
     function DoReadLine{$IFDEF UNICODE}(const UTF8: Boolean){$ENDIF}: AnsiString;
     function GetEof: Boolean;
@@ -114,6 +115,7 @@ type
 {$IFDEF UNICODE}
     function ReadAnsiLine: AnsiString;
 {$ENDIF}
+    property CodePage: Cardinal write FCodePage;
     property Eof: Boolean read GetEof;
   end;
 
@@ -307,6 +309,24 @@ begin
     RaiseLastError;
 end;
 
+class function TFile.IsUTF8File(const Filename: String): Boolean;
+var
+  F: TFile;
+  S: RawByteString;
+begin
+  F := TFile.Create(Filename, fdOpenExisting, faRead, fsRead);
+  try
+    if F.Size.Lo > 2 then begin
+      SetLength(S, 3);
+      F.ReadBuffer(S[1], 3);
+      Result := (S[1] = #$EF) and (S[2] = #$BB) and (S[3] = #$BF);
+    end else
+      Result := False;
+  finally
+    F.Free;
+  end;
+end;
+
 procedure TFile.WriteBuffer(const Buffer; Count: Cardinal);
 var
   BytesWritten: DWORD;
@@ -442,19 +462,18 @@ end;
 function TTextFileReader.ReadLine: String;
 {$IFDEF UNICODE}
 var
-  S: AnsiString;
+ S: RawByteString;
 {$ENDIF}
 begin
 {$IFDEF UNICODE}
-  S := DoReadLine(True);
-  if FUTF8 then
-    Result := UTF8ToString(S)
-  else
-    Result := String(S);
+ S := DoReadLine(True);
+ if FCodePage <> 0 then
+   SetCodePage(S, FCodePage, False);
+ Result := String(S);
 {$ELSE}
-  Result := DoReadLine;
+ Result := DoReadLine;
 {$ENDIF}
-end;
+end; 
 
 {$IFDEF UNICODE}
 function TTextFileReader.ReadAnsiLine: AnsiString;
@@ -510,7 +529,7 @@ begin
     { Handle UTF8 BOM if requested }
     if UTF8 and (Length(S) > 2) and (S[1] = #$EF) and (S[2] = #$BB) and (S[3] = #$BF) then begin
       Delete(S, 1, 3);
-      FUTF8 := True;
+      FCodePage := CP_UTF8;
     end;
     FSawFirstLine := True;
   end;