* New keywords for ECMA2021

packages/fcl-js/src/jsscanner.pp

@@ -21,6 +21,18 @@ interface
 
 uses SysUtils, Classes, jstoken;
 
+Type
+  TECMAVersion = (ecma5,ecma2021);
+
+Const
+  // TJSToken is the maximum known set of keywords.
+  // Here we specify what keywords are not keywords for 'older' versions.
+  NonJSKeywords : Array [TECMAVersion] of TJSTokens
+    = (
+        [tjsAwait, tjsClass, tjsConst,tjsDebugger,tjsEnum, tjsExport, tjsExtends, tjsImport, tjsSUPER, tjsYield], //ecma5
+        [] // ecma2022
+      );
+
 resourcestring
   SErrInvalidCharacter = 'Invalid character ''%s''';
   SErrOpenString = 'string exceeds end of line';
@@ -71,6 +83,7 @@ Type
 
   TJSScanner = class
   private
+    FECMAVersion: TECMAVersion;
     FReturnComments: Boolean;
     FReturnWhiteSpace: Boolean;
     FSourceFile: TLineReader;
@@ -83,6 +96,7 @@ Type
     FWasEndOfLine : Boolean;
     FSourceStream : TStream;
     FOwnSourceFile : Boolean;
+    FNonKeyWords : TJSTokens;
     function CommentDiv: TJSToken;
     function DoIdentifier : TJSToken;
     function DoMultiLineComment: TJSToken;
@@ -94,12 +108,13 @@ Type
     function GetCurColumn: Integer;
     function ReadUnicodeEscape: WideChar;
     Function ReadRegex : TJSToken;
+    procedure SetECMAVersion(AValue: TECMAVersion);
   protected
     procedure Error(const Msg: string);overload;
     procedure Error(const Msg: string; Args: array of Const);overload;
   public
-    constructor Create(ALineReader: TLineReader);
-    constructor Create(AStream : TStream);
+    constructor Create(ALineReader: TLineReader; ECMAVersion : TECMAVersion = ecma5);
+    constructor Create(AStream : TStream; ECMAVersion : TECMAVersion = ecma5);
     destructor Destroy; override;
     procedure OpenFile(const AFilename: string);
     Function FetchRegexprToken: TJSToken;
@@ -115,6 +130,7 @@ Type
     property CurColumn: Integer read GetCurColumn;
     property CurToken: TJSToken read FCurToken;
     property CurTokenString: string read FCurTokenString;
+    property ECMAVersion : TECMAVersion Read FECMAVersion Write SetECMAVersion;
   end;
 
 
@@ -146,13 +162,14 @@ begin
   ReadLn(FTextFile, Result);
 end;
 
-constructor TJSScanner.Create(ALineReader: TLineReader);
+constructor TJSScanner.Create(ALineReader: TLineReader; ECMAVersion: TECMAVersion);
 begin
   inherited Create;
   FSourceFile := ALineReader;
+  FNonKeyWords:=NonJSKeywords[ECMAVersion];
 end;
 
-constructor TJSScanner.Create(AStream: TStream);
+constructor TJSScanner.Create(AStream: TStream; ECMAVersion: TECMAVersion);
 begin
   FSourceStream:=ASTream;
   FOwnSourceFile:=True;
@@ -371,6 +388,13 @@ begin
   Result:=tjsRegEx;
 end;
 
+procedure TJSScanner.SetECMAVersion(AValue: TECMAVersion);
+begin
+  if FECMAVersion=AValue then Exit;
+  FECMAVersion:=AValue;
+  FNonKeyWords:=NonJSKeywords[aValue];
+end;
+
 function TJSScanner.DoStringLiteral: TJSToken;
 
 Var
@@ -507,12 +531,13 @@ begin
   // Check if this is a keyword or identifier
   // !!!: Optimize this!
   for i:=FirstKeyword to Lastkeyword do
-    if CurTokenString=TokenInfos[i] then
-      begin
-      Result := i;
-      FCurToken := Result;
-      exit;
-      end;
+    if Not (I in FNonKeyWords) then
+      if (CurTokenString=TokenInfos[i]) then
+        begin
+        Result := i;
+        FCurToken := Result;
+        exit;
+        end;
 end;
 
 Function TJSScanner.FetchToken: TJSToken;

packages/fcl-js/src/jstoken.pp

@@ -38,23 +38,24 @@ type
      tjsRSHIFT, tjsRSHIFTEQ,
      tjsSEQ, tjsSNE, tjsMULEQ,
      { Reserved words start here. They must be last }
-     tjsBREAK,tjsCASE, tjsCATCH, tjsCONTINUE,
-     tjsDEFAULT, tjsDELETE, tjsDO,
-     tjsELSE,
-     tjsFalse, tjsFINALLY, tjsFOR, tjsFUNCTION,
-     tjsIF, tjsIN, tjsINSTANCEOF,
-     tjsNEW,tjsNULL,
+     tjsAWAIT, tjsBREAK, tjsCASE, tjsCATCH, tjsCLASS, tjsCONST, tjsCONTINUE,
+     tjsDEBUGGER, tjsDEFAULT, tjsDELETE, tjsDO,
+     tjsELSE, tjsENUM, tjsEXPORT, tjsEXTENDS,
+     tjsFALSE, tjsFINALLY, tjsFOR, tjsFUNCTION,
+     tjsIF, tjsIMPORT, tjsIN, tjsINSTANCEOF,
+     tjsNEW, tjsNULL,
      tjsRETURN,
-     tjsSWITCH,
+     tjsSUPER, tjsSWITCH,
      tjsTHIS, tjsTHROW, tjsTrue, tjsTRY, tjsTYPEOF,
      tjsVAR, tjsVOID,
      tjsWHILE, tjsWITH,
-     tjsAWAIT
+     tjsYield
    );
+   TJSTokens = Set of TJSToken;
 
 const
-  FirstKeyword = tjsBreak;
-  LastKeyWord = tJSWith;
+  FirstKeyword = tjsAwait;
+  LastKeyWord = tJSYield;
 
   TokenInfos: array[TJSToken] of string = ('unknown',
        // Specials
@@ -74,18 +75,18 @@ const
         '>>', '>>=',
         '===', '!==', '*=',
         // Identifiers last
-        'break','case','catch', 'continue',
-     'default','delete', 'do',
-     'else',
+        'await', 'break','case','catch', 'class','const','continue',
+     'debugger','default','delete', 'do',
+     'else','enum','export','extends',
      'false','finally', 'for', 'function',
-     'if', 'in', 'instanceof',
+     'if', 'import', 'in', 'instanceof',
      'new','null',
      'return',
-     'switch',
+     'super', 'switch',
      'this', 'throw', 'true', 'try', 'typeof',
      'var', 'void',
      'while', 'with',
-     'await'
+     'yield'
     );
 
 

packages/fcl-js/tests/tcscanner.pp

@@ -39,14 +39,14 @@ type
     FScanner : TJSScanner;
     FErrorSource : String;
     procedure AssertEquals(AMessage: String; AExpected, AActual : TJSToken); overload;
-    procedure CheckToken(AToken: TJSToken; ASource: String);
-    procedure CheckTokens(ASource: String; ATokens: array of TJSToken);
+    procedure CheckToken(AToken: TJSToken; ASource: String; aVersion : TECMAVersion = ecma5);
+    procedure CheckTokens(ASource: String; ATokens: array of TJSToken; aVersion : TECMAVersion = ecma5);
     procedure DoTestFloat(F: Double);
     procedure DoTestFloat(F: Double; S: String);
     procedure DoTestString(S: String);
     procedure TestErrorSource;
   protected
-    Function CreateScanner(AInput : String) : TJSScanner;
+    Function CreateScanner(AInput : String; aVersion : TECMAVersion = ecma5) : TJSScanner;
     procedure FreeScanner;
     procedure SetUp; override;
     procedure TearDown; override;
@@ -101,22 +101,40 @@ type
     procedure TestStarEq;
     procedure TestURShift;
     procedure TestURShiftEq;
+    procedure TestAwaitECMA5;
+    procedure TestAwaitECMA2021;
     procedure TestBreak;
     procedure TestCase;
     procedure TestCatch;
+    procedure TestClassECMA5;
+    procedure TestClassECMA2021;
+    procedure TestConstECMA5;
+    procedure TestConstECMA2021;
     procedure TestContinue;
+    procedure TestDebuggerECMA5;
+    procedure TestDebuggerECMA2021;
     procedure TestDefault;
     procedure TestDelete;
     procedure TestDO;
     procedure TestElse;
+    procedure TestEnumECMA5;
+    procedure TestEnumECMA2021;
+    procedure TestExportECMA5;
+    procedure TestExportECMA2021;
+    procedure TestExtendsECMA5;
+    procedure TestExtendsECMA2021;
     procedure TestFinally;
     procedure TestFor;
     procedure TestFunction;
     procedure TestIf;
+    procedure TestImportECMA5;
+    procedure TestImportECMA2021;
     procedure TestIn;
     procedure TestInstanceOf;
     procedure TestNew;
     procedure TestReturn;
+    procedure TestSuperECMA5;
+    procedure TestSuperECMA2021;
     procedure TestSwitch;
     procedure TestThis;
     procedure TestThrow;
@@ -126,6 +144,8 @@ type
     procedure TestVoid;
     procedure TestWhile;
     procedure TestWith;
+    procedure TestYieldECMA5;
+    procedure TestYieldECMA2021;
     Procedure Test2Words;
     procedure Test3Words;
     procedure TestIdentifier;
@@ -147,17 +167,18 @@ type
     procedure TestFloat;
     procedure TestStringError;
     procedure TestFloatError;
+
   end;
 
 
 implementation
 
-Function TTestJSScanner.CreateScanner(AInput : String) : TJSScanner;
+Function TTestJSScanner.CreateScanner(AInput : String; aVersion : TECMAVersion = ecma5) : TJSScanner;
 
 begin
   FStream:=TStringStream.Create(AInput);
   FLineReader:=TStreamLineReader.Create(Fstream);
-  FScanner:=TJSScanner.Create(FLineReader);
+  FScanner:=TJSScanner.Create(FLineReader,aVersion);
   Result:=FScanner;
 end;
 
@@ -202,14 +223,14 @@ begin
     end;
 end;
 
-procedure TTestJSScanner.CheckToken(AToken : TJSToken; ASource : String);
+procedure TTestJSScanner.CheckToken(AToken: TJSToken; ASource: String; aVersion: TECMAVersion);
 
 Var
   J : TJSToken;
   EN2 : String;
 
 begin
-  CreateScanner(ASource);
+  CreateScanner(ASource,aVersion);
   J:=Scanner.FetchToken;
   EN2:=GetEnumName(TypeINfo(TJSToken),Ord(AToken));
   AssertEquals(Format('Source %s should result in %s.',[ASource,EN2]),AToken,J);
@@ -461,6 +482,16 @@ begin
   CheckToken(tjsURSHIFTEQ,'>>>=');
 end;
 
+procedure TTestJSScanner.TestAwaitECMA5;
+begin
+  CheckToken(tjsIdentifier,'await');
+end;
+
+procedure TTestJSScanner.TestAwaitECMA2021;
+begin
+  CheckToken(tjsAwait,'await',ecma2021);
+end;
+
 procedure TTestJSScanner.TestRShift;
 
 begin
@@ -509,12 +540,42 @@ begin
   CheckToken(tjscatch,'catch');
 end;
 
+procedure TTestJSScanner.TestClassECMA5;
+begin
+  CheckToken(tjsIdentifier,'class');
+end;
+
+procedure TTestJSScanner.TestClassECMA2021;
+begin
+  CheckToken(tjsClass,'class',ecma2021);
+end;
+
+procedure TTestJSScanner.TestConstECMA5;
+begin
+  CheckToken(tjsIdentifier,'const');
+end;
+
+procedure TTestJSScanner.TestConstECMA2021;
+begin
+  CheckToken(tjsConst,'const',ecma2021);
+end;
+
 procedure TTestJSScanner.TestContinue;
 
 begin
   CheckToken(tjscontinue,'continue');
 end;
 
+procedure TTestJSScanner.TestDebuggerECMA5;
+begin
+  CheckToken(tjsidentifier,'debugger');
+end;
+
+procedure TTestJSScanner.TestDebuggerECMA2021;
+begin
+  CheckToken(tjsDebugger,'debugger',ecma2021);
+end;
+
 procedure TTestJSScanner.TestDefault;
 
 begin
@@ -539,6 +600,36 @@ begin
   CheckToken(tjselse,'else');
 end;
 
+procedure TTestJSScanner.TestEnumECMA5;
+begin
+  CheckToken(tjsIdentifier,'enum');
+end;
+
+procedure TTestJSScanner.TestEnumECMA2021;
+begin
+  CheckToken(tjsenum,'enum',ecma2021);
+end;
+
+procedure TTestJSScanner.TestExportECMA5;
+begin
+  CheckToken(tjsIdentifier,'export');
+end;
+
+procedure TTestJSScanner.TestExportECMA2021;
+begin
+  CheckToken(tjsexport,'export',ecma2021);
+end;
+
+procedure TTestJSScanner.TestExtendsECMA5;
+begin
+  CheckToken(tjsIdentifier,'extends');
+end;
+
+procedure TTestJSScanner.TestExtendsECMA2021;
+begin
+  CheckToken(tjsextends,'extends',ecma2021);
+end;
+
 procedure TTestJSScanner.TestFinally;
 
 begin
@@ -563,6 +654,16 @@ begin
   CheckToken(tjsif,'if');
 end;
 
+procedure TTestJSScanner.TestImportECMA5;
+begin
+  CheckToken(tjsIdentifier,'import');
+end;
+
+procedure TTestJSScanner.TestImportECMA2021;
+begin
+  CheckToken(tjsImport,'import',ecma2021);
+end;
+
 procedure TTestJSScanner.TestIn;
 
 begin
@@ -587,6 +688,16 @@ begin
   CheckToken(tjsreturn,'return');
 end;
 
+procedure TTestJSScanner.TestSuperECMA5;
+begin
+  CheckToken(tjsIdentifier,'super');
+end;
+
+procedure TTestJSScanner.TestSuperECMA2021;
+begin
+  CheckToken(tjsSuper,'super',ecma2021);
+end;
+
 procedure TTestJSScanner.TestSwitch;
 
 begin
@@ -641,7 +752,17 @@ begin
   CheckToken(tjswith,'with');
 end;
 
-procedure TTestJSScanner.CheckTokens(ASource : String; ATokens : Array of TJSToken);
+procedure TTestJSScanner.TestYieldECMA5;
+begin
+  CheckToken(tjsIdentifier,'yield');
+end;
+
+procedure TTestJSScanner.TestYieldECMA2021;
+begin
+  CheckToken(tjsYield,'yield',ecma2021);
+end;
+
+procedure TTestJSScanner.CheckTokens(ASource : String; ATokens : Array of TJSToken; aVersion: TECMAVersion = ecma5);
 
 Var
   I : Integer;
@@ -649,7 +770,7 @@ Var
   S : String;
 
 begin
-  CreateScanner(ASource);
+  CreateScanner(ASource,aVersion);
   For I:=Low(ATokens) to High(ATokens) do
     begin
     J:=FScanner.FetchToken;