* tokendat compiles for tp

  * tokens.dat supplied by default

compiler/tokendat.pas

@@ -21,190 +21,20 @@
 
  ****************************************************************************
 }
-{$ifdef TP}
 program tokendat;
 
-uses    tokens,globtype;
+{$ifdef FPC}
+  {$FATAL Use tp 7 to compile, FPC can't be used because the records are written different.}
+{$else}
+  {$ifndef TP}
+    -- You need to define -dTP and -dI386
+  {$endif}
+  {$ifndef I386}
+    -- You need to define -dTP and -dI386
+  {$endif}
+{$endif}
 
-{$define IncludeTokens}
-{$define IncludeCreateTokenIndex}
-{$endif TP}
-
-{$ifdef IncludeTokens}
-const
-  arraytokeninfo : ttokenarray =(
-      (str:''              ;special:true ;keyword:m_none),
-    { Operators which can be overloaded }
-      (str:'+'             ;special:true ;keyword:m_none),
-      (str:'-'             ;special:true ;keyword:m_none),
-      (str:'*'             ;special:true ;keyword:m_none),
-      (str:'/'             ;special:true ;keyword:m_none),
-      (str:'='             ;special:true ;keyword:m_none),
-      (str:'>'             ;special:true ;keyword:m_none),
-      (str:'<'             ;special:true ;keyword:m_none),
-      (str:'>='            ;special:true ;keyword:m_none),
-      (str:'<='            ;special:true ;keyword:m_none),
-      (str:'><'            ;special:true ;keyword:m_none),
-      (str:'**'            ;special:true ;keyword:m_none),
-      (str:'is'            ;special:true ;keyword:m_none),
-      (str:'as'            ;special:true ;keyword:m_none),
-      (str:'in'            ;special:true ;keyword:m_none),
-      (str:':='            ;special:true ;keyword:m_none),
-    { Special chars }
-      (str:'^'             ;special:true ;keyword:m_none),
-      (str:'<>'            ;special:true ;keyword:m_none),
-      (str:'['             ;special:true ;keyword:m_none),
-      (str:']'             ;special:true ;keyword:m_none),
-      (str:'.'             ;special:true ;keyword:m_none),
-      (str:','             ;special:true ;keyword:m_none),
-      (str:'('             ;special:true ;keyword:m_none),
-      (str:')'             ;special:true ;keyword:m_none),
-      (str:':'             ;special:true ;keyword:m_none),
-      (str:';'             ;special:true ;keyword:m_none),
-      (str:'@'             ;special:true ;keyword:m_none),
-      (str:'..'            ;special:true ;keyword:m_none),
-      (str:'@@'            ;special:true ;keyword:m_none),
-      (str:'end of file'   ;special:true ;keyword:m_none),
-      (str:'identifier'    ;special:true ;keyword:m_none),
-      (str:'non identifier';special:true ;keyword:m_none),
-      (str:'const real'    ;special:true ;keyword:m_none),
-      (str:'ordinal const' ;special:true ;keyword:m_none),
-      (str:'const string'  ;special:true ;keyword:m_none),
-      (str:'const char'    ;special:true ;keyword:m_none),
-    { C like operators }
-      (str:'+='            ;special:true ;keyword:m_none),
-      (str:'-='            ;special:true ;keyword:m_none),
-      (str:'&='            ;special:true ;keyword:m_none),
-      (str:'|='            ;special:true ;keyword:m_none),
-      (str:'*='            ;special:true ;keyword:m_none),
-      (str:'/='            ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-    { Normal words }
-      (str:'AS'            ;special:false;keyword:m_class),
-      (str:'AT'            ;special:false;keyword:m_none),
-      (str:'DO'            ;special:false;keyword:m_all),
-      (str:'IF'            ;special:false;keyword:m_all),
-      (str:'IN'            ;special:false;keyword:m_all),
-      (str:'IS'            ;special:false;keyword:m_class),
-      (str:'OF'            ;special:false;keyword:m_all),
-      (str:'ON'            ;special:false;keyword:m_class),
-      (str:'OR'            ;special:false;keyword:m_all),
-      (str:'TO'            ;special:false;keyword:m_all),
-      (str:'AND'           ;special:false;keyword:m_all),
-      (str:'ASM'           ;special:false;keyword:m_all),
-      (str:'DIV'           ;special:false;keyword:m_all),
-      (str:'END'           ;special:false;keyword:m_all),
-      (str:'FAR'           ;special:false;keyword:m_none),
-      (str:'FOR'           ;special:false;keyword:m_all),
-      (str:'MOD'           ;special:false;keyword:m_all),
-      (str:'NEW'           ;special:false;keyword:m_all),
-      (str:'NIL'           ;special:false;keyword:m_all),
-      (str:'NOT'           ;special:false;keyword:m_all),
-      (str:'SET'           ;special:false;keyword:m_all),
-      (str:'SHL'           ;special:false;keyword:m_all),
-      (str:'SHR'           ;special:false;keyword:m_all),
-      (str:'TRY'           ;special:false;keyword:m_class),
-      (str:'VAR'           ;special:false;keyword:m_all),
-      (str:'XOR'           ;special:false;keyword:m_all),
-      (str:'CASE'          ;special:false;keyword:m_all),
-      (str:'CVAR'          ;special:false;keyword:m_none),
-      (str:'ELSE'          ;special:false;keyword:m_all),
-      (str:'EXIT'          ;special:false;keyword:m_all),
-      (str:'FAIL'          ;special:false;keyword:m_none), { only set within constructors PM }
-      (str:'FILE'          ;special:false;keyword:m_all),
-      (str:'GOTO'          ;special:false;keyword:m_all),
-      (str:'NAME'          ;special:false;keyword:m_none),
-      (str:'NEAR'          ;special:false;keyword:m_none),
-      (str:'READ'          ;special:false;keyword:m_none),
-      (str:'SELF'          ;special:false;keyword:m_none), {set inside methods only PM }
-      (str:'THEN'          ;special:false;keyword:m_all),
-      (str:'TRUE'          ;special:false;keyword:m_all),
-      (str:'TYPE'          ;special:false;keyword:m_all),
-      (str:'UNIT'          ;special:false;keyword:m_all),
-      (str:'USES'          ;special:false;keyword:m_all),
-      (str:'WITH'          ;special:false;keyword:m_all),
-      (str:'ALIAS'         ;special:false;keyword:m_none),
-      (str:'ARRAY'         ;special:false;keyword:m_all),
-      (str:'BEGIN'         ;special:false;keyword:m_all),
-      (str:'BREAK'         ;special:false;keyword:m_none),
-      (str:'CDECL'         ;special:false;keyword:m_none),
-      (str:'CLASS'         ;special:false;keyword:m_class),
-      (str:'CONST'         ;special:false;keyword:m_all),
-      (str:'FALSE'         ;special:false;keyword:m_all),
-      (str:'INDEX'         ;special:false;keyword:m_none),
-      (str:'LABEL'         ;special:false;keyword:m_all),
-      (str:'RAISE'         ;special:false;keyword:m_class),
-      (str:'UNTIL'         ;special:false;keyword:m_all),
-      (str:'WHILE'         ;special:false;keyword:m_all),
-      (str:'WRITE'         ;special:false;keyword:m_none),
-      (str:'DOWNTO'        ;special:false;keyword:m_all),
-      (str:'EXCEPT'        ;special:false;keyword:m_class),
-      (str:'EXPORT'        ;special:false;keyword:m_none),
-      (str:'INLINE'        ;special:false;keyword:m_none),
-      (str:'OBJECT'        ;special:false;keyword:m_all),
-      (str:'PACKED'        ;special:false;keyword:m_all),
-      (str:'PASCAL'        ;special:false;keyword:m_none),
-      (str:'PUBLIC'        ;special:false;keyword:m_none),
-      (str:'RECORD'        ;special:false;keyword:m_all),
-      (str:'REPEAT'        ;special:false;keyword:m_all),
-      (str:'RESULT'        ;special:false;keyword:m_none),
-      (str:'STATIC'        ;special:false;keyword:m_none),
-      (str:'STORED'        ;special:false;keyword:m_none),
-      (str:'STRING'        ;special:false;keyword:m_all),
-      (str:'SYSTEM'        ;special:false;keyword:m_none),
-      (str:'ASMNAME'       ;special:false;keyword:m_none),
-      (str:'DEFAULT'       ;special:false;keyword:m_none),
-      (str:'DISPOSE'       ;special:false;keyword:m_all),
-      (str:'DYNAMIC'       ;special:false;keyword:m_none),
-      (str:'EXPORTS'       ;special:false;keyword:m_all),
-      (str:'FINALLY'       ;special:false;keyword:m_class),
-      (str:'FORWARD'       ;special:false;keyword:m_none),
-      (str:'IOCHECK'       ;special:false;keyword:m_none),
-      (str:'LIBRARY'       ;special:false;keyword:m_all),
-      (str:'MESSAGE'       ;special:false;keyword:m_none),
-      (str:'PRIVATE'       ;special:false;keyword:m_none),
-      (str:'PROGRAM'       ;special:false;keyword:m_all),
-      (str:'STDCALL'       ;special:false;keyword:m_none),
-      (str:'SYSCALL'       ;special:false;keyword:m_none),
-      (str:'VIRTUAL'       ;special:false;keyword:m_none),
-      (str:'ABSOLUTE'      ;special:false;keyword:m_none),
-      (str:'ABSTRACT'      ;special:false;keyword:m_none),
-      (str:'CONTINUE'      ;special:false;keyword:m_none),
-      (str:'EXTERNAL'      ;special:false;keyword:m_none),
-      (str:'FUNCTION'      ;special:false;keyword:m_all),
-      (str:'OPERATOR'      ;special:false;keyword:m_fpc),
-      (str:'OVERRIDE'      ;special:false;keyword:m_none),
-      (str:'POPSTACK'      ;special:false;keyword:m_none),
-      (str:'PROPERTY'      ;special:false;keyword:m_class),
-      (str:'REGISTER'      ;special:false;keyword:m_none),
-      (str:'RESIDENT'      ;special:false;keyword:m_none),
-      (str:'SAFECALL'      ;special:false;keyword:m_none),
-      (str:'ASSEMBLER'     ;special:false;keyword:m_none),
-      (str:'INHERITED'     ;special:false;keyword:m_all),
-      (str:'INTERFACE'     ;special:false;keyword:m_all),
-      (str:'INTERRUPT'     ;special:false;keyword:m_none),
-      (str:'NODEFAULT'     ;special:false;keyword:m_none),
-      (str:'OTHERWISE'     ;special:false;keyword:m_all),
-      (str:'PROCEDURE'     ;special:false;keyword:m_all),
-      (str:'PROTECTED'     ;special:false;keyword:m_none),
-      (str:'PUBLISHED'     ;special:false;keyword:m_none),
-      (str:'THREADVAR'     ;special:false;keyword:m_class),
-      (str:'DESTRUCTOR'    ;special:false;keyword:m_all),
-      (str:'INTERNPROC'    ;special:false;keyword:m_none),
-      (str:'OPENSTRING'    ;special:false;keyword:m_none),
-      (str:'CONSTRUCTOR'   ;special:false;keyword:m_all),
-      (str:'INTERNCONST'   ;special:false;keyword:m_none),
-      (str:'SHORTSTRING'   ;special:false;keyword:m_none),
-      (str:'FINALIZATION'  ;special:false;keyword:m_initfinal),
-      (str:'SAVEREGISTERS' ;special:false;keyword:m_none),
-      (str:'IMPLEMENTATION';special:false;keyword:m_all),
-      (str:'INITIALIZATION';special:false;keyword:m_initfinal),
-      (str:'RESOURCESTRING';special:false;keyword:m_class)
-  );
-{$endif IncludeTokens}
+uses    tokens;
 
 {Header is designed both to identify the file and to display a nice
  message when you use the type command on it.
@@ -216,35 +46,11 @@ Explanation:
 #26     End of file. Causes type to stop reading the file.
 }
 
-{$ifdef IncludeCreateTokenIndex}
-
-procedure create_tokenidx;
-
-{ create an index with the first and last token for every possible token
- length, so a search only will be done in that small part }
-
-var t : ttoken;
-
-begin
-  fillchar(tokenidx^,sizeof(tokenidx^),0);
-  for t:=low(ttoken) to high(ttoken) do
-   begin
-     if not arraytokeninfo[t].special then
-      begin
-        if ord(tokenidx^[length(arraytokeninfo[t].str),arraytokeninfo[t].str[1]].first)=0 then
-         tokenidx^[length(arraytokeninfo[t].str),arraytokeninfo[t].str[1]].first:=t;
-        tokenidx^[length(arraytokeninfo[t].str),arraytokeninfo[t].str[1]].last:=t;
-      end;
-   end;
-end;
-{$endif IncludeCreateTokenIndex}
-
-{$ifdef TP}
-const   headerstr:string[length(tokheader)]=tokheader;
-
-var f:file;
-    a:longint;
-
+const
+  headerstr:string[length(tokheader)]=tokheader;
+var
+  f:file;
+  a:longint;
 begin
     new(tokenidx);
     create_tokenidx;
@@ -253,14 +59,20 @@ begin
     {Write header...}
     blockwrite(f,headerstr,sizeof(headerstr));
     {Write size of tokeninfo.}
-    a:=sizeof(tokeninfo);
+    a:=sizeof(arraytokeninfo);
     blockwrite(f,a,sizeof(a));
     {Write tokeninfo.}
-    blockwrite(f,tokeninfo,sizeof(tokeninfo));
+    blockwrite(f,arraytokeninfo,sizeof(arraytokeninfo));
     {Write tokenindex.}
     blockwrite(f,tokenidx^,sizeof(tokenidx^));
     close(f);
     dispose(tokenidx);
 end.
+{
+  $Log$
+  Revision 1.4  1999-09-08 16:02:03  peter
+    * tokendat compiles for tp
+    * tokens.dat supplied by default
+
+}
 
-{$endif TP}

compiler/tokens.pas

@@ -25,7 +25,6 @@ interface
 uses
   globtype;
 
-
 const
   tokenidlen=14;
   tokheader=#8'Free Pascal Compiler -- Token data'#13#10#26;
@@ -220,39 +219,218 @@ type
   ptokenidx=^ttokenidx;
   ttokenidx=array[2..tokenidlen,'A'..'Z'] of tokenidxrec;
 
+const
+  arraytokeninfo : ttokenarray =(
+      (str:''              ;special:true ;keyword:m_none),
+    { Operators which can be overloaded }
+      (str:'+'             ;special:true ;keyword:m_none),
+      (str:'-'             ;special:true ;keyword:m_none),
+      (str:'*'             ;special:true ;keyword:m_none),
+      (str:'/'             ;special:true ;keyword:m_none),
+      (str:'='             ;special:true ;keyword:m_none),
+      (str:'>'             ;special:true ;keyword:m_none),
+      (str:'<'             ;special:true ;keyword:m_none),
+      (str:'>='            ;special:true ;keyword:m_none),
+      (str:'<='            ;special:true ;keyword:m_none),
+      (str:'><'            ;special:true ;keyword:m_none),
+      (str:'**'            ;special:true ;keyword:m_none),
+      (str:'is'            ;special:true ;keyword:m_none),
+      (str:'as'            ;special:true ;keyword:m_none),
+      (str:'in'            ;special:true ;keyword:m_none),
+      (str:':='            ;special:true ;keyword:m_none),
+    { Special chars }
+      (str:'^'             ;special:true ;keyword:m_none),
+      (str:'<>'            ;special:true ;keyword:m_none),
+      (str:'['             ;special:true ;keyword:m_none),
+      (str:']'             ;special:true ;keyword:m_none),
+      (str:'.'             ;special:true ;keyword:m_none),
+      (str:','             ;special:true ;keyword:m_none),
+      (str:'('             ;special:true ;keyword:m_none),
+      (str:')'             ;special:true ;keyword:m_none),
+      (str:':'             ;special:true ;keyword:m_none),
+      (str:';'             ;special:true ;keyword:m_none),
+      (str:'@'             ;special:true ;keyword:m_none),
+      (str:'..'            ;special:true ;keyword:m_none),
+      (str:'@@'            ;special:true ;keyword:m_none),
+      (str:'end of file'   ;special:true ;keyword:m_none),
+      (str:'identifier'    ;special:true ;keyword:m_none),
+      (str:'non identifier';special:true ;keyword:m_none),
+      (str:'const real'    ;special:true ;keyword:m_none),
+      (str:'ordinal const' ;special:true ;keyword:m_none),
+      (str:'const string'  ;special:true ;keyword:m_none),
+      (str:'const char'    ;special:true ;keyword:m_none),
+    { C like operators }
+      (str:'+='            ;special:true ;keyword:m_none),
+      (str:'-='            ;special:true ;keyword:m_none),
+      (str:'&='            ;special:true ;keyword:m_none),
+      (str:'|='            ;special:true ;keyword:m_none),
+      (str:'*='            ;special:true ;keyword:m_none),
+      (str:'/='            ;special:true ;keyword:m_none),
+      (str:''              ;special:true ;keyword:m_none),
+      (str:''              ;special:true ;keyword:m_none),
+      (str:''              ;special:true ;keyword:m_none),
+      (str:''              ;special:true ;keyword:m_none),
+    { Normal words }
+      (str:'AS'            ;special:false;keyword:m_class),
+      (str:'AT'            ;special:false;keyword:m_none),
+      (str:'DO'            ;special:false;keyword:m_all),
+      (str:'IF'            ;special:false;keyword:m_all),
+      (str:'IN'            ;special:false;keyword:m_all),
+      (str:'IS'            ;special:false;keyword:m_class),
+      (str:'OF'            ;special:false;keyword:m_all),
+      (str:'ON'            ;special:false;keyword:m_class),
+      (str:'OR'            ;special:false;keyword:m_all),
+      (str:'TO'            ;special:false;keyword:m_all),
+      (str:'AND'           ;special:false;keyword:m_all),
+      (str:'ASM'           ;special:false;keyword:m_all),
+      (str:'DIV'           ;special:false;keyword:m_all),
+      (str:'END'           ;special:false;keyword:m_all),
+      (str:'FAR'           ;special:false;keyword:m_none),
+      (str:'FOR'           ;special:false;keyword:m_all),
+      (str:'MOD'           ;special:false;keyword:m_all),
+      (str:'NEW'           ;special:false;keyword:m_all),
+      (str:'NIL'           ;special:false;keyword:m_all),
+      (str:'NOT'           ;special:false;keyword:m_all),
+      (str:'SET'           ;special:false;keyword:m_all),
+      (str:'SHL'           ;special:false;keyword:m_all),
+      (str:'SHR'           ;special:false;keyword:m_all),
+      (str:'TRY'           ;special:false;keyword:m_class),
+      (str:'VAR'           ;special:false;keyword:m_all),
+      (str:'XOR'           ;special:false;keyword:m_all),
+      (str:'CASE'          ;special:false;keyword:m_all),
+      (str:'CVAR'          ;special:false;keyword:m_none),
+      (str:'ELSE'          ;special:false;keyword:m_all),
+      (str:'EXIT'          ;special:false;keyword:m_all),
+      (str:'FAIL'          ;special:false;keyword:m_none), { only set within constructors PM }
+      (str:'FILE'          ;special:false;keyword:m_all),
+      (str:'GOTO'          ;special:false;keyword:m_all),
+      (str:'NAME'          ;special:false;keyword:m_none),
+      (str:'NEAR'          ;special:false;keyword:m_none),
+      (str:'READ'          ;special:false;keyword:m_none),
+      (str:'SELF'          ;special:false;keyword:m_none), {set inside methods only PM }
+      (str:'THEN'          ;special:false;keyword:m_all),
+      (str:'TRUE'          ;special:false;keyword:m_all),
+      (str:'TYPE'          ;special:false;keyword:m_all),
+      (str:'UNIT'          ;special:false;keyword:m_all),
+      (str:'USES'          ;special:false;keyword:m_all),
+      (str:'WITH'          ;special:false;keyword:m_all),
+      (str:'ALIAS'         ;special:false;keyword:m_none),
+      (str:'ARRAY'         ;special:false;keyword:m_all),
+      (str:'BEGIN'         ;special:false;keyword:m_all),
+      (str:'BREAK'         ;special:false;keyword:m_none),
+      (str:'CDECL'         ;special:false;keyword:m_none),
+      (str:'CLASS'         ;special:false;keyword:m_class),
+      (str:'CONST'         ;special:false;keyword:m_all),
+      (str:'FALSE'         ;special:false;keyword:m_all),
+      (str:'INDEX'         ;special:false;keyword:m_none),
+      (str:'LABEL'         ;special:false;keyword:m_all),
+      (str:'RAISE'         ;special:false;keyword:m_class),
+      (str:'UNTIL'         ;special:false;keyword:m_all),
+      (str:'WHILE'         ;special:false;keyword:m_all),
+      (str:'WRITE'         ;special:false;keyword:m_none),
+      (str:'DOWNTO'        ;special:false;keyword:m_all),
+      (str:'EXCEPT'        ;special:false;keyword:m_class),
+      (str:'EXPORT'        ;special:false;keyword:m_none),
+      (str:'INLINE'        ;special:false;keyword:m_none),
+      (str:'OBJECT'        ;special:false;keyword:m_all),
+      (str:'PACKED'        ;special:false;keyword:m_all),
+      (str:'PASCAL'        ;special:false;keyword:m_none),
+      (str:'PUBLIC'        ;special:false;keyword:m_none),
+      (str:'RECORD'        ;special:false;keyword:m_all),
+      (str:'REPEAT'        ;special:false;keyword:m_all),
+      (str:'RESULT'        ;special:false;keyword:m_none),
+      (str:'STATIC'        ;special:false;keyword:m_none),
+      (str:'STORED'        ;special:false;keyword:m_none),
+      (str:'STRING'        ;special:false;keyword:m_all),
+      (str:'SYSTEM'        ;special:false;keyword:m_none),
+      (str:'ASMNAME'       ;special:false;keyword:m_none),
+      (str:'DEFAULT'       ;special:false;keyword:m_none),
+      (str:'DISPOSE'       ;special:false;keyword:m_all),
+      (str:'DYNAMIC'       ;special:false;keyword:m_none),
+      (str:'EXPORTS'       ;special:false;keyword:m_all),
+      (str:'FINALLY'       ;special:false;keyword:m_class),
+      (str:'FORWARD'       ;special:false;keyword:m_none),
+      (str:'IOCHECK'       ;special:false;keyword:m_none),
+      (str:'LIBRARY'       ;special:false;keyword:m_all),
+      (str:'MESSAGE'       ;special:false;keyword:m_none),
+      (str:'PRIVATE'       ;special:false;keyword:m_none),
+      (str:'PROGRAM'       ;special:false;keyword:m_all),
+      (str:'STDCALL'       ;special:false;keyword:m_none),
+      (str:'SYSCALL'       ;special:false;keyword:m_none),
+      (str:'VIRTUAL'       ;special:false;keyword:m_none),
+      (str:'ABSOLUTE'      ;special:false;keyword:m_none),
+      (str:'ABSTRACT'      ;special:false;keyword:m_none),
+      (str:'CONTINUE'      ;special:false;keyword:m_none),
+      (str:'EXTERNAL'      ;special:false;keyword:m_none),
+      (str:'FUNCTION'      ;special:false;keyword:m_all),
+      (str:'OPERATOR'      ;special:false;keyword:m_fpc),
+      (str:'OVERRIDE'      ;special:false;keyword:m_none),
+      (str:'POPSTACK'      ;special:false;keyword:m_none),
+      (str:'PROPERTY'      ;special:false;keyword:m_class),
+      (str:'REGISTER'      ;special:false;keyword:m_none),
+      (str:'RESIDENT'      ;special:false;keyword:m_none),
+      (str:'SAFECALL'      ;special:false;keyword:m_none),
+      (str:'ASSEMBLER'     ;special:false;keyword:m_none),
+      (str:'INHERITED'     ;special:false;keyword:m_all),
+      (str:'INTERFACE'     ;special:false;keyword:m_all),
+      (str:'INTERRUPT'     ;special:false;keyword:m_none),
+      (str:'NODEFAULT'     ;special:false;keyword:m_none),
+      (str:'OTHERWISE'     ;special:false;keyword:m_all),
+      (str:'PROCEDURE'     ;special:false;keyword:m_all),
+      (str:'PROTECTED'     ;special:false;keyword:m_none),
+      (str:'PUBLISHED'     ;special:false;keyword:m_none),
+      (str:'THREADVAR'     ;special:false;keyword:m_class),
+      (str:'DESTRUCTOR'    ;special:false;keyword:m_all),
+      (str:'INTERNPROC'    ;special:false;keyword:m_none),
+      (str:'OPENSTRING'    ;special:false;keyword:m_none),
+      (str:'CONSTRUCTOR'   ;special:false;keyword:m_all),
+      (str:'INTERNCONST'   ;special:false;keyword:m_none),
+      (str:'SHORTSTRING'   ;special:false;keyword:m_none),
+      (str:'FINALIZATION'  ;special:false;keyword:m_initfinal),
+      (str:'SAVEREGISTERS' ;special:false;keyword:m_none),
+      (str:'IMPLEMENTATION';special:false;keyword:m_all),
+      (str:'INITIALIZATION';special:false;keyword:m_initfinal),
+      (str:'RESOURCESTRING';special:false;keyword:m_class)
+  );
 
-var tokeninfo:ptokenarray;
-    tokenidx:ptokenidx;
+var
+  tokeninfo:ptokenarray;
+  tokenidx:ptokenidx;
 
 procedure inittokens;
 procedure donetokens;
-
-{$ifndef TP}
-{$define IncludeTokens}
-
-{$i tokendat.pas}
-
-{$endif not TP}
+procedure create_tokenidx;
 
 implementation
 
-uses    globals;
+uses
+  globals;
 
-{$ifndef TP}
-{$undef IncludeTokens}
-{$define IncludeCreateTokenIndex}
-{$i tokendat.pas}
-{$endif not TP}
+procedure create_tokenidx;
+{ create an index with the first and last token for every possible token
+  length, so a search only will be done in that small part }
+var
+  t : ttoken;
+begin
+  fillchar(tokenidx^,sizeof(tokenidx^),0);
+  for t:=low(ttoken) to high(ttoken) do
+   begin
+     if not arraytokeninfo[t].special then
+      begin
+        if ord(tokenidx^[length(arraytokeninfo[t].str),arraytokeninfo[t].str[1]].first)=0 then
+         tokenidx^[length(arraytokeninfo[t].str),arraytokeninfo[t].str[1]].first:=t;
+        tokenidx^[length(arraytokeninfo[t].str),arraytokeninfo[t].str[1]].last:=t;
+      end;
+   end;
+end;
 
 procedure inittokens;
-
 {$ifdef TP}
 var
-    f:file;
-    header:string;
-    a:longint;
+  f:file;
+  header:string;
+  a:longint;
 {$endif TP}
-
 begin
 {$ifdef TP}
     assign(f,exepath+'tokens.dat');
@@ -286,8 +464,8 @@ begin
 {$endif not TP}
 end;
 
-procedure donetokens;
 
+procedure donetokens;
 begin
 {$ifdef TP}
     dispose(tokeninfo);
@@ -300,7 +478,11 @@ end;
 end.
 {
   $Log$
-  Revision 1.13  1999-09-03 08:37:34  pierre
+  Revision 1.14  1999-09-08 16:02:04  peter
+    * tokendat compiles for tp
+    * tokens.dat supplied by default
+
+  Revision 1.13  1999/09/03 08:37:34  pierre
     *  tokens.dat only used for TP, and also removed from
        compiler dependencies