* bugfix of reading file with TP conditional

ide/fake/compiler/tokens.pas

@@ -1,6 +1,6 @@
 {
     $Id$
-    Copyright (c) 1993-98 by Florian Klaempfl, Pierre Muller
+    Copyright (c) 1998-2000 by Florian Klaempfl, Pierre Muller
 
     Tokens used by the compiler
 
@@ -43,9 +43,16 @@ type
     _LTE,
     _SYMDIF,
     _STARSTAR,
-    _OP_IS,
     _OP_AS,
     _OP_IN,
+    _OP_IS,
+    _OP_OR,
+    _OP_AND,
+    _OP_DIV,
+    _OP_MOD,
+    _OP_SHL,
+    _OP_SHR,
+    _OP_XOR,
     _ASSIGNMENT,
     { special chars }
     _CARET,
@@ -206,6 +213,7 @@ type
     str     : string[tokenidlen];
     special : boolean;
     keyword : tmodeswitch;
+    op      : ttoken;
     encoded : longint;
   end;
 
@@ -221,176 +229,183 @@ type
 
 const
   arraytokeninfo : ttokenarray =(
-      (str:''              ;special:true ;keyword:m_none),
+      (str:''              ;special:true ;keyword:m_none;op:NOTOKEN),
     { Operators which can be overloaded }
-      (str:'+'             ;special:true ;keyword:m_none),
-      (str:'-'             ;special:true ;keyword:m_none),
-      (str:'*'             ;special:true ;keyword:m_none),
-      (str:'/'             ;special:true ;keyword:m_none),
-      (str:'='             ;special:true ;keyword:m_none),
-      (str:'>'             ;special:true ;keyword:m_none),
-      (str:'<'             ;special:true ;keyword:m_none),
-      (str:'>='            ;special:true ;keyword:m_none),
-      (str:'<='            ;special:true ;keyword:m_none),
-      (str:'><'            ;special:true ;keyword:m_none),
-      (str:'**'            ;special:true ;keyword:m_none),
-      (str:'is'            ;special:true ;keyword:m_none),
-      (str:'as'            ;special:true ;keyword:m_none),
-      (str:'in'            ;special:true ;keyword:m_none),
-      (str:':='            ;special:true ;keyword:m_none),
+      (str:'+'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'-'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'*'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'/'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'='             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'>'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'<'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'>='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'<='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'><'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'**'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'as'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'in'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'is'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'or'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'and'           ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'div'           ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'mod'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'shl'           ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'shr'           ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'xor'           ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:':='            ;special:true ;keyword:m_none;op:NOTOKEN),
     { Special chars }
-      (str:'^'             ;special:true ;keyword:m_none),
-      (str:'<>'            ;special:true ;keyword:m_none),
-      (str:'['             ;special:true ;keyword:m_none),
-      (str:']'             ;special:true ;keyword:m_none),
-      (str:'.'             ;special:true ;keyword:m_none),
-      (str:','             ;special:true ;keyword:m_none),
-      (str:'('             ;special:true ;keyword:m_none),
-      (str:')'             ;special:true ;keyword:m_none),
-      (str:':'             ;special:true ;keyword:m_none),
-      (str:';'             ;special:true ;keyword:m_none),
-      (str:'@'             ;special:true ;keyword:m_none),
-      (str:'..'            ;special:true ;keyword:m_none),
-      (str:'@@'            ;special:true ;keyword:m_none),
-      (str:'end of file'   ;special:true ;keyword:m_none),
-      (str:'identifier'    ;special:true ;keyword:m_none),
-      (str:'non identifier';special:true ;keyword:m_none),
-      (str:'const real'    ;special:true ;keyword:m_none),
-      (str:'ordinal const' ;special:true ;keyword:m_none),
-      (str:'const string'  ;special:true ;keyword:m_none),
-      (str:'const char'    ;special:true ;keyword:m_none),
+      (str:'^'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'<>'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'['             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:']'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'.'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:','             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'('             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:')'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:':'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:';'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'@'             ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'..'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'@@'            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'end of file'   ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'identifier'    ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'non identifier';special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'const real'    ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'ordinal const' ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'const string'  ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'const char'    ;special:true ;keyword:m_none;op:NOTOKEN),
     { C like operators }
-      (str:'+='            ;special:true ;keyword:m_none),
-      (str:'-='            ;special:true ;keyword:m_none),
-      (str:'&='            ;special:true ;keyword:m_none),
-      (str:'|='            ;special:true ;keyword:m_none),
-      (str:'*='            ;special:true ;keyword:m_none),
-      (str:'/='            ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
-      (str:''              ;special:true ;keyword:m_none),
+      (str:'+='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'-='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'&='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'|='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'*='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:'/='            ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:''              ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:''              ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:''              ;special:true ;keyword:m_none;op:NOTOKEN),
+      (str:''              ;special:true ;keyword:m_none;op:NOTOKEN),
     { Normal words }
-      (str:'AS'            ;special:false;keyword:m_class),
-      (str:'AT'            ;special:false;keyword:m_none),
-      (str:'DO'            ;special:false;keyword:m_all),
-      (str:'IF'            ;special:false;keyword:m_all),
-      (str:'IN'            ;special:false;keyword:m_all),
-      (str:'IS'            ;special:false;keyword:m_class),
-      (str:'OF'            ;special:false;keyword:m_all),
-      (str:'ON'            ;special:false;keyword:m_class),
-      (str:'OR'            ;special:false;keyword:m_all),
-      (str:'TO'            ;special:false;keyword:m_all),
-      (str:'AND'           ;special:false;keyword:m_all),
-      (str:'ASM'           ;special:false;keyword:m_all),
-      (str:'DIV'           ;special:false;keyword:m_all),
-      (str:'END'           ;special:false;keyword:m_all),
-      (str:'FAR'           ;special:false;keyword:m_none),
-      (str:'FOR'           ;special:false;keyword:m_all),
-      (str:'MOD'           ;special:false;keyword:m_all),
-      (str:'NEW'           ;special:false;keyword:m_all),
-      (str:'NIL'           ;special:false;keyword:m_all),
-      (str:'NOT'           ;special:false;keyword:m_all),
-      (str:'SET'           ;special:false;keyword:m_all),
-      (str:'SHL'           ;special:false;keyword:m_all),
-      (str:'SHR'           ;special:false;keyword:m_all),
-      (str:'TRY'           ;special:false;keyword:m_class),
-      (str:'VAR'           ;special:false;keyword:m_all),
-      (str:'XOR'           ;special:false;keyword:m_all),
-      (str:'CASE'          ;special:false;keyword:m_all),
-      (str:'CVAR'          ;special:false;keyword:m_none),
-      (str:'ELSE'          ;special:false;keyword:m_all),
-      (str:'EXIT'          ;special:false;keyword:m_all),
-      (str:'FAIL'          ;special:false;keyword:m_none), { only set within constructors PM }
-      (str:'FILE'          ;special:false;keyword:m_all),
-      (str:'GOTO'          ;special:false;keyword:m_all),
-      (str:'NAME'          ;special:false;keyword:m_none),
-      (str:'NEAR'          ;special:false;keyword:m_none),
-      (str:'READ'          ;special:false;keyword:m_none),
-      (str:'SELF'          ;special:false;keyword:m_none), {set inside methods only PM }
-      (str:'THEN'          ;special:false;keyword:m_all),
-      (str:'TRUE'          ;special:false;keyword:m_all),
-      (str:'TYPE'          ;special:false;keyword:m_all),
-      (str:'UNIT'          ;special:false;keyword:m_all),
-      (str:'USES'          ;special:false;keyword:m_all),
-      (str:'WITH'          ;special:false;keyword:m_all),
-      (str:'ALIAS'         ;special:false;keyword:m_none),
-      (str:'ARRAY'         ;special:false;keyword:m_all),
-      (str:'BEGIN'         ;special:false;keyword:m_all),
-      (str:'BREAK'         ;special:false;keyword:m_none),
-      (str:'CDECL'         ;special:false;keyword:m_none),
-      (str:'CLASS'         ;special:false;keyword:m_class),
-      (str:'CONST'         ;special:false;keyword:m_all),
-      (str:'FALSE'         ;special:false;keyword:m_all),
-      (str:'INDEX'         ;special:false;keyword:m_none),
-      (str:'LABEL'         ;special:false;keyword:m_all),
-      (str:'RAISE'         ;special:false;keyword:m_class),
-      (str:'UNTIL'         ;special:false;keyword:m_all),
-      (str:'WHILE'         ;special:false;keyword:m_all),
-      (str:'WRITE'         ;special:false;keyword:m_none),
-      (str:'DOWNTO'        ;special:false;keyword:m_all),
-      (str:'EXCEPT'        ;special:false;keyword:m_class),
-      (str:'EXPORT'        ;special:false;keyword:m_none),
-      (str:'INLINE'        ;special:false;keyword:m_none),
-      (str:'OBJECT'        ;special:false;keyword:m_all),
-      (str:'PACKED'        ;special:false;keyword:m_all),
-      (str:'PASCAL'        ;special:false;keyword:m_none),
-      (str:'PUBLIC'        ;special:false;keyword:m_none),
-      (str:'RECORD'        ;special:false;keyword:m_all),
-      (str:'REPEAT'        ;special:false;keyword:m_all),
-      (str:'RESULT'        ;special:false;keyword:m_none),
-      (str:'STATIC'        ;special:false;keyword:m_none),
-      (str:'STORED'        ;special:false;keyword:m_none),
-      (str:'STRING'        ;special:false;keyword:m_all),
-      (str:'SYSTEM'        ;special:false;keyword:m_none),
-      (str:'ASMNAME'       ;special:false;keyword:m_none),
-      (str:'DEFAULT'       ;special:false;keyword:m_none),
-      (str:'DISPOSE'       ;special:false;keyword:m_all),
-      (str:'DYNAMIC'       ;special:false;keyword:m_none),
-      (str:'EXPORTS'       ;special:false;keyword:m_all),
-      (str:'FINALLY'       ;special:false;keyword:m_class),
-      (str:'FORWARD'       ;special:false;keyword:m_none),
-      (str:'IOCHECK'       ;special:false;keyword:m_none),
-      (str:'LIBRARY'       ;special:false;keyword:m_all),
-      (str:'MESSAGE'       ;special:false;keyword:m_none),
-      (str:'PRIVATE'       ;special:false;keyword:m_none),
-      (str:'PROGRAM'       ;special:false;keyword:m_all),
-      (str:'STDCALL'       ;special:false;keyword:m_none),
-      (str:'SYSCALL'       ;special:false;keyword:m_none),
-      (str:'VIRTUAL'       ;special:false;keyword:m_none),
-      (str:'ABSOLUTE'      ;special:false;keyword:m_none),
-      (str:'ABSTRACT'      ;special:false;keyword:m_none),
-      (str:'CONTINUE'      ;special:false;keyword:m_none),
-      (str:'EXTERNAL'      ;special:false;keyword:m_none),
-      (str:'FUNCTION'      ;special:false;keyword:m_all),
-      (str:'OPERATOR'      ;special:false;keyword:m_fpc),
-      (str:'OVERRIDE'      ;special:false;keyword:m_none),
-      (str:'POPSTACK'      ;special:false;keyword:m_none),
-      (str:'PROPERTY'      ;special:false;keyword:m_class),
-      (str:'REGISTER'      ;special:false;keyword:m_none),
-      (str:'RESIDENT'      ;special:false;keyword:m_none),
-      (str:'SAFECALL'      ;special:false;keyword:m_none),
-      (str:'ASSEMBLER'     ;special:false;keyword:m_none),
-      (str:'INHERITED'     ;special:false;keyword:m_all),
-      (str:'INTERFACE'     ;special:false;keyword:m_all),
-      (str:'INTERRUPT'     ;special:false;keyword:m_none),
-      (str:'NODEFAULT'     ;special:false;keyword:m_none),
-      (str:'OTHERWISE'     ;special:false;keyword:m_all),
-      (str:'PROCEDURE'     ;special:false;keyword:m_all),
-      (str:'PROTECTED'     ;special:false;keyword:m_none),
-      (str:'PUBLISHED'     ;special:false;keyword:m_none),
-      (str:'THREADVAR'     ;special:false;keyword:m_class),
-      (str:'DESTRUCTOR'    ;special:false;keyword:m_all),
-      (str:'INTERNPROC'    ;special:false;keyword:m_none),
-      (str:'OPENSTRING'    ;special:false;keyword:m_none),
-      (str:'CONSTRUCTOR'   ;special:false;keyword:m_all),
-      (str:'INTERNCONST'   ;special:false;keyword:m_none),
-      (str:'SHORTSTRING'   ;special:false;keyword:m_none),
-      (str:'FINALIZATION'  ;special:false;keyword:m_initfinal),
-      (str:'SAVEREGISTERS' ;special:false;keyword:m_none),
-      (str:'IMPLEMENTATION';special:false;keyword:m_all),
-      (str:'INITIALIZATION';special:false;keyword:m_initfinal),
-      (str:'RESOURCESTRING';special:false;keyword:m_class)
+      (str:'AS'            ;special:false;keyword:m_class;op:_OP_AS),
+      (str:'AT'            ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'DO'            ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'IF'            ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'IN'            ;special:false;keyword:m_all;op:_OP_IN),
+      (str:'IS'            ;special:false;keyword:m_class;op:_OP_IS),
+      (str:'OF'            ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'ON'            ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'OR'            ;special:false;keyword:m_all;op:_OP_OR),
+      (str:'TO'            ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'AND'           ;special:false;keyword:m_all;op:_OP_AND),
+      (str:'ASM'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'DIV'           ;special:false;keyword:m_all;op:_OP_DIV),
+      (str:'END'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'FAR'           ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'FOR'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'MOD'           ;special:false;keyword:m_all;op:_OP_MOD),
+      (str:'NEW'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'NIL'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'NOT'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'SET'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'SHL'           ;special:false;keyword:m_all;op:_OP_SHL),
+      (str:'SHR'           ;special:false;keyword:m_all;op:_OP_SHR),
+      (str:'TRY'           ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'VAR'           ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'XOR'           ;special:false;keyword:m_all;op:_OP_XOR),
+      (str:'CASE'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'CVAR'          ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'ELSE'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'EXIT'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'FAIL'          ;special:false;keyword:m_none;op:NOTOKEN), { only set within constructors PM }
+      (str:'FILE'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'GOTO'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'NAME'          ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'NEAR'          ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'READ'          ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'SELF'          ;special:false;keyword:m_none;op:NOTOKEN), {set inside methods only PM }
+      (str:'THEN'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'TRUE'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'TYPE'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'UNIT'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'USES'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'WITH'          ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'ALIAS'         ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'ARRAY'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'BEGIN'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'BREAK'         ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'CDECL'         ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'CLASS'         ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'CONST'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'FALSE'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'INDEX'         ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'LABEL'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'RAISE'         ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'UNTIL'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'WHILE'         ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'WRITE'         ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'DOWNTO'        ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'EXCEPT'        ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'EXPORT'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'INLINE'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'OBJECT'        ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'PACKED'        ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'PASCAL'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'PUBLIC'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'RECORD'        ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'REPEAT'        ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'RESULT'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'STATIC'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'STORED'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'STRING'        ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'SYSTEM'        ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'ASMNAME'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'DEFAULT'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'DISPOSE'       ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'DYNAMIC'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'EXPORTS'       ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'FINALLY'       ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'FORWARD'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'IOCHECK'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'LIBRARY'       ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'MESSAGE'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'PRIVATE'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'PROGRAM'       ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'STDCALL'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'SYSCALL'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'VIRTUAL'       ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'ABSOLUTE'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'ABSTRACT'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'CONTINUE'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'EXTERNAL'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'FUNCTION'      ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'OPERATOR'      ;special:false;keyword:m_fpc;op:NOTOKEN),
+      (str:'OVERRIDE'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'POPSTACK'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'PROPERTY'      ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'REGISTER'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'RESIDENT'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'SAFECALL'      ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'ASSEMBLER'     ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'INHERITED'     ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'INTERFACE'     ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'INTERRUPT'     ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'NODEFAULT'     ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'OTHERWISE'     ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'PROCEDURE'     ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'PROTECTED'     ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'PUBLISHED'     ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'THREADVAR'     ;special:false;keyword:m_class;op:NOTOKEN),
+      (str:'DESTRUCTOR'    ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'INTERNPROC'    ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'OPENSTRING'    ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'CONSTRUCTOR'   ;special:false;keyword:m_all;op:NOTOKEN),
+      (str:'INTERNCONST'   ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'SHORTSTRING'   ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'FINALIZATION'  ;special:false;keyword:m_initfinal;op:NOTOKEN),
+      (str:'SAVEREGISTERS' ;special:false;keyword:m_none;op:NOTOKEN),
+      (str:'IMPLEMENTATION';special:false;keyword:m_all;op:NOTOKEN),
+      (str:'INITIALIZATION';special:false;keyword:m_initfinal;op:NOTOKEN),
+      (str:'RESOURCESTRING';special:false;keyword:m_class;op:NOTOKEN)
   );
 
 var
@@ -408,13 +423,6 @@ uses
   dos;
 {$endif}
 
-const
-{$ifdef LINUX}
-   DirSep = '/';
-{$else}
-   DirSep = '\';
-{$endif}
-
 procedure create_tokenidx;
 { create an index with the first and last token for every possible token
   length, so a search only will be done in that small part }
@@ -446,20 +454,20 @@ var
 begin
 {$ifdef TP}
     fsplit(paramstr(0),d,n,e);
-    if copy(d,length(d),1)<>DirSep then
-      d:=d+DirSep;
-    assign(f,d+'tokens.dat');
+    { when debugging d=''!!!! FK }
+    if d='' then
+      assign(f,'tokens.dat')
+    else
+      assign(f,d+'tokens.dat');
     {$I-}
     reset(f,1);
     {We are not sure that the msg file is loaded!}
     if ioresult<>0 then
         begin
-            close(f);
             { Very nice indeed !!! PM }
             writeln('Fatal: File tokens.dat not found.');
+            close(f);
             halt(3);
-            { FV restores screen contents on exit, so this won't be visible
-              for the user anyway... :( - Gabor }
         end;
     blockread(f,header,1);
     blockread(f,header[1],length(header));
@@ -467,8 +475,8 @@ begin
     if (ioresult<>0) or
        (header<>tokheader) or (a<>sizeof(ttokenarray)) then
      begin
-       close(f);
        writeln('Fatal: File tokens.dat corrupt.');
+       close(f);
        halt(3);
      end;
     new(tokeninfo);
@@ -503,11 +511,54 @@ end;
 end.
 {
   $Log$
-  Revision 1.4  2000-02-07 08:30:12  michael
-  [*] the fake (!) TOKENS.PAS still contained the typo bug
-       FSplit(,n,d,e) (correctly FSplit(,d,n,e))
+  Revision 1.5  2000-02-12 23:55:26  carl
+    * bugfix of reading file with TP conditional
+
+  Revision 1.20  2000/02/09 13:23:08  peter
+    * log truncated
+
+  Revision 1.19  2000/01/07 01:14:48  peter
+    * updated copyright to 2000
+
+  Revision 1.18  1999/11/15 17:53:00  pierre
+    + one field added for ttoken record for operator
+      linking the id to the corresponding operator token that
+      can now now all be overloaded
+    * overloaded operators are resetted to nil in InitSymtable
+      (bug when trying to compile a uint that overloads operators twice)
+
+  Revision 1.17  1999/09/21 20:53:23  florian
+    * fixed 1/s problem from mailing list
+
+  Revision 1.16  1999/09/17 09:17:49  peter
+    * removed uses globals
+
+  Revision 1.15  1999/09/16 13:41:37  peter
+    * better error checking
+
+  Revision 1.14  1999/09/08 16:02:04  peter
+    * tokendat compiles for tp
+    * tokens.dat supplied by default
+
+  Revision 1.13  1999/09/03 08:37:34  pierre
+    *  tokens.dat only used for TP, and also removed from
+       compiler dependencies
+
+  Revision 1.12  1999/09/02 18:47:49  daniel
+    * Could not compile with TP, some arrays moved to heap
+    * NOAG386BIN default for TP
+    * AG386* files were not compatible with TP, fixed.
+
+  Revision 1.11  1999/08/04 13:03:17  jonas
+    * all tokens now start with an underscore
+    * PowerPC compiles!!
+
+  Revision 1.10  1999/08/03 22:03:39  peter
+    * moved bitmask constants to sets
+    * some other type/const renamings
 
-  Revision 1.3  1999/09/17 09:16:13  peter
-    * updated with compiler versions
+  Revision 1.9  1999/07/22 09:38:01  florian
+    + resourcestring implemented
+    + start of longstring support
 
 }