stb_c_lexer: allow including stb_c_lexer.h without defining overrides (all tokens are always defined; token values have changed)

stb_c_lexer.h

@@ -42,6 +42,7 @@
 //
 //   See end of file for license information.
 
+#ifdef STB_C_LEXER_IMPLEMENTATION
 #ifndef STB_C_LEXER_DEFINITIONS
 // to change the default parsing rules, copy the following lines
 // into your C/C++ file *before* including this, and then replace
@@ -54,7 +55,6 @@
 #error "Can only use stb_c_lexer in contexts where the preprocessor symbols 'Y' and 'N' are not defined"
 #endif
 
-
 #define STB_C_LEX_C_DECIMAL_INTS    Y   //  "0|[1-9][0-9]*"                        CLEX_intlit
 #define STB_C_LEX_C_HEX_INTS        Y   //  "0x[0-9a-fA-F]+"                       CLEX_intlit
 #define STB_C_LEX_C_OCTAL_INTS      Y   //  "[0-7]+"                               CLEX_intlit
@@ -102,7 +102,7 @@
 
 #define STB_C_LEXER_DEFINITIONS         // This line prevents the header file from replacing your definitions
 // --END--
-
+#endif
 #endif
 
 #ifndef INCLUDE_STB_C_LEXER_H
@@ -172,78 +172,41 @@ extern void stb_c_lexer_get_location(const stb_lexer *lexer, const char *where,
 }
 #endif
 
-// Hacky definitions so we can easily #if on them
-#define Y(x) 1
-#define N(x) 0
-
-// Config variable that influence which lexer tokens get declared need to go here
-#if STB_C_LEX_C_DECIMAL_INTS(x) || STB_C_LEX_C_HEX_INTS(x) || STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x)
-#define STB__clex_define_int
-#endif
-
-#if (STB_C_LEX_C_ARITHEQ(x) && STB_C_LEX_C_SHIFTS(x)) || STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x)
-#define STB__clex_define_shifts
-#endif
-
-// Now pick a definition of Y/N that's conducive to
-// defining the enum of token names.
-#if STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x) || defined(STB_C_LEXER_SELF_TEST)
-  #undef  N
-  #define N(a) Y(a)
-#else
-  #undef  N
-  #define N(a)
-#endif
-
-#undef  Y
-#define Y(a) a,
-
 enum
 {
    CLEX_eof = 256,
    CLEX_parse_error,
-
-#ifdef STB__clex_define_int
-   CLEX_intlit,
-#endif
-
-   STB_C_LEX_C_DECIMAL_FLOATS( CLEX_floatlit    )
-   STB_C_LEX_C_IDENTIFIERS(  CLEX_id            )
-   STB_C_LEX_C_DQ_STRINGS(   CLEX_dqstring      )
-   STB_C_LEX_C_SQ_STRINGS(   CLEX_sqstring      )
-   STB_C_LEX_C_CHARS(        CLEX_charlit       )
-   STB_C_LEX_C_COMPARISONS(  CLEX_eq            )
-   STB_C_LEX_C_COMPARISONS(  CLEX_noteq         )
-   STB_C_LEX_C_COMPARISONS(  CLEX_lesseq        )
-   STB_C_LEX_C_COMPARISONS(  CLEX_greatereq     )
-   STB_C_LEX_C_LOGICAL(      CLEX_andand        )
-   STB_C_LEX_C_LOGICAL(      CLEX_oror          )
-   STB_C_LEX_C_SHIFTS(       CLEX_shl           )
-   STB_C_LEX_C_SHIFTS(       CLEX_shr           )
-   STB_C_LEX_C_INCREMENTS(   CLEX_plusplus      )
-   STB_C_LEX_C_INCREMENTS(   CLEX_minusminus    )
-   STB_C_LEX_C_ARITHEQ(      CLEX_pluseq        )
-   STB_C_LEX_C_ARITHEQ(      CLEX_minuseq       )
-   STB_C_LEX_C_ARITHEQ(      CLEX_muleq         )
-   STB_C_LEX_C_ARITHEQ(      CLEX_diveq         )
-   STB_C_LEX_C_ARITHEQ(      CLEX_modeq         )
-   STB_C_LEX_C_BITWISEEQ(    CLEX_andeq         )
-   STB_C_LEX_C_BITWISEEQ(    CLEX_oreq          )
-   STB_C_LEX_C_BITWISEEQ(    CLEX_xoreq         )
-   STB_C_LEX_C_ARROW(        CLEX_arrow         )
-   STB_C_LEX_EQUAL_ARROW(    CLEX_eqarrow       )
-
-#ifdef STB__clex_define_shifts
+   CLEX_intlit        ,
+   CLEX_floatlit      ,
+   CLEX_id            ,
+   CLEX_dqstring      ,
+   CLEX_sqstring      ,
+   CLEX_charlit       ,
+   CLEX_eq            ,
+   CLEX_noteq         ,
+   CLEX_lesseq        ,
+   CLEX_greatereq     ,
+   CLEX_andand        ,
+   CLEX_oror          ,
+   CLEX_shl           ,
+   CLEX_shr           ,
+   CLEX_plusplus      ,
+   CLEX_minusminus    ,
+   CLEX_pluseq        ,
+   CLEX_minuseq       ,
+   CLEX_muleq         ,
+   CLEX_diveq         ,
+   CLEX_modeq         ,
+   CLEX_andeq         ,
+   CLEX_oreq          ,
+   CLEX_xoreq         ,
+   CLEX_arrow         ,
+   CLEX_eqarrow       ,
    CLEX_shleq, CLEX_shreq,
-#endif
 
    CLEX_first_unused_token
 
 };
-
-#undef Y
-#undef N
-
 #endif // INCLUDE_STB_C_LEXER_H
 
 #ifdef STB_C_LEXER_IMPLEMENTATION

tests/test_c_lexer.c

@@ -1 +1,50 @@
 #include "stb_c_lexer.h" 
+
+#define STB_C_LEX_C_DECIMAL_INTS    Y   //  "0|[1-9][0-9]*"                        CLEX_intlit
+#define STB_C_LEX_C_HEX_INTS        Y   //  "0x[0-9a-fA-F]+"                       CLEX_intlit
+#define STB_C_LEX_C_OCTAL_INTS      Y   //  "[0-7]+"                               CLEX_intlit
+#define STB_C_LEX_C_DECIMAL_FLOATS  Y   //  "[0-9]*(.[0-9]*([eE][-+]?[0-9]+)?)     CLEX_floatlit
+#define STB_C_LEX_C99_HEX_FLOATS    N   //  "0x{hex}+(.{hex}*)?[pP][-+]?{hex}+     CLEX_floatlit
+#define STB_C_LEX_C_IDENTIFIERS     Y   //  "[_a-zA-Z][_a-zA-Z0-9]*"               CLEX_id
+#define STB_C_LEX_C_DQ_STRINGS      Y   //  double-quote-delimited strings with escapes  CLEX_dqstring
+#define STB_C_LEX_C_SQ_STRINGS      N   //  single-quote-delimited strings with escapes  CLEX_ssstring
+#define STB_C_LEX_C_CHARS           Y   //  single-quote-delimited character with escape CLEX_charlits
+#define STB_C_LEX_C_COMMENTS        Y   //  "/* comment */"
+#define STB_C_LEX_CPP_COMMENTS      Y   //  "// comment to end of line\n"
+#define STB_C_LEX_C_COMPARISONS     Y   //  "==" CLEX_eq  "!=" CLEX_noteq   "<=" CLEX_lesseq  ">=" CLEX_greatereq
+#define STB_C_LEX_C_LOGICAL         Y   //  "&&"  CLEX_andand   "||"  CLEX_oror
+#define STB_C_LEX_C_SHIFTS          Y   //  "<<"  CLEX_shl      ">>"  CLEX_shr
+#define STB_C_LEX_C_INCREMENTS      Y   //  "++"  CLEX_plusplus "--"  CLEX_minusminus
+#define STB_C_LEX_C_ARROW           Y   //  "->"  CLEX_arrow
+#define STB_C_LEX_EQUAL_ARROW       N   //  "=>"  CLEX_eqarrow
+#define STB_C_LEX_C_BITWISEEQ       Y   //  "&="  CLEX_andeq    "|="  CLEX_oreq     "^="  CLEX_xoreq
+#define STB_C_LEX_C_ARITHEQ         Y   //  "+="  CLEX_pluseq   "-="  CLEX_minuseq
+                                        //  "*="  CLEX_muleq    "/="  CLEX_diveq    "%=" CLEX_modeq
+                                        //  if both STB_C_LEX_SHIFTS & STB_C_LEX_ARITHEQ:
+                                        //                      "<<=" CLEX_shleq    ">>=" CLEX_shreq
+
+#define STB_C_LEX_PARSE_SUFFIXES    N   // letters after numbers are parsed as part of those numbers, and must be in suffix list below
+#define STB_C_LEX_DECIMAL_SUFFIXES  ""  // decimal integer suffixes e.g. "uUlL" -- these are returned as-is in string storage
+#define STB_C_LEX_HEX_SUFFIXES      ""  // e.g. "uUlL"
+#define STB_C_LEX_OCTAL_SUFFIXES    ""  // e.g. "uUlL"
+#define STB_C_LEX_FLOAT_SUFFIXES    ""  //
+
+#define STB_C_LEX_0_IS_EOF             Y  // if Y, ends parsing at '\0'; if N, returns '\0' as token
+#define STB_C_LEX_INTEGERS_AS_DOUBLES  N  // parses integers as doubles so they can be larger than 'int', but only if STB_C_LEX_STDLIB==N
+#define STB_C_LEX_MULTILINE_DSTRINGS   N  // allow newlines in double-quoted strings
+#define STB_C_LEX_MULTILINE_SSTRINGS   N  // allow newlines in single-quoted strings
+#define STB_C_LEX_USE_STDLIB           Y  // use strtod,strtol for parsing #s; otherwise inaccurate hack
+#define STB_C_LEX_DOLLAR_IDENTIFIER    Y  // allow $ as an identifier character
+#define STB_C_LEX_FLOAT_NO_DECIMAL     Y  // allow floats that have no decimal point if they have an exponent
+
+#define STB_C_LEX_DEFINE_ALL_TOKEN_NAMES  N   // if Y, all CLEX_ token names are defined, even if never returned
+                                              // leaving it as N should help you catch config bugs
+
+#define STB_C_LEX_DISCARD_PREPROCESSOR    Y   // discard C-preprocessor directives (e.g. after prepocess
+                                              // still have #line, #pragma, etc)
+
+//#define STB_C_LEX_ISWHITE(str)    ... // return length in bytes of whitespace characters if first char is whitespace
+
+#define STB_C_LEXER_DEFINITIONS         // This line prevents the header file from replacing your definitions
+
+#include "stb_c_lexer.h" 

tests/test_cpp_compilation.cpp

@@ -183,53 +183,4 @@ void dummy3(void)
   stb_textedit_paste(0,0,0,0);
 }
 
-#if 0
-#define STB_C_LEX_C_DECIMAL_INTS    Y   //  "0|[1-9][0-9]*"                        CLEX_intlit
-#define STB_C_LEX_C_HEX_INTS        Y   //  "0x[0-9a-fA-F]+"                       CLEX_intlit
-#define STB_C_LEX_C_OCTAL_INTS      Y   //  "[0-7]+"                               CLEX_intlit
-#define STB_C_LEX_C_DECIMAL_FLOATS  Y   //  "[0-9]*(.[0-9]*([eE][-+]?[0-9]+)?)     CLEX_floatlit
-#define STB_C_LEX_C99_HEX_FLOATS    N   //  "0x{hex}+(.{hex}*)?[pP][-+]?{hex}+     CLEX_floatlit
-#define STB_C_LEX_C_IDENTIFIERS     Y   //  "[_a-zA-Z][_a-zA-Z0-9]*"               CLEX_id
-#define STB_C_LEX_C_DQ_STRINGS      Y   //  double-quote-delimited strings with escapes  CLEX_dqstring
-#define STB_C_LEX_C_SQ_STRINGS      N   //  single-quote-delimited strings with escapes  CLEX_ssstring
-#define STB_C_LEX_C_CHARS           Y   //  single-quote-delimited character with escape CLEX_charlits
-#define STB_C_LEX_C_COMMENTS        Y   //  "/* comment */"
-#define STB_C_LEX_CPP_COMMENTS      Y   //  "// comment to end of line\n"
-#define STB_C_LEX_C_COMPARISONS     Y   //  "==" CLEX_eq  "!=" CLEX_noteq   "<=" CLEX_lesseq  ">=" CLEX_greatereq
-#define STB_C_LEX_C_LOGICAL         Y   //  "&&"  CLEX_andand   "||"  CLEX_oror
-#define STB_C_LEX_C_SHIFTS          Y   //  "<<"  CLEX_shl      ">>"  CLEX_shr
-#define STB_C_LEX_C_INCREMENTS      Y   //  "++"  CLEX_plusplus "--"  CLEX_minusminus
-#define STB_C_LEX_C_ARROW           Y   //  "->"  CLEX_arrow
-#define STB_C_LEX_EQUAL_ARROW       N   //  "=>"  CLEX_eqarrow
-#define STB_C_LEX_C_BITWISEEQ       Y   //  "&="  CLEX_andeq    "|="  CLEX_oreq     "^="  CLEX_xoreq
-#define STB_C_LEX_C_ARITHEQ         Y   //  "+="  CLEX_pluseq   "-="  CLEX_minuseq
-                                        //  "*="  CLEX_muleq    "/="  CLEX_diveq    "%=" CLEX_modeq
-                                        //  if both STB_C_LEX_SHIFTS & STB_C_LEX_ARITHEQ:
-                                        //                      "<<=" CLEX_shleq    ">>=" CLEX_shreq
-
-#define STB_C_LEX_PARSE_SUFFIXES    N   // letters after numbers are parsed as part of those numbers, and must be in suffix list below
-#define STB_C_LEX_DECIMAL_SUFFIXES  ""  // decimal integer suffixes e.g. "uUlL" -- these are returned as-is in string storage
-#define STB_C_LEX_HEX_SUFFIXES      ""  // e.g. "uUlL"
-#define STB_C_LEX_OCTAL_SUFFIXES    ""  // e.g. "uUlL"
-#define STB_C_LEX_FLOAT_SUFFIXES    ""  //
-
-#define STB_C_LEX_0_IS_EOF             Y  // if Y, ends parsing at '\0'; if N, returns '\0' as token
-#define STB_C_LEX_INTEGERS_AS_DOUBLES  N  // parses integers as doubles so they can be larger than 'int', but only if STB_C_LEX_STDLIB==N
-#define STB_C_LEX_MULTILINE_DSTRINGS   N  // allow newlines in double-quoted strings
-#define STB_C_LEX_MULTILINE_SSTRINGS   N  // allow newlines in single-quoted strings
-#define STB_C_LEX_USE_STDLIB           Y  // use strtod,strtol for parsing #s; otherwise inaccurate hack
-#define STB_C_LEX_DOLLAR_IDENTIFIER    Y  // allow $ as an identifier character
-#define STB_C_LEX_FLOAT_NO_DECIMAL     Y  // allow floats that have no decimal point if they have an exponent
-
-#define STB_C_LEX_DEFINE_ALL_TOKEN_NAMES  N   // if Y, all CLEX_ token names are defined, even if never returned
-                                              // leaving it as N should help you catch config bugs
-
-#define STB_C_LEX_DISCARD_PREPROCESSOR    Y   // discard C-preprocessor directives (e.g. after prepocess
-                                              // still have #line, #pragma, etc)
-
-//#define STB_C_LEX_ISWHITE(str)    ... // return length in bytes of whitespace characters if first char is whitespace
-
-#define STB_C_LEXER_DEFINITIONS         // This line prevents the header file from replacing your definitions
-#endif
-
 #include "stb_c_lexer.h"