* fcl-db: sql parser:

- support reading (and ignoring) SET AUTODDL statements generated by isql.
  This allows the parser to read isql-generated metadata extraction scripts from Firebird databases
- tests

git-svn-id: trunk@27921 -

packages/fcl-db/src/sql/README.txt

@@ -2,6 +2,8 @@ SQL scanner/parser/Abstract Syntax Tree units
 
 This can parse the complete Firebird dialect 3 SQL syntax (which should come pretty close to SQL-92) and builds a syntax tree from it. The Abstract Syntax Tree can re-create the SQL with limited formatting support.
 
+Additionally, the parser has limited support for SET commands used by the Firebird isql tool, enough to read DDL dumps from databases generated by isql.
+
 It comes with extensive test suite (over 830 test cases; see the fcl-db\tests directory). It has been tested on almost 400,000 SQL statements. Nevertheless bugs may remain, so any test results you may produce are welcome. Especially the GRANT/REVOKE statements are tested only theoretically.
 
 The scanner/parser have been designed so they should be able to cope with other SQL dialects (using a set of flags) such as MySQL, but this support is currently not implemented.

packages/fcl-db/src/sql/fpsqlparser.pas

@@ -3412,8 +3412,22 @@ begin
   // On Entry, we're on the SET statement
   Consume(tsqlSet);
   Case CurrentToken of
-    tsqlGenerator : Result:=ParseSetGeneratorStatement(AParent);
-    tsqlTerm : Result:=ParseSetTermStatement(AParent);
+    tsqlGenerator : Result:=ParseSetGeneratorStatement(AParent); //SET GENERATOR
+    tsqlTerm : Result:=ParseSetTermStatement(AParent); //SET TERM
+    tsqlAutoDDL : //SET AUTODDL: ignore these isql commands for now
+      begin
+      // SET AUTODDL ON, SET AUTODDL OFF; optional arguments
+      if (PeekNextToken in [tsqlOn, tsqlOff]) then
+        begin
+        GetNextToken;
+        Consume([tsqlOn,tsqlOff]);
+        end
+      else
+        begin
+        Consume(tsqlAutoDDL);
+        end;
+        Result:=nil; //ignore
+      end;
   else
     // For the time being
     UnexpectedToken;

packages/fcl-db/src/sql/fpsqlscanner.pp

@@ -50,7 +50,7 @@ type
    tsqlEQ,tsqlGE,tsqlLE,tsqlNE,
    { Reserved words/keywords start here. They must be last }
    { Note: if adding before tsqlALL or after tsqlWHEN please update FirstKeyword/LastKeyword }
-   tsqlALL, tsqlAND, tsqlANY, tsqlASC, tsqlASCENDING, tsqlAVG, tsqlALTER, tsqlAdd, tsqlActive, tsqlAction, tsqlAs,tsqlAt, tsqlAuto,tsqlAfter,tsqlAdmin,
+   tsqlALL, tsqlAND, tsqlANY, tsqlASC, tsqlASCENDING, tsqlAVG, tsqlALTER, tsqlAdd, tsqlActive, tsqlAction, tsqlAs,tsqlAt, tsqlAuto, tsqlAutoDDL {not an FB reserved word but used in isql scripts}, tsqlAfter,tsqlAdmin,
    tsqlBETWEEN, tsqlBinary, tsqlBY, tsqlBLOB, tsqlBegin, tsqlBefore,
    tsqlCOLLATE, tsqlCONTAINING, tsqlCOUNT, tsqlCREATE, tsqlCOLUMN, tsqlCONSTRAINT, tsqlChar,tsqlCHARACTER, tsqlCHECK, tsqlComputed,tsqlCASCADE, tsqlCast, tsqlCommit,tsqlConnect,tsqlCache,tsqlConditional,tsqlCString,
    tsqlDESC, tsqlDESCENDING, tsqlDISTINCT, tsqlDEFAULT, tsqlDELETE, tsqlDO, tsqlDouble, tsqlDECLARE, tsqlDROP, tsqlDomain, tsqlDecimal, tsqlDate,tsqlDatabase,
@@ -64,11 +64,11 @@ type
    tsqlLEFT, tsqlLIKE, tsqlLength,
    tsqlMAX, tsqlMIN, tsqlMERGE, tsqlManual, tsqlModuleName,
    tsqlNOT, tsqlNULL, tsqlNUMERIC , tsqlNChar, tsqlNATIONAL,tsqlNO, tsqlNatural,
-   tsqlON, tsqlOR, tsqlORDER, tsqlOUTER, tsqlOption,
+   tsqlOFF {not an FB reserved word; used in isql scripts}, tsqlON, tsqlOR, tsqlORDER, tsqlOUTER, tsqlOption,
    tsqlPrecision, tsqlPRIMARY,  tsqlProcedure, tsqlPosition, tsqlPlan, tsqlPassword, tsqlPage,tsqlPages,tsqlPageSize,tsqlPostEvent,tsqlPrivileges,tsqlPublic,
    tsqlRIGHT, tsqlROLE, tsqlReferences, tsqlRollBack, tsqlRelease,  tsqlretain,  tsqlReturningValues,tsqlReturns, tsqlrevoke,
    tsqlSELECT, tsqlSET, tsqlSINGULAR, tsqlSOME, tsqlSTARTING, tsqlSUM, tsqlSKIP,tsqlSUBTYPE,tsqlSize,tsqlSegment, tsqlSORT, tsqlSnapShot,tsqlSchema,tsqlShadow,tsqlSuspend,tsqlSQLCode,tsqlSmallint,
-   tSQLTABLE, tsqlText, tsqlTerm, tsqlTrigger, tsqlTime, tsqlTimeStamp, tsqlType, tsqlTo, tsqlTransaction, tsqlThen,
+   tSQLTABLE, tsqlText, tsqlTerm {not an FB reserved word, used in isql scripts}, tsqlTrigger, tsqlTime, tsqlTimeStamp, tsqlType, tsqlTo, tsqlTransaction, tsqlThen,
    tsqlUNION, tsqlUPDATE, tsqlUPPER,  tsqlUNIQUE, tsqlUSER,
    tsqlValue, tsqlVALUES, tsqlVARIABLE,  tsqlVIEW, tsqlVARCHAR,TSQLVARYING,
    tsqlWHERE, tsqlWITH, tsqlWHILE, tsqlWork, tsqlWhen
@@ -96,7 +96,7 @@ const
        '+','-','*','/','||',
        '=','>=','<=','<>',
        // Identifiers last:
-       'ALL', 'AND', 'ANY', 'ASC', 'ASCENDING', 'AVG', 'ALTER', 'ADD','ACTIVE','ACTION', 'AS', 'AT', 'AUTO', 'AFTER', 'ADMIN',
+       'ALL', 'AND', 'ANY', 'ASC', 'ASCENDING', 'AVG', 'ALTER', 'ADD','ACTIVE','ACTION', 'AS', 'AT', 'AUTO', 'AUTODDL', 'AFTER', 'ADMIN',
        'BETWEEN', 'BINARY', 'BY', 'BLOB','BEGIN', 'BEFORE',
        'COLLATE', 'CONTAINING', 'COUNT', 'CREATE', 'COLUMN', 'CONSTRAINT', 'CHAR','CHARACTER','CHECK', 'COMPUTED','CASCADE','CAST', 'COMMIT', 'CONNECT', 'CACHE','CONDITIONAL', 'CSTRING',
        'DESC', 'DESCENDING', 'DISTINCT',  'DEFAULT', 'DELETE', 'DO', 'DOUBLE', 'DECLARE', 'DROP', 'DOMAIN', 'DECIMAL', 'DATE','DATABASE',
@@ -110,7 +110,7 @@ const
        'LEFT', 'LIKE', 'LENGTH',
        'MAX', 'MIN', 'MERGE', 'MANUAL', 'MODULE_NAME',
        'NOT', 'NULL', 'NUMERIC','NCHAR','NATIONAL', 'NO', 'NATURAL',
-       'ON', 'OR', 'ORDER', 'OUTER', 'OPTION',
+       'OFF', 'ON', 'OR', 'ORDER', 'OUTER', 'OPTION',
        'PRECISION', 'PRIMARY', 'PROCEDURE','POSITION','PLAN', 'PASSWORD','PAGE','PAGES','PAGE_SIZE','POST_EVENT','PRIVILEGES','PUBLIC',
        'RIGHT', 'ROLE', 'REFERENCES', 'ROLLBACK','RELEASE', 'RETAIN', 'RETURNING_VALUES', 'RETURNS','REVOKE',
        'SELECT', 'SET', 'SINGULAR', 'SOME', 'STARTING', 'SUM', 'SKIP','SUB_TYPE', 'SIZE', 'SEGMENT', 'SORT', 'SNAPSHOT','SCHEMA','SHADOW','SUSPEND','SQLCODE','SMALLINT',

packages/fcl-db/tests/tcgensql.pas

@@ -2014,14 +2014,14 @@ begin
   AssertSQL(S,'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('A');
-  P.ParamType:=CreatetypeDefinition(sdtInteger,0);
+  P.ParamType:=CreateTypeDefinition(sdtInteger,0);
   FToFree:=S;
   S.LocalVariables.Add(P);
   AssertSQL(S,'DECLARE VARIABLE A INT;'+sLineBreak+'BEGIN'+sLineBreak+'EXIT;'+sLineBreak+'END');
   AssertSQL(S,'DECLARE VARIABLE A INT;'+sLineBreak+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('B');
-  P.ParamType:=CreatetypeDefinition(sdtChar,5);
+  P.ParamType:=CreateTypeDefinition(sdtChar,5);
   FToFree:=S;
   S.LocalVariables.Add(P);
   AssertSQL(S,'DECLARE VARIABLE A INT;'+sLineBreak+'DECLARE VARIABLE B CHAR(5);'+sLineBreak+'BEGIN'+sLineBreak+'EXIT;'+sLineBreak+'END');
@@ -2046,7 +2046,7 @@ begin
   AssertSQL(S,H+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('I');
-  P.ParamType:=CreatetypeDefinition(sdtInteger,0);
+  P.ParamType:=CreateTypeDefinition(sdtInteger,0);
   FToFree:=S;
   S.InputVariables.Add(P);
   H:=PHEAD+' (I INT)'+sLineBreak+'AS'+sLineBreak;
@@ -2054,7 +2054,7 @@ begin
   AssertSQL(S,H+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('J');
-  P.ParamType:=CreatetypeDefinition(sdtChar,5);
+  P.ParamType:=CreateTypeDefinition(sdtChar,5);
   FToFree:=S;
   S.InputVariables.Add(P);
   H:=PHEAD+' (I INT , J CHAR(5))'+sLineBreak+'AS'+sLineBreak;
@@ -2062,7 +2062,7 @@ begin
   AssertSQL(S,H+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('R');
-  P.ParamType:=CreatetypeDefinition(sdtInteger,0);
+  P.ParamType:=CreateTypeDefinition(sdtInteger,0);
   FToFree:=S;
   S.OutputVariables.Add(P);
   H:=PHEAD+' (I INT , J CHAR(5))'+sLineBreak+'RETURNS (R INT)'+sLineBreak+'AS'+sLineBreak;
@@ -2070,7 +2070,7 @@ begin
   AssertSQL(S,H+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('S');
-  P.ParamType:=CreatetypeDefinition(sdtChar,5);
+  P.ParamType:=CreateTypeDefinition(sdtChar,5);
   FToFree:=S;
   S.OutputVariables.Add(P);
   H:=PHEAD+' (I INT , J CHAR(5))'+sLineBreak+'RETURNS (R INT , S CHAR(5))'+sLineBreak+'AS'+sLineBreak;
@@ -2078,14 +2078,14 @@ begin
   AssertSQL(S,H+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('A');
-  P.ParamType:=CreatetypeDefinition(sdtInteger,0);
+  P.ParamType:=CreateTypeDefinition(sdtInteger,0);
   FToFree:=S;
   S.LocalVariables.Add(P);
   AssertSQL(S,H+'DECLARE VARIABLE A INT;'+sLineBreak+'BEGIN'+sLineBreak+'EXIT;'+sLineBreak+'END');
   AssertSQL(S,H+'DECLARE VARIABLE A INT;'+sLineBreak+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('B');
-  P.ParamType:=CreatetypeDefinition(sdtChar,5);
+  P.ParamType:=CreateTypeDefinition(sdtChar,5);
   FToFree:=S;
   S.LocalVariables.Add(P);
   AssertSQL(S,H+'DECLARE VARIABLE A INT;'+sLineBreak+'DECLARE VARIABLE B CHAR(5);'+sLineBreak+'BEGIN'+sLineBreak+'EXIT;'+sLineBreak+'END');
@@ -2141,14 +2141,14 @@ begin
   AssertSQL(S,H+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('A');
-  P.ParamType:=CreatetypeDefinition(sdtInteger,0);
+  P.ParamType:=CreateTypeDefinition(sdtInteger,0);
   FToFree:=S;
   S.LocalVariables.Add(P);
   AssertSQL(S,H+'DECLARE VARIABLE A INT;'+sLineBreak+'BEGIN'+sLineBreak+'EXIT;'+sLineBreak+'END');
   AssertSQL(S,H+'DECLARE VARIABLE A INT;'+sLineBreak+'BEGIN'+sLineBreak+'  EXIT;'+sLineBreak+'END',[sfoIndentProcedureBlock]);
   P:=TSQLProcedureParamDef.Create(Nil);
   P.ParamName:=CreateIdentifier('B');
-  P.ParamType:=CreatetypeDefinition(sdtChar,5);
+  P.ParamType:=CreateTypeDefinition(sdtChar,5);
   FToFree:=S;
   S.LocalVariables.Add(P);
   AssertSQL(S,H+'DECLARE VARIABLE A INT;'+sLineBreak+'DECLARE VARIABLE B CHAR(5);'+sLineBreak+'BEGIN'+sLineBreak+'EXIT;'+sLineBreak+'END');

packages/fcl-db/tests/tcparser.pas

@@ -844,10 +844,14 @@ type
     procedure Test2RolesToUser;
   end;
 
-  { TTestTermParser }
+  { TTestSetParser }
 
-  TTestTermParser = Class(TTestSQLParser)
+  TTestSetParser = Class(TTestSQLParser)
   published
+    procedure TestSetAutoDDL;
+    procedure TestSetAutoDDLOn;
+    procedure TestSetAutoDDLOff;
+    procedure TestSetAutoDDLCreateProcedure;
     procedure TestSetTerm;
     procedure TestSetTermSemicolon;
     procedure TestSetTermCreateProcedure;
@@ -865,9 +869,43 @@ implementation
 
 uses typinfo;
 
-{ TTestTermParser }
+{ TTestSetParser }
 
-procedure TTestTermParser.TestSetTerm;
+procedure TTestSetParser.TestSetAutoDDL;
+begin
+  CreateParser('SET AUTODDL;');
+  AssertNull('SET AUTODDL should be ignored and give nil result',Parser.Parse);
+end;
+
+procedure TTestSetParser.TestSetAutoDDLOn;
+begin
+  CreateParser('SET AUTODDL ON;');
+  AssertNull('SET AUTODDL should be ignored and give nil result',Parser.Parse);
+end;
+
+procedure TTestSetParser.TestSetAutoDDLOff;
+begin
+  CreateParser('SET AUTODDL OFF;');
+  AssertNull('SET AUTODDL should be ignored and give nil result',Parser.Parse);
+end;
+
+procedure TTestSetParser.TestSetAutoDDLCreateProcedure;
+Const
+  SQL =
+   'SET AUTODDL ;'+LineEnding+
+   ''+LineEnding+
+   'CREATE PROCEDURE PROCNAME'+LineEnding+
+   'AS'+LineEnding+
+   'BEGIN'+LineEnding+
+   '  /* Empty procedure */'+LineEnding+
+   'END;';
+begin
+  CreateParser(SQL);
+  Parser.ParseScript;
+  //todo: test name etc of procedure
+end;
+
+procedure TTestSetParser.TestSetTerm;
 Var
   S : TSQLSetTermStatement;
 
@@ -881,7 +919,7 @@ begin
   AssertEquals('End of stream reached',tsqlEOF,Parser.CurrentToken);
 end;
 
-procedure TTestTermParser.TestSetTermSemicolon;
+procedure TTestSetParser.TestSetTermSemicolon;
 Var
   S : TSQLSetTermStatement;
 
@@ -898,48 +936,50 @@ begin
   AssertEquals('End of stream reached',tsqlEOF,Parser.CurrentToken);
 end;
 
-procedure TTestTermParser.TestSetTermCreateProcedure;
+procedure TTestSetParser.TestSetTermCreateProcedure;
 Const
   SQL =
-   'SET TERM ^ ;'+#13+#10+
-   ''+#13+#10+
-   'CREATE PROCEDURE PROCNAME'+#13+#10+
-   'AS'+#13+#10+
-   'BEGIN'+#13+#10+
-   '  /* Empty procedure */'+#13+#10+
-   'END^'+#13+#10+
-   ''+#13+#10+
+   'SET TERM ^ ;'+LineEnding+
+   ''+LineEnding+
+   'CREATE PROCEDURE PROCNAME'+LineEnding+
+   'AS'+LineEnding+
+   'BEGIN'+LineEnding+
+   '  /* Empty procedure */'+LineEnding+
+   'END^'+LineEnding+
+   ''+LineEnding+
    'SET TERM ; ^';
 
 begin
   CreateParser(SQL);
   Parser.ParseScript;
+  //todo: test name etc of procedure
 end;
 
-procedure TTestTermParser.TestSetTermCreateProcedureVar;
+procedure TTestSetParser.TestSetTermCreateProcedureVar;
 // Procedure with variable
 Const
   SQL =
-    'SET TERM ^ ;'+#13+#10+
-    'CREATE PROCEDURE PROCWITHVAR'+#13+#10+
-    'RETURNS (LANGUAGES VARCHAR(15) CHARACTER SET NONE)'+#13+#10+
-    'AS'+#13+#10+
-    'DECLARE VARIABLE i INTEGER;'+#13+#10+
-    'BEGIN'+#13+#10+
-    '  i = 1;'+#13+#10+
-    '  WHILE (i <= 5) DO'+#13+#10+
-    '  BEGIN'+#13+#10+
-    '    SELECT language_req[:i] FROM job'+#13+#10+
-    '    INTO :languages;'+#13+#10+
-    '    i = i +1;'+#13+#10+
-    '    SUSPEND;'+#13+#10+
-    '  END'+#13+#10+
-    'END ^'+#13+#10+
+    'SET TERM ^ ;'+LineEnding+
+    'CREATE PROCEDURE PROCWITHVAR'+LineEnding+
+    'RETURNS (LANGUAGES VARCHAR(15) CHARACTER SET NONE)'+LineEnding+
+    'AS'+LineEnding+
+    'DECLARE VARIABLE i INTEGER;'+LineEnding+
+    'BEGIN'+LineEnding+
+    '  i = 1;'+LineEnding+
+    '  WHILE (i <= 5) DO'+LineEnding+
+    '  BEGIN'+LineEnding+
+    '    SELECT language_req[:i] FROM job'+LineEnding+
+    '    INTO :languages;'+LineEnding+
+    '    i = i +1;'+LineEnding+
+    '    SUSPEND;'+LineEnding+
+    '  END'+LineEnding+
+    'END ^'+LineEnding+
     'SET TERM ; ^';
 
 begin
   CreateParser(SQL);
   Parser.ParseScript;
+  //todo: test name etc of procedure
 end;
 
 
@@ -8259,7 +8299,7 @@ initialization
                  TTestDeclareExternalFunctionParser,
                  TTestGrantParser,
                  TTestRevokeParser,
-                 TTestTermParser,
+                 TTestSetParser,
                  TTestGlobalParser]);
 end.