Feature SET AUTOTERM in ISQL. (#7868)

builds/win32/msvc15/isql_static.vcxproj

@@ -23,6 +23,7 @@
     <ClCompile Include="..\..\..\src\isql\Extender.cpp" />
     <ClCompile Include="..\..\..\gen\isql\extract.cpp" />
     <ClCompile Include="..\..\..\src\common\fb_exception.cpp" />
+    <ClCompile Include="..\..\..\src\isql\FrontendLexer.cpp" />
     <ClCompile Include="..\..\..\src\isql\InputDevices.cpp" />
     <ClCompile Include="..\..\..\gen\isql\isql.cpp" />
     <ClCompile Include="..\..\..\src\isql\iutils.cpp" />
@@ -38,6 +39,7 @@
     <ClInclude Include="..\..\..\src\isql\ColList.h" />
     <ClInclude Include="..\..\..\src\isql\Extender.h" />
     <ClInclude Include="..\..\..\src\isql\extra_proto.h" />
+    <ClInclude Include="..\..\..\src\isql\FrontendLexer.h" />
     <ClInclude Include="..\..\..\src\isql\InputDevices.h" />
     <ClInclude Include="..\..\..\src\isql\isql.h" />
     <ClInclude Include="..\..\..\src\isql\isql_proto.h" />

builds/win32/msvc15/isql_static.vcxproj.filters

@@ -27,6 +27,9 @@
     <ClCompile Include="..\..\..\src\common\fb_exception.cpp">
       <Filter>ISQL files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\src\isql\FrontendLexer.cpp">
+      <Filter>ISQL files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\src\isql\InputDevices.cpp">
       <Filter>ISQL files</Filter>
     </ClCompile>
@@ -67,6 +70,9 @@
     <ClInclude Include="..\..\..\src\isql\extra_proto.h">
       <Filter>Header files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\src\isql\FrontendLexer.h">
+      <Filter>Header files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\src\isql\InputDevices.h">
       <Filter>Header files</Filter>
     </ClInclude>

builds/win32/msvc15/isql_test.vcxproj

@@ -176,6 +176,7 @@
     </ResourceCompile>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\src\isql\tests\FrontendLexerTest.cpp" />
     <ClCompile Include="..\..\..\src\isql\tests\ISqlTest.cpp" />
   </ItemGroup>
   <ItemGroup>

builds/win32/msvc15/isql_test.vcxproj.filters

@@ -7,6 +7,9 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\..\src\isql\tests\FrontendLexerTest.cpp">
+      <Filter>source</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\src\isql\tests\ISqlTest.cpp">
       <Filter>source</Filter>
     </ClCompile>

doc/README.isql_enhancements.txt

@@ -330,7 +330,7 @@ SQL> SET PER_TAB OFF;
 Isql enhancements in Firebird v6.
 ---------------------------------
 
-EXPLAIN statement.
+12) EXPLAIN statement.
 
 Author: Adriano dos Santos Fernandes
 
@@ -353,3 +353,69 @@ CON>     select id from employees where id = ? into id;
 CON> end!
 SQL>
 SQL> set term ;!
+
+
+13) SET AUTOTERM ON/OFF
+
+Author: Adriano dos Santos Fernandes
+
+When set to ON, terminator defined with SET TERM is changed to semicolon and a new logic
+for TERM detection is used, where engine helps ISQL to detect valid usage of semicolons
+inside statements.
+
+At each semicolon (outside quotes or comments), ISQL prepares the query buffer with
+engine using flag IStatement::PREPARE_REQUIRE_SEMICOLON.
+
+If engine prepares the statement correctly, it's run and ISQL is put in new statement
+mode.
+
+If engine returns error isc_command_end_err2, then ISQL is put in statement
+continuation mode and asks for another line, repeating the process.
+
+If engine returns a different error, the error is shown and ISQL is put in new statement
+mode.
+
+Notes:
+- This option can also be activated with command line parameter -autot(erm)
+- It can only be used with Firebird engine/server v6 or later
+- SET TERM command automatically sets AUTOTERM to OFF
+- SET AUTOTERM ON command automatically sets TERM to semicolon
+- While AUTOTERM ON can be used in non-interactive scripts, at each semicolon,
+  statement may be tried to be compiled using the server/engine.
+  That may be slow for big scripts with PSQL statements spanning many lines.
+
+Examples:
+
+SQL> SET AUTOTERM ON;
+
+SQL> execute block returns (o1 integer)
+CON> as
+CON> begin
+CON>     o1 = 1;
+CON>     suspend;
+CON> end;
+
+          O1
+============
+           1
+
+SQL> select 1 from rdb$database;
+
+    CONSTANT
+============
+           1
+
+SQL> select 1
+CON>     from rdb$database;
+
+    CONSTANT
+============
+           1
+
+SQL> select 1
+CON>     from rdb$database
+CON>     where true;
+
+    CONSTANT
+============
+           1

src/dsql/Parser.cpp

@@ -41,12 +41,14 @@ using namespace Jrd;
 
 
 Parser::Parser(thread_db* tdbb, MemoryPool& pool, MemoryPool* aStatementPool, DsqlCompilerScratch* aScratch,
-			USHORT aClientDialect, USHORT aDbDialect, const TEXT* string, size_t length, SSHORT charSetId)
+			USHORT aClientDialect, USHORT aDbDialect, bool aRequireSemicolon,
+			const TEXT* string, size_t length, SSHORT charSetId)
 	: PermanentStorage(pool),
 	  statementPool(aStatementPool),
 	  scratch(aScratch),
 	  client_dialect(aClientDialect),
 	  db_dialect(aDbDialect),
+	  requireSemicolon(aRequireSemicolon),
 	  transformedString(pool),
 	  strMarks(pool),
 	  stmt_ambiguous(false)

src/dsql/Parser.h

@@ -132,7 +132,8 @@ public:
 
 public:
 	Parser(thread_db* tdbb, MemoryPool& pool, MemoryPool* aStatementPool, DsqlCompilerScratch* aScratch,
-		USHORT aClientDialect, USHORT aDbDialect, const TEXT* string, size_t length, SSHORT charSetId);
+		USHORT aClientDialect, USHORT aDbDialect, bool aRequireSemicolon,
+		const TEXT* string, size_t length, SSHORT charSetId);
 	~Parser();
 
 public:
@@ -363,6 +364,7 @@ private:
 	DsqlCompilerScratch* scratch;
 	USHORT client_dialect;
 	USHORT db_dialect;
+	const bool requireSemicolon;
 	USHORT parser_version;
 	CharSet* charSet;
 

src/dsql/dsql.cpp

@@ -84,9 +84,9 @@ using namespace Firebird;
 
 static ULONG	get_request_info(thread_db*, DsqlRequest*, ULONG, UCHAR*);
 static dsql_dbb*	init(Jrd::thread_db*, Jrd::Attachment*);
-static DsqlRequest* prepareRequest(thread_db*, dsql_dbb*, jrd_tra*, ULONG, const TEXT*, USHORT, bool);
+static DsqlRequest* prepareRequest(thread_db*, dsql_dbb*, jrd_tra*, ULONG, const TEXT*, USHORT, unsigned, bool);
 static RefPtr<DsqlStatement> prepareStatement(thread_db*, dsql_dbb*, jrd_tra*, ULONG, const TEXT*, USHORT,
-	bool, ntrace_result_t* traceResult);
+	unsigned, bool, ntrace_result_t* traceResult);
 static UCHAR*	put_item(UCHAR, const USHORT, const UCHAR*, UCHAR*, const UCHAR* const);
 static void		sql_info(thread_db*, DsqlRequest*, ULONG, const UCHAR*, ULONG, UCHAR*);
 static UCHAR*	var_info(const dsql_msg*, const UCHAR*, const UCHAR* const, UCHAR*,
@@ -261,7 +261,7 @@ DsqlRequest* DSQL_prepare(thread_db* tdbb,
 		// Allocate a new request block and then prepare the request.
 
 		dsqlRequest = prepareRequest(tdbb, database, transaction, length, string, dialect,
-			isInternalRequest);
+			prepareFlags, isInternalRequest);
 
 		// Can not prepare a CREATE DATABASE/SCHEMA statement
 
@@ -336,7 +336,7 @@ void DSQL_execute_immediate(thread_db* tdbb, Jrd::Attachment* attachment, jrd_tr
 	try
 	{
 		dsqlRequest = prepareRequest(tdbb, database, *tra_handle, length, string, dialect,
-			isInternalRequest);
+			0, isInternalRequest);
 
 		const auto dsqlStatement = dsqlRequest->getDsqlStatement();
 
@@ -443,7 +443,7 @@ static dsql_dbb* init(thread_db* tdbb, Jrd::Attachment* attachment)
 // Prepare a request for execution.
 // Note: caller is responsible for pool handling.
 static DsqlRequest* prepareRequest(thread_db* tdbb, dsql_dbb* database, jrd_tra* transaction,
-	ULONG textLength, const TEXT* text, USHORT clientDialect, bool isInternalRequest)
+	ULONG textLength, const TEXT* text, USHORT clientDialect, unsigned prepareFlags, bool isInternalRequest)
 {
 	TraceDSQLPrepare trace(database->dbb_attachment, transaction, textLength, text, isInternalRequest);
 
@@ -451,7 +451,7 @@ static DsqlRequest* prepareRequest(thread_db* tdbb, dsql_dbb* database, jrd_tra*
 	try
 	{
 		auto statement = prepareStatement(tdbb, database, transaction, textLength, text,
-			clientDialect, isInternalRequest, &traceResult);
+			clientDialect, prepareFlags, isInternalRequest, &traceResult);
 
 		auto dsqlRequest = statement->createRequest(tdbb, database);
 
@@ -472,7 +472,8 @@ static DsqlRequest* prepareRequest(thread_db* tdbb, dsql_dbb* database, jrd_tra*
 // Prepare a statement for execution.
 // Note: caller is responsible for pool handling.
 static RefPtr<DsqlStatement> prepareStatement(thread_db* tdbb, dsql_dbb* database, jrd_tra* transaction,
-	ULONG textLength, const TEXT* text, USHORT clientDialect, bool isInternalRequest, ntrace_result_t* traceResult)
+	ULONG textLength, const TEXT* text, USHORT clientDialect, unsigned prepareFlags, bool isInternalRequest,
+	ntrace_result_t* traceResult)
 {
 	Database* const dbb = tdbb->getDatabase();
 
@@ -493,15 +494,18 @@ static RefPtr<DsqlStatement> prepareStatement(thread_db* tdbb, dsql_dbb* databas
 				  Arg::Gds(isc_command_end_err2) << Arg::Num(1) << Arg::Num(1));
 	}
 
-	// Get rid of the trailing ";" if there is one.
-
-	for (const TEXT* p = text + textLength; p-- > text;)
+	if (!(prepareFlags & IStatement::PREPARE_REQUIRE_SEMICOLON))
 	{
-		if (*p != ' ')
+		// Get rid of the trailing ";" if there is one.
+
+		for (const TEXT* p = text + textLength; p-- > text;)
 		{
-			if (*p == ';')
-				textLength = p - text;
-			break;
+			if (*p != ' ')
+			{
+				if (*p == ';')
+					textLength = p - text;
+				break;
+			}
 		}
 	}
 
@@ -556,7 +560,9 @@ static RefPtr<DsqlStatement> prepareStatement(thread_db* tdbb, dsql_dbb* databas
 				scratch->flags |= DsqlCompilerScratch::FLAG_INTERNAL_REQUEST;
 
 			Parser parser(tdbb, *scratchPool, statementPool, scratch, clientDialect,
-				dbDialect, text, textLength, charSetId);
+				dbDialect,
+				(prepareFlags & IStatement::PREPARE_REQUIRE_SEMICOLON),
+				text, textLength, charSetId);
 
 			// Parse the SQL statement.  If it croaks, return
 			dsqlStatement = parser.parse();

src/dsql/parse.y

@@ -868,8 +868,15 @@ using namespace Firebird;
 // list of possible statements
 
 top
-	: statement			{ parsedStatement = $1; }
-	| statement ';'		{ parsedStatement = $1; }
+	: statement
+		{
+			if (requireSemicolon)
+				yyerrorIncompleteCmd(YYPOSNARG(1));
+
+			parsedStatement = $1;
+		}
+	| statement ';'
+		{ parsedStatement = $1; }
 	;
 
 %type <dsqlStatement> statement

src/include/firebird/FirebirdInterface.idl

@@ -474,6 +474,7 @@ interface Statement : ReferenceCounted
 	const uint PREPARE_PREFETCH_DETAILED_PLAN		= 0x10;
 	const uint PREPARE_PREFETCH_AFFECTED_RECORDS	= 0x20;	// not used yet
 	const uint PREPARE_PREFETCH_FLAGS				= 0x40;
+	const uint PREPARE_REQUIRE_SEMICOLON			= 0x80;
 	const uint PREPARE_PREFETCH_METADATA =
 		PREPARE_PREFETCH_TYPE | PREPARE_PREFETCH_FLAGS |
 		PREPARE_PREFETCH_INPUT_PARAMETERS | PREPARE_PREFETCH_OUTPUT_PARAMETERS;

src/include/firebird/IdlFbInterfaces.h

@@ -1914,6 +1914,7 @@ namespace Firebird
 		static CLOOP_CONSTEXPR unsigned PREPARE_PREFETCH_DETAILED_PLAN = 0x10;
 		static CLOOP_CONSTEXPR unsigned PREPARE_PREFETCH_AFFECTED_RECORDS = 0x20;
 		static CLOOP_CONSTEXPR unsigned PREPARE_PREFETCH_FLAGS = 0x40;
+		static CLOOP_CONSTEXPR unsigned PREPARE_REQUIRE_SEMICOLON = 0x80;
 		static CLOOP_CONSTEXPR unsigned PREPARE_PREFETCH_METADATA = IStatement::PREPARE_PREFETCH_TYPE | IStatement::PREPARE_PREFETCH_FLAGS | IStatement::PREPARE_PREFETCH_INPUT_PARAMETERS | IStatement::PREPARE_PREFETCH_OUTPUT_PARAMETERS;
 		static CLOOP_CONSTEXPR unsigned PREPARE_PREFETCH_ALL = IStatement::PREPARE_PREFETCH_METADATA | IStatement::PREPARE_PREFETCH_LEGACY_PLAN | IStatement::PREPARE_PREFETCH_DETAILED_PLAN | IStatement::PREPARE_PREFETCH_AFFECTED_RECORDS;
 		static CLOOP_CONSTEXPR unsigned FLAG_HAS_CURSOR = 0x1;

src/include/firebird/impl/msg/isql.h

@@ -202,3 +202,5 @@ FB_IMPL_MSG_SYMBOL(ISQL, 202, NO_PUBLICATIONS, "There is no publications in this
 FB_IMPL_MSG_SYMBOL(ISQL, 203, MSG_PUBLICATIONS, "Publications:")
 FB_IMPL_MSG_SYMBOL(ISQL, 204, MSG_PROCEDURES, "Procedures:")
 FB_IMPL_MSG_SYMBOL(ISQL, 205, HLP_EXPLAIN, "EXPLAIN                    -- explain a query access plan")
+FB_IMPL_MSG_SYMBOL(ISQL, 206, USAGE_AUTOTERM, "	-autot(erm)             use auto statement terminator (set autoterm on)")
+FB_IMPL_MSG_SYMBOL(ISQL, 207, AUTOTERM_NOT_SUPPORTED, "SET AUTOTERM ON is not supported in engine/server and has been disabled")

src/include/gen/Firebird.pas

@@ -1533,6 +1533,7 @@ type
 		const PREPARE_PREFETCH_DETAILED_PLAN = Cardinal($10);
 		const PREPARE_PREFETCH_AFFECTED_RECORDS = Cardinal($20);
 		const PREPARE_PREFETCH_FLAGS = Cardinal($40);
+		const PREPARE_REQUIRE_SEMICOLON = Cardinal($80);
 		const PREPARE_PREFETCH_METADATA = Cardinal(IStatement.PREPARE_PREFETCH_TYPE or IStatement.PREPARE_PREFETCH_FLAGS or IStatement.PREPARE_PREFETCH_INPUT_PARAMETERS or IStatement.PREPARE_PREFETCH_OUTPUT_PARAMETERS);
 		const PREPARE_PREFETCH_ALL = Cardinal(IStatement.PREPARE_PREFETCH_METADATA or IStatement.PREPARE_PREFETCH_LEGACY_PLAN or IStatement.PREPARE_PREFETCH_DETAILED_PLAN or IStatement.PREPARE_PREFETCH_AFFECTED_RECORDS);
 		const FLAG_HAS_CURSOR = Cardinal($1);

src/isql/FrontendLexer.cpp

@@ -0,0 +1,386 @@
+/*
+ *  The contents of this file are subject to the Initial
+ *  Developer's Public License Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the
+ *  License. You may obtain a copy of the License at
+ *  http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl.
+ *
+ *  Software distributed under the License is distributed AS IS,
+ *  WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing rights
+ *  and limitations under the License.
+ *
+ *  The Original Code was created by Adriano dos Santos Fernandes
+ *  for the Firebird Open Source RDBMS project.
+ *
+ *  Copyright (c) 2023 Adriano dos Santos Fernandes <adrianosf at gmail.com>
+ *  and all contributors signed below.
+ *
+ *  All Rights Reserved.
+ *  Contributor(s): ______________________________________.
+ *
+ */
+
+#include "firebird.h"
+#include "../isql/FrontendLexer.h"
+#include <algorithm>
+#include <cctype>
+
+
+static std::string trim(std::string_view str);
+
+
+static std::string trim(std::string_view str)
+{
+	auto finish = str.end();
+	auto start = str.begin();
+
+	while (start != finish && isspace(*start))
+		++start;
+
+	--finish;
+
+	while (finish > start && isspace(*finish))
+		--finish;
+
+	return std::string(start, finish + 1);
+}
+
+
+std::string FrontendLexer::stripComments(std::string_view statement)
+{
+	FrontendLexer lexer(statement);
+	std::string processedStatement;
+
+	while (lexer.pos < lexer.end)
+	{
+		auto oldPos = lexer.pos;
+
+		lexer.skipSpacesAndComments();
+
+		if (lexer.pos > oldPos)
+			processedStatement += ' ';
+
+		oldPos = lexer.pos;
+
+		if (!lexer.getStringToken().has_value() && lexer.pos < lexer.end)
+			++lexer.pos;
+
+		processedStatement += std::string(oldPos, lexer.pos);
+	}
+
+	return trim(processedStatement);
+}
+
+bool FrontendLexer::isBufferEmpty() const
+{
+	return trim(std::string(deletePos, end)).empty();
+}
+
+void FrontendLexer::appendBuffer(std::string_view newBuffer)
+{
+	const auto posIndex = pos - buffer.begin();
+	const auto deletePosIndex = deletePos - buffer.begin();
+	buffer.append(newBuffer);
+	pos = buffer.begin() + posIndex;
+	end = buffer.end();
+	deletePos = buffer.begin() + deletePosIndex;
+}
+
+void FrontendLexer::reset()
+{
+	buffer.clear();
+	pos = buffer.begin();
+	end = buffer.end();
+	deletePos = buffer.begin();
+}
+
+std::variant<FrontendLexer::SingleStatement, FrontendLexer::IncompleteTokenError> FrontendLexer::getSingleStatement(
+	std::string_view term)
+{
+	const auto posIndex = pos - deletePos;
+	buffer.erase(buffer.begin(), deletePos);
+	pos = buffer.begin() + posIndex;
+	end = buffer.end();
+	deletePos = buffer.begin();
+
+	try
+	{
+		if (pos < end)
+		{
+			skipSpacesAndComments();
+
+			const auto savePos = pos;
+
+			if (pos + 1 < end && *pos == '?')
+			{
+				if (*++pos == '\r')
+					++pos;
+
+				if (pos < end && *pos == '\n')
+				{
+					deletePos = ++pos;
+					const auto statement = trim(std::string(buffer.cbegin(), pos));
+					return SingleStatement{statement, statement};
+				}
+			}
+
+			pos = savePos;
+		}
+
+		while (pos < end)
+		{
+			if (end - pos >= term.length() && std::equal(term.begin(), term.end(), pos))
+			{
+				const auto initialStatement = std::string(buffer.cbegin(), pos);
+				pos += term.length();
+				const auto trailingPos = pos;
+				skipSpacesAndComments();
+				deletePos = pos;
+
+				const auto statement1 = initialStatement + std::string(trailingPos, pos);
+				const auto statement2 = initialStatement + ";" + std::string(trailingPos, pos);
+
+				return SingleStatement{trim(statement1), trim(statement2)};
+			}
+
+			if (!getStringToken().has_value() && pos < end)
+				++pos;
+
+			skipSpacesAndComments();
+		}
+	}
+	catch (const IncompleteTokenError& error)
+	{
+		return error;
+	}
+
+	return IncompleteTokenError{false};
+}
+
+FrontendLexer::Token FrontendLexer::getToken()
+{
+	skipSpacesAndComments();
+
+	Token token;
+
+	if (pos >= end)
+	{
+		token.type = Token::TYPE_EOF;
+		return token;
+	}
+
+	if (const auto optStringToken = getStringToken(); optStringToken.has_value())
+		return optStringToken.value();
+
+	const auto start = pos;
+
+	switch (toupper(*pos))
+	{
+		case '(':
+			token.type = Token::TYPE_OPEN_PAREN;
+			token.processedText = *pos++;
+			break;
+
+		case ')':
+			token.type = Token::TYPE_CLOSE_PAREN;
+			token.processedText = *pos++;
+			break;
+
+		case ',':
+			token.type = Token::TYPE_COMMA;
+			token.processedText = *pos++;
+			break;
+
+		case ';':
+			token.type = Token::TYPE_OTHER;
+			token.processedText = *pos++;
+			break;
+
+		default:
+			while (pos != end && !isspace(*pos))
+				++pos;
+
+			token.processedText = std::string(start, pos);
+			std::transform(token.processedText.begin(), token.processedText.end(),
+				token.processedText.begin(), toupper);
+			break;
+	}
+
+	token.rawText = std::string(start, pos);
+
+	return token;
+}
+
+std::optional<FrontendLexer::Token> FrontendLexer::getStringToken()
+{
+	Token token;
+
+	if (pos >= end)
+		return std::nullopt;
+
+	const auto start = pos;
+
+	switch (toupper(*pos))
+	{
+		case '\'':
+		case '"':
+		{
+			const auto quote = *pos++;
+
+			while (pos != end)
+			{
+				if (*pos == quote)
+				{
+					if ((pos + 1) < end && *(pos + 1) == quote)
+						++pos;
+					else
+						break;
+				}
+
+				token.processedText += *pos++;
+			}
+
+			if (pos == end)
+			{
+				pos = start;
+				throw IncompleteTokenError{false};
+			}
+			else
+			{
+				++pos;
+				token.type = quote == '\'' ? Token::TYPE_STRING : Token::TYPE_META_STRING;
+			}
+
+			break;
+		}
+
+		case 'Q':
+			if (pos + 1 != end && pos[1] == '\'')
+			{
+				if (pos + 4 < end)
+				{
+					char endChar;
+
+					switch (pos[2])
+					{
+						case '{':
+							endChar = '}';
+							break;
+
+						case '[':
+							endChar = ']';
+							break;
+
+						case '(':
+							endChar = ')';
+							break;
+
+						case '<':
+							endChar = '>';
+							break;
+
+						default:
+							endChar = pos[2];
+							break;
+					}
+
+					pos += 3;
+
+					while (pos + 1 < end)
+					{
+						if (*pos == endChar && pos[1] == '\'')
+						{
+							pos += 2;
+							token.type = Token::TYPE_STRING;
+							break;
+						}
+
+						token.processedText += *pos++;
+					}
+				}
+
+				if (token.type != Token::TYPE_STRING)
+				{
+					pos = start;
+					throw IncompleteTokenError{false};
+				}
+
+				break;
+			}
+			[[fallthrough]];
+
+		default:
+			return std::nullopt;
+	}
+
+	token.rawText = std::string(start, pos);
+
+	return token;
+}
+
+void FrontendLexer::skipSpacesAndComments()
+{
+	while (pos != end && (isspace(*pos) || *pos == '-' || *pos == '/'))
+	{
+		while (pos != end && isspace(*pos))
+			++pos;
+
+		if (pos == end)
+			break;
+
+		if (*pos == '-')
+		{
+			if (pos + 1 != end && pos[1] == '-')
+			{
+				pos += 2;
+
+				while (pos != end)
+				{
+					const auto c = *pos++;
+
+					if (c == '\r')
+					{
+						if (pos != end && *pos == '\n')
+							++pos;
+						break;
+					}
+					else if (c == '\n')
+						break;
+				}
+			}
+			else
+				break;
+		}
+		else if (*pos == '/')
+		{
+			const auto start = pos;
+
+			if (pos + 1 != end && pos[1] == '*')
+			{
+				bool finished = false;
+				pos += 2;
+
+				while (pos != end)
+				{
+					const auto c = *pos++;
+
+					if (c == '*' && pos != end && *pos == '/')
+					{
+						++pos;
+						finished = true;
+						break;
+					}
+				}
+
+				if (!finished)
+				{
+					pos = start;
+					throw IncompleteTokenError{true};
+				}
+			}
+			else
+				break;
+		}
+	}
+}

src/isql/FrontendLexer.h

@@ -0,0 +1,113 @@
+/*
+ *  The contents of this file are subject to the Initial
+ *  Developer's Public License Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the
+ *  License. You may obtain a copy of the License at
+ *  http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl.
+ *
+ *  Software distributed under the License is distributed AS IS,
+ *  WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing rights
+ *  and limitations under the License.
+ *
+ *  The Original Code was created by Adriano dos Santos Fernandes
+ *  for the Firebird Open Source RDBMS project.
+ *
+ *  Copyright (c) 2023 Adriano dos Santos Fernandes <adrianosf at gmail.com>
+ *  and all contributors signed below.
+ *
+ *  All Rights Reserved.
+ *  Contributor(s): ______________________________________.
+ *
+ */
+
+#ifndef FB_ISQL_FRONTEND_LEXER_H
+#define FB_ISQL_FRONTEND_LEXER_H
+
+#include <optional>
+#include <string>
+#include <string_view>
+#include <variant>
+
+class FrontendLexer
+{
+public:
+	struct Token
+	{
+		enum Type
+		{
+			TYPE_EOF,
+			TYPE_STRING,
+			TYPE_META_STRING,
+			TYPE_OPEN_PAREN,
+			TYPE_CLOSE_PAREN,
+			TYPE_COMMA,
+			TYPE_OTHER
+		};
+
+		Type type = TYPE_OTHER;
+		std::string rawText;
+		std::string processedText;
+	};
+
+	struct SingleStatement
+	{
+		std::string withoutSemicolon;
+		std::string withSemicolon;
+	};
+
+	struct IncompleteTokenError
+	{
+		bool insideComment;
+	};
+
+public:
+	FrontendLexer(std::string_view aBuffer = {})
+		: buffer(aBuffer),
+		  pos(buffer.begin()),
+		  end(buffer.end()),
+		  deletePos(buffer.begin())
+	{
+	}
+
+	FrontendLexer(const FrontendLexer&) = delete;
+	FrontendLexer& operator=(const FrontendLexer&) = delete;
+
+public:
+	static std::string stripComments(std::string_view statement);
+
+public:
+	auto getBuffer() const
+	{
+		return buffer;
+	}
+
+	auto getPos() const
+	{
+		return pos;
+	}
+
+	void rewind()
+	{
+		deletePos = buffer.begin();
+	}
+
+	bool isBufferEmpty() const;
+
+	void appendBuffer(std::string_view newBuffer);
+	void reset();
+	std::variant<SingleStatement, FrontendLexer::IncompleteTokenError> getSingleStatement(std::string_view term);
+	Token getToken();
+
+private:
+	std::optional<Token> getStringToken();
+	void skipSpacesAndComments();
+
+private:
+	std::string buffer;
+	std::string::const_iterator pos;
+	std::string::const_iterator end;
+	std::string::const_iterator deletePos;
+};
+
+#endif	// FB_ISQL_FRONTEND_LEXER_H

src/isql/InputDevices.cpp

@@ -251,7 +251,8 @@ void InputDevices::saveCommand(const char* statement, const char* term)
 		if (f)
 		{
 			fputs(statement, f);
-			fputs(term, f);
+			if (*term)
+				fputs(term, f);
 			// Add newline to make the file more readable.
 			fputc('\n', f);
 		}

src/isql/isql.epp

@@ -54,6 +54,7 @@
 #include <math.h>
 #include <ctype.h>
 #include <errno.h>
+#include "../isql/FrontendLexer.h"
 #include "../common/utils_proto.h"
 #include "../common/classes/array.h"
 #include "../common/classes/init.h"
@@ -123,10 +124,7 @@ enum literal_string_type
 #include "../common/classes/MsgPrint.h"
 #include "../common/classes/array.h"
 
-using Firebird::string;
-using Firebird::PathName;
-using Firebird::TempFile;
-using Firebird::TimeZoneUtil;
+using namespace Firebird;
 using MsgFormat::SafeArg;
 
 #include "../isql/ColList.h"
@@ -304,11 +302,6 @@ static inline int fb_isspace(const char c)
 	return isspace((int)(UCHAR)c);
 }
 
-static inline int fb_isspace(const SSHORT c)
-{
-	return isspace((int)(UCHAR)c);
-}
-
 static inline int fb_isdigit(const char c)
 {
 	return isdigit((int)(UCHAR)c);
@@ -466,6 +459,7 @@ static processing_state add_row(TEXT*);
 static processing_state blobedit(const TEXT*, const TEXT* const*);
 static processing_state bulk_insert_hack(const char* command);
 static bool bulk_insert_retriever(const char* prompt);
+static void check_autoterm();
 static bool check_date(const tm& times);
 static bool check_time(const tm& times);
 static bool check_timestamp(const tm& times, const int msec);
@@ -488,7 +482,6 @@ static void frontend_load_parms(const TEXT* p, TEXT* parms[], TEXT* lparms[],
 static processing_state do_set_command(const TEXT*, bool*);
 static processing_state get_dialect(const char* const dialect_str,
 	char* const bad_dialect_buf, bool& bad_dialect);
-static processing_state get_statement(string&, const TEXT*);
 static bool get_numeric(const UCHAR*, USHORT, SSHORT*, SINT64*);
 static void print_set(const char* str, bool v);
 static processing_state print_sets();
@@ -515,7 +508,7 @@ static void process_plan();
 static void process_exec_path();
 static SINT64 process_record_count(const unsigned statement_type);
 static unsigned process_message_display(Firebird::IMessageMetadata* msg, unsigned pad[]);
-static processing_state process_statement(const TEXT*);
+static processing_state process_statement(const std::string&);
 #ifdef WIN_NT
 static BOOL CALLBACK query_abort(DWORD);
 #else
@@ -581,6 +574,7 @@ public:
 		Plan = false;
 		Planonly = false;
 		ExplainPlan = false;
+		AutoTerm = false;
 		Heading = true;
 		BailOnError = false;
 		StmtTimeout = 0;
@@ -607,6 +601,7 @@ public:
 	bool Plan;
 	bool Planonly;
 	bool ExplainPlan;
+	bool AutoTerm;
 	bool Heading;
 	bool BailOnError;
 	unsigned int StmtTimeout;
@@ -625,7 +620,7 @@ static FILE* Help;
 
 static const TEXT* const sql_prompt = "SQL> ";
 
-// Keep in sync with the chars that have their own "case" in get_statement(...).
+// Keep in sync with the chars that have their own "case" in the frontend lexer.
 static const char FORBIDDEN_TERM_CHARS[] = { '\n', '-', '*', '/', SINGLE_QUOTE, DBL_QUOTE };
 static const char FORBIDDEN_TERM_CHARS_DISPLAY[] = "<ENTER>, -, *, /, SINGLE_QUOTE, DOUBLE_QUOTE";
 
@@ -721,6 +716,32 @@ private:
 static Firebird::GlobalPtr<PerTableStats> perTableStats;
 
 
+class StatementGetter
+{
+public:
+	StatementGetter()
+	{
+		// Lookup the continuation prompt once
+		if (!*conPrompt)
+			IUTILS_msg_get(CON_PROMPT, conPrompt);
+	}
+
+public:
+	std::pair<FrontendLexer::SingleStatement, processing_state> getStatement();
+
+	void rewind()
+	{
+		lexer.rewind();
+	}
+
+private:
+	static TEXT conPrompt[MSG_LENGTH];
+	FrontendLexer lexer;
+};
+
+TEXT StatementGetter::conPrompt[MSG_LENGTH] = "";
+
+
 static UCHAR predefined_blob_subtype_bpb[] =
 {
 	isc_bpb_version1,
@@ -3984,6 +4005,51 @@ static bool bulk_insert_retriever(const char* prompt)
 }
 
 
+// Check if SET AUTOTERM is allowed. If not, disable it.
+static void check_autoterm()
+{
+	if (!DB || !setValues.AutoTerm)
+		return;
+
+	static const UCHAR protocolInfo[] =
+	{
+		fb_info_protocol_version,
+		isc_info_end
+	};
+
+	UCHAR buffer[BUFFER_LENGTH128];
+
+	DB->getInfo(fbStatus, sizeof(protocolInfo), protocolInfo, sizeof(buffer), buffer);
+	if (ISQL_errmsg(fbStatus))
+		return;
+
+	SLONG protocolVersion = -1;
+
+	for (ClumpletReader p(ClumpletReader::InfoResponse, buffer, sizeof(buffer)); !p.isEof(); p.moveNext())
+	{
+		switch (p.getClumpTag())
+		{
+			case isc_info_end:
+				break;
+
+			case fb_info_protocol_version:
+				protocolVersion = p.getInt();
+				break;
+		}
+	}
+
+	if (!(protocolVersion == 0 || protocolVersion >= 19) &&	// PROTOCOL_VERSION19
+		  ENCODE_ODS(isqlGlob.major_ods, isqlGlob.minor_ods) >= ODS_13_2)
+	{
+		setValues.AutoTerm = false;
+
+		TEXT errbuf[MSG_LENGTH];
+		IUTILS_msg_get(AUTOTERM_NOT_SUPPORTED, errbuf);
+		STDERROUT(errbuf);
+	}
+}
+
+
 // *******************
 // c h e c k _ d a t e
 // *******************
@@ -4432,13 +4498,11 @@ static void do_isql()
 	// Read statements and process them from Ifp until the ret
 	// code tells us we are done
 
-	string stmt;
-	processing_state ret;
-
+	StatementGetter statementGetter;
 	bool done = false;
+
 	while (!done)
 	{
-
 		if (Abort_flag)
 		{
 			if (D__trans)
@@ -4498,7 +4562,10 @@ static void do_isql()
 			}
 		}
 
-		ret = get_statement(stmt, sql_prompt);
+		auto [statement, ret] = statementGetter.getStatement();
+
+		if (!statement.withoutSemicolon.empty())
+			ret = frontend(FrontendLexer::stripComments(statement.withoutSemicolon).c_str());
 
 		// If there is no database yet, remind us of the need to connect
 
@@ -4511,6 +4578,7 @@ static void do_isql()
 				IUTILS_msg_get(NO_DB, errbuf);
 				STDERROUT(errbuf);
 			}
+
 			if (!Interactive && setValues.BailOnError)
 				ret = FAIL;
 			else
@@ -4520,16 +4588,29 @@ static void do_isql()
 		switch (ret)
 		{
 		case CONT:
-			if (process_statement(stmt.c_str()) == ps_ERR)
+			switch (process_statement(setValues.AutoTerm ? statement.withSemicolon : statement.withoutSemicolon))
 			{
-				Exit_value = FINI_ERROR;
-				if (!Interactive && setValues.BailOnError)
-					Abort_flag = true;
+				case TRUNCATED:
+					statementGetter.rewind();
+					break;
+
+				case ps_ERR:
+					Exit_value = FINI_ERROR;
+					if (!Interactive && setValues.BailOnError)
+						Abort_flag = true;
+					[[fallthrough]];
+
+				default:
+					// Place each non frontend statement in the temp file if we are reading from stdin.
+					Filelist->saveCommand(
+						(setValues.AutoTerm ? statement.withSemicolon : statement.withoutSemicolon).c_str(),
+						(setValues.AutoTerm ? "" : isqlGlob.global_Term));
+					break;
 			}
 			break;
 
 		case END:
-		case EOF:
+		case FOUND_EOF:
 		case EXIT:
 			if (Abort_flag)
 			{
@@ -4623,10 +4704,6 @@ static void do_isql()
 			done = true;
 			break;
 
-		case ERR_BUFFER_OVERFLOW:
-			IUTILS_msg_get(BUFFER_OVERFLOW, errbuf);
-			STDERROUT(errbuf);
-
 		case EXTRACT:
 		case EXTRACTALL:
 		default:
@@ -5368,7 +5445,7 @@ static processing_state frontend_set(const char* cmd, const char* const* parms,
 		enum set_commands
 		{
 			stat, count, list, plan, planonly, explain, blobdisplay, echo, autoddl,
-			width, transaction, terminator, names, time,
+			autoterm, width, transaction, terminator, names, time,
 			sqlda_display,
 			exec_path_display,
 			sql, warning, sqlCont, heading, bail,
@@ -5392,6 +5469,7 @@ static processing_state frontend_set(const char* cmd, const char* const* parms,
 		{SetOptions::blobdisplay, "BLOBDISPLAY", 4},
 		{SetOptions::echo, "ECHO", 0},
 		{SetOptions::autoddl, "AUTODDL", 4},
+		{SetOptions::autoterm, "AUTOTERM", 5},
 		{SetOptions::width, "WIDTH", 0},
 		{SetOptions::transaction, "TRANSACTION", 5},
 		{SetOptions::terminator, "TERMINATOR", 4},
@@ -5482,6 +5560,15 @@ static processing_state frontend_set(const char* cmd, const char* const* parms,
 		ret = do_set_command(parms[2], &setValues.Autocommit);
 		break;
 
+	case SetOptions::autoterm:
+		ret = do_set_command(parms[2], &setValues.AutoTerm);
+		if (setValues.AutoTerm)
+		{
+			isqlGlob.Termlen = 1;
+			strcpy(isqlGlob.global_Term, ";");
+		}
+		break;
+
 	case SetOptions::width:
 		ret = newsize(parms[2][0] == '"' ? lparms[2] : parms[2], parms[3]);
 		break;
@@ -5504,6 +5591,8 @@ static processing_state frontend_set(const char* cmd, const char* const* parms,
 				}
 			}
 
+			setValues.AutoTerm = false;
+
 			isqlGlob.Termlen = strlen(a);
 			if (isqlGlob.Termlen < MAXTERM_SIZE)
 			{
@@ -5808,85 +5897,59 @@ static processing_state get_dialect(const char* const dialect_str,
 }
 
 
-static processing_state get_statement(string& statement,
-							const TEXT* statement_prompt)
+std::pair<FrontendLexer::SingleStatement, processing_state> StatementGetter::getStatement()
 {
-/**************************************
- *
- *	g e t _ s t a t e m e n t
- *
- **************************************
- *
- * Functional description
- *	Get an SQL statement, or QUIT/EXIT command to process
- *
- *	Arguments:  Pointer to statement, size of statement_buffer and prompt msg.
- *
- **************************************/
-	processing_state ret = CONT;
-
-	// Lookup the continuation prompt once
-	TEXT con_prompt[MSG_LENGTH];
-	IUTILS_msg_get(CON_PROMPT, con_prompt);
-
-	if ((Interactive && !Input_file) || setValues.Echo) {
-		ISQL_prompt(statement_prompt);
-	}
-
-	// Clear out statement buffer
-	statement.resize(0);
-
-	// Set count of characters to zero
-
-	size_t valuable_count = 0; // counter of valuable (non-space) chars
-	size_t comment_pos = 0; // position of block comment start
-	size_t non_comment_pos = 0; // position of char after block comment
-	const size_t term_length = isqlGlob.Termlen - 1; // additional variable for decreasing calculation
-
 	Filelist->Ifp().indev_line = Filelist->Ifp().indev_aux;
-	bool done = false;
 
-	enum
-	{
-		normal,
-		in_single_line_comment,
-		in_block_comment,
-		in_single_quoted_string,
-		in_double_quoted_string
-	} state = normal;
-
-	char lastChar = '\0';
-	char altQuoteChar = '\0';
-	unsigned altQuoteStringLength = 0;
+	const auto* prompt = lexer.isBufferEmpty() ? sql_prompt : conPrompt;
+	std::string_view term(isqlGlob.global_Term, isqlGlob.Termlen);
+	std::string buffer;
 
-	while (!done)
+	while (true)
 	{
+		if ((Interactive && !Input_file) || setValues.Echo)
+			ISQL_prompt(prompt);
+
 		SSHORT c = getNextInputChar();
-		switch (c)
+
+		if (c == EOF)
 		{
-		case EOF:
 			// Go back to getc if we get interrupted by a signal.
 
 			if (SYSCALL_INTERRUPTED(errno))
 			{
 				errno = 0;
-				break;
+				continue;
 			}
 
+			lexer.appendBuffer(buffer);
+			buffer.clear();
+
 			// If there was something valuable before EOF - error
-			if (valuable_count > 0)
+			if (!lexer.isBufferEmpty())
 			{
-				TEXT errbuf[MSG_LENGTH];
-				IUTILS_msg_get(UNEXPECTED_EOF, errbuf);
-				STDERROUT(errbuf);
-				Exit_value = FINI_ERROR;
-				ret = FAIL;
+				bool isEmpty = false;
+
+				try
+				{
+					isEmpty = FrontendLexer::stripComments(lexer.getBuffer()).empty();
+				}
+				catch (const FrontendLexer::IncompleteTokenError&)
+				{
+				}
+
+				if (!isEmpty)
+				{
+					TEXT errbuf[MSG_LENGTH];
+					IUTILS_msg_get(UNEXPECTED_EOF, errbuf);
+					STDERROUT(errbuf);
+				}
 			}
 
 			// If we hit EOF at the top of the flist, exit time
 
 			if (Filelist->count() == 1)
-				return FOUND_EOF;
+				return {{}, FOUND_EOF};
 
 			// If this is not tmpfile, close it
 
@@ -5898,7 +5961,7 @@ static processing_state get_statement(string& statement,
 			Filelist->removeIntoIfp();
 
 			if ((Interactive && !Input_file) || setValues.Echo)
-				ISQL_prompt(statement_prompt);
+				prompt = sql_prompt;
 
 			// CVC: Let's detect if we went back to the first level.
 			if (Filelist->readingStdin())
@@ -5910,182 +5973,43 @@ static processing_state get_statement(string& statement,
 			// Try to convince the new routines to go back to previous file(s)
 			// This should fix the INPUT bug introduced with editline.
 			getColumn = -1;
-			break;
 
-		case '\n':
-//		case '\0': // In particular with readline the \n is removed
-			if (state == in_single_line_comment)
+			if (!lexer.isBufferEmpty())
 			{
-				state = normal;
+				lexer.reset();
+				Exit_value = FINI_ERROR;
+				return {{}, FAIL};
 			}
+		}
+		else
+		{
+			buffer += (char) c;
 
-			// Catch the help ? without a terminator
-			if (statement.length() == 1 && statement[0] == '?')
+			if (c == '\n' ||
+				(buffer.length() >= isqlGlob.Termlen &&
+					std::equal(buffer.end() - isqlGlob.Termlen, buffer.end(), term.begin())))
 			{
-				c = 0;
-				done = true;
-				break;
-			}
+				lexer.appendBuffer(buffer);
+				buffer.clear();
 
-			// If in a comment, keep reading
-			if ((Interactive && !Input_file) || setValues.Echo)
-			{
-				if (state == in_block_comment)
-				{
-					//  Block comment prompt.
-					ISQL_prompt("--> ");
-				}
-				else if (valuable_count == 0)
-				{
-					// Ignore a series of nothing at the beginning
-					ISQL_prompt(statement_prompt);
-				}
-				else
+				const auto singleStatementVar = lexer.getSingleStatement(term);
+
+				if (const auto singleStatement = std::get_if<FrontendLexer::SingleStatement>(&singleStatementVar))
+					return {*singleStatement, CONT};
+				else if (const auto incompleteTokenError =
+							std::get_if<FrontendLexer::IncompleteTokenError>(&singleStatementVar))
 				{
-					ISQL_prompt(con_prompt);
+					prompt =
+						incompleteTokenError->insideComment ? "---> " :
+						lexer.isBufferEmpty() ? sql_prompt :
+						conPrompt;
 				}
 			}
-
-			break;
-
-		case '-':
-			// Could this the be start of a single-line comment.
-			if (state == normal && statement.length() > 0 &&
-				statement[statement.length() - 1] == '-')
-			{
-				state = in_single_line_comment;
-				if (valuable_count == 1)
-					valuable_count = 0;
-			}
-			break;
-
-		case '*':
-			// Could this the be start of a comment.  We can only look back,
-			// not forward.
-			// Ignore possibilities of a comment beginning inside
-			// quoted strings.
-			if (state == normal && statement.length() > 0 &&
-				statement[statement.length() - 1] == '/' && statement.length() != non_comment_pos)
-			{
-				state = in_block_comment;
-				comment_pos = statement.length() - 1;
-				if (valuable_count == 1)
-					valuable_count = 0;
-			}
-			break;
-
-		case '/':
-			// Perhaps this is the end of a comment.
-			// Ignore possibilities of a comment ending inside
-			// quoted strings.
-			// Ignore things like /*/ since it isn't a block comment; only the start of it. Or end.
-			if (state == in_block_comment && statement.length() > 2 &&
-				statement[statement.length() - 1] == '*' && statement.length() > comment_pos + 2)
-			{
-				state = normal;
-				non_comment_pos = statement.length() + 1; // mark start of non-comment to track this: /**/*
-				valuable_count--; // This char is not valuable
-			}
-			break;
-
-		case SINGLE_QUOTE:
-			switch (state)
-			{
-				case normal:
-					if (lastChar == 'q' || lastChar == 'Q')
-					{
-						statement += (lastChar = c);
-						altQuoteChar = c = getNextInputChar();
-						altQuoteStringLength = statement.length();
-
-						switch (altQuoteChar)
-						{
-							case '{':
-								altQuoteChar = '}';
-								break;
-							case '(':
-								altQuoteChar = ')';
-								break;
-							case '[':
-								altQuoteChar = ']';
-								break;
-							case '<':
-								altQuoteChar = '>';
-								break;
-						}
-					}
-					else
-						altQuoteChar = '\0';
-
-					state = in_single_quoted_string;
-					break;
-				case in_single_quoted_string:
-					if (!altQuoteChar || (statement.length() != altQuoteStringLength + 1 && lastChar == altQuoteChar))
-						state = normal;
-					break;
-			}
-			break;
-
-		case DBL_QUOTE:
-			switch (state)
-			{
-				case normal:
-					state = in_double_quoted_string;
-					break;
-				case in_double_quoted_string:
-					state = normal;
-					break;
-			}
-			break;
-
-
-		default:
-			if (state == normal && c == isqlGlob.global_Term[term_length] &&
-				// one-char terminator or the beginning also match
-				(isqlGlob.Termlen == 1u ||
-				 (valuable_count >= term_length &&
-				  strncmp(&statement[statement.length() - term_length],
-				  	isqlGlob.global_Term, term_length) == 0)))
-			{
-				c = 0;
-				done = true;
-				statement.resize(statement.length() - term_length);
-			}
 		}
-
-		// Any non-space character is significant if not in comment
-		if (state != in_block_comment &&
-			state != in_single_line_comment &&
-			!fb_isspace(c) && c != EOF)
-		{
-			valuable_count++;
-			if (valuable_count == 1) // this is the first valuable char in stream
-			{
-				// ignore all previous crap
-				statement.resize(0);
-				non_comment_pos = 0;
-			}
-		}
-
-		statement += (lastChar = c);
 	}
 
-	// If this was a null statement, skip it
-	if (ret == CONT && statement.isEmpty())
-		ret = SKIP;
-
-	if (ret == CONT)
-		ret = frontend(statement.c_str());
-
-	if (ret == CONT)
-	{
-		// Place each non frontend statement in the temp file if we are reading
-		// from stdin.
-
-		Filelist->saveCommand(statement.c_str(), isqlGlob.global_Term);
-	}
-
-	return ret;
+	fb_assert(false);
+	return {{}, FOUND_EOF};
 }
 
 
@@ -6484,6 +6408,9 @@ static processing_state print_sets()
 			p = p->next;
 		}
 	}
+
+	print_set("Auto Term:", setValues.AutoTerm);
+
 	isqlGlob.printf("%-25s%s%s", "Terminator:", isqlGlob.global_Term, NEWLINE);
 
 	print_set("Time:", setValues.Time_display);
@@ -6991,6 +6918,8 @@ static processing_state newdb(TEXT* dbname,
 
 	global_Stmt = NULL;
 
+	check_autoterm();
+
 	return SKIP;
 }
 
@@ -7463,6 +7392,7 @@ static processing_state parse_arg(int argc, SCHAR** argv, SCHAR* tabname)
 					break;
 
 				case IN_SW_ISQL_TERM:
+					setValues.AutoTerm = false;
 					isqlGlob.Termlen = strlen(swarg_str);
 					if (isqlGlob.Termlen >= MAXTERM_SIZE) {
 						isqlGlob.Termlen = MAXTERM_SIZE - 1;
@@ -7604,6 +7534,10 @@ static processing_state parse_arg(int argc, SCHAR** argv, SCHAR* tabname)
 					}
 					break;
 
+				case IN_SW_ISQL_AUTOTERM:
+					setValues.AutoTerm = true;
+					break;
+
 				case IN_SW_ISQL_HELP:
 					ret = ps_ERR;
 					break;
@@ -8942,7 +8876,7 @@ static unsigned process_message_display(Firebird::IMessageMetadata* message, uns
 }
 
 
-static processing_state process_statement(const TEXT* str2)
+static processing_state process_statement(const std::string& str)
 {
 /**************************************
  *
@@ -9033,9 +8967,20 @@ static processing_state process_statement(const TEXT* str2)
 			flags |= Firebird::IStatement::PREPARE_PREFETCH_LEGACY_PLAN;
 	}
 
-	global_Stmt = DB->prepare(fbStatus, prepare_trans, 0, str2, isqlGlob.SQL_dialect, flags);
+	if (setValues.AutoTerm)
+		flags |= IStatement::PREPARE_REQUIRE_SEMICOLON;
+
+	global_Stmt = DB->prepare(fbStatus, prepare_trans,
+		0, str.c_str(), isqlGlob.SQL_dialect, flags);
+
 	if (failed())
 	{
+		if (setValues.AutoTerm &&
+			fb_utils::containsErrorCode(fbStatus->getErrors(), isc_command_end_err2))
+		{
+			return TRUNCATED;
+		}
+
 		if (isqlGlob.SQL_dialect == SQL_DIALECT_V6_TRANSITION && Input_file)
 		{
 			isqlGlob.printf("%s%s%s%s%s%s",
@@ -9043,7 +8988,7 @@ static processing_state process_statement(const TEXT* str2)
 							"**** Error preparing statement:",
 							NEWLINE,
 							NEWLINE,
-							str2,
+							str.c_str(),
 							NEWLINE);
 		}
 		ISQL_errmsg(fbStatus);
@@ -9160,7 +9105,7 @@ static processing_state process_statement(const TEXT* str2)
 			(statement_type == isc_info_sql_stmt_ddl ||
 			 statement_type == isc_info_sql_stmt_set_generator))
 		{
-			DB->execute(fbStatus, D__trans, 0, str2, isqlGlob.SQL_dialect, NULL, NULL, NULL, NULL);
+			DB->execute(fbStatus, D__trans, 0, str.c_str(), isqlGlob.SQL_dialect, NULL, NULL, NULL, NULL);
 			setValues.StmtTimeout = 0;
 			if (ISQL_errmsg(fbStatus))
 			{
@@ -9195,9 +9140,9 @@ static processing_state process_statement(const TEXT* str2)
 		if (statement_type == isc_info_sql_stmt_start_trans)
 		{
 			// CVC: Starting a txn can fail, too. Let's check it, although I
-			// suspect isql will catch it in frontend_set() through get_statement(),
+			// suspect isql will catch it in frontend_set() through StatementGetter,
 			// so this place has little chance to be reached.
-			if (newtrans(str2) == FAIL)
+			if (newtrans(str.c_str()) == FAIL)
 				return ps_ERR;
 
 			if (setValues.Stats && (print_performance(perf_before) == ps_ERR))

src/isql/isql.h

@@ -73,7 +73,7 @@ enum processing_state {
 	EXTRACTALL	=	8,
 	FETCH		=	9,
 	OBJECT_NOT_FOUND = 10,
-	ERR_BUFFER_OVERFLOW = 11
+	TRUNCATED = 11
 };
 
 // Which blob subtypes to print

src/isql/isqlswi.h

@@ -32,36 +32,37 @@ enum isql_switches
 {
 	IN_SW_ISQL_0			= 0,
 	IN_SW_ISQL_EXTRACTALL	= 1,
-	IN_SW_ISQL_BAIL 		= 2,
-	IN_SW_ISQL_CACHE		= 3,
-	IN_SW_ISQL_CHARSET		= 4,
-	IN_SW_ISQL_DATABASE 	= 5,
-	IN_SW_ISQL_ECHO 		= 6,
-	IN_SW_ISQL_EXTRACT		= 7,
-	IN_SW_ISQL_FETCHPASS	= 8,
-	IN_SW_ISQL_INPUT		= 9,
-	IN_SW_ISQL_MERGE		= 10,
-	IN_SW_ISQL_MERGE2		= 11,
-	IN_SW_ISQL_NOAUTOCOMMIT = 12,
-	IN_SW_ISQL_NODBTRIGGERS = 13,
-	IN_SW_ISQL_NOWARN		= 14,
-	IN_SW_ISQL_OUTPUT		= 15,
-	IN_SW_ISQL_PAGE 		= 16,
-	IN_SW_ISQL_PASSWORD 	= 17,
-	IN_SW_ISQL_QUIET		= 18,
-	IN_SW_ISQL_ROLE 		= 19,
-	IN_SW_ISQL_ROLE2		= 20,
-	IN_SW_ISQL_SQLDIALECT	= 21,
-	IN_SW_ISQL_TERM 		= 22,
+	IN_SW_ISQL_AUTOTERM		= 2,
+	IN_SW_ISQL_BAIL 		= 3,
+	IN_SW_ISQL_CACHE		= 4,
+	IN_SW_ISQL_CHARSET		= 5,
+	IN_SW_ISQL_DATABASE 	= 6,
+	IN_SW_ISQL_ECHO 		= 7,
+	IN_SW_ISQL_EXTRACT		= 8,
+	IN_SW_ISQL_FETCHPASS	= 9,
+	IN_SW_ISQL_INPUT		= 10,
+	IN_SW_ISQL_MERGE		= 11,
+	IN_SW_ISQL_MERGE2		= 12,
+	IN_SW_ISQL_NOAUTOCOMMIT = 13,
+	IN_SW_ISQL_NODBTRIGGERS = 14,
+	IN_SW_ISQL_NOWARN		= 15,
+	IN_SW_ISQL_OUTPUT		= 16,
+	IN_SW_ISQL_PAGE 		= 17,
+	IN_SW_ISQL_PASSWORD 	= 18,
+	IN_SW_ISQL_QUIET		= 19,
+	IN_SW_ISQL_ROLE 		= 20,
+	IN_SW_ISQL_ROLE2		= 21,
+	IN_SW_ISQL_SQLDIALECT	= 22,
+	IN_SW_ISQL_TERM 		= 23,
 #ifdef TRUSTED_AUTH
-	IN_SW_ISQL_TRUSTED		= 23,
+	IN_SW_ISQL_TRUSTED		= 24,
 #endif
-	IN_SW_ISQL_USER 		= 24,
-	IN_SW_ISQL_VERSION		= 25,
+	IN_SW_ISQL_USER 		= 25,
+	IN_SW_ISQL_VERSION		= 26,
 #ifdef DEV_BUILD
-	IN_SW_ISQL_EXTRACTTBL	= 26,
+	IN_SW_ISQL_EXTRACTTBL	= 27,
 #endif
-	IN_SW_ISQL_HELP 		= 27
+	IN_SW_ISQL_HELP 		= 28
 };
 
 
@@ -69,7 +70,8 @@ enum IsqlOptionType { iqoArgNone, iqoArgInteger, iqoArgString };
 
 static const Switches::in_sw_tab_t isql_in_sw_table[] =
 {
-	{IN_SW_ISQL_EXTRACTALL	, 0, "ALL"				, 0, 0, 0, false, false, 11	, 1, NULL, iqoArgNone},
+	{IN_SW_ISQL_EXTRACTALL	, 0, "ALL"				, 0, 0, 0, false, false, 11		, 1, NULL, iqoArgNone},
+	{IN_SW_ISQL_AUTOTERM	, 0, "AUTOTERM"			, 0, 0, 0, false, false, 205	, 5, NULL, iqoArgNone},
 	{IN_SW_ISQL_BAIL 		, 0, "BAIL"				, 0, 0, 0, false, false, 104	, 1, NULL, iqoArgNone},
 	{IN_SW_ISQL_CACHE		, 0, "CACHE"			, 0, 0, 0, false, false, 111	, 1, NULL, iqoArgInteger},
 	{IN_SW_ISQL_CHARSET		, 0, "CHARSET"			, 0, 0, 0, false, false, 122	, 2, NULL, iqoArgString},

src/isql/tests/FrontendLexerTest.cpp

@@ -0,0 +1,148 @@
+/*
+ *  The contents of this file are subject to the Initial
+ *  Developer's Public License Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the
+ *  License. You may obtain a copy of the License at
+ *  http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl.
+ *
+ *  Software distributed under the License is distributed AS IS,
+ *  WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing rights
+ *  and limitations under the License.
+ *
+ *  The Original Code was created by Adriano dos Santos Fernandes
+ *  for the Firebird Open Source RDBMS project.
+ *
+ *  Copyright (c) 2023 Adriano dos Santos Fernandes <adrianosf at gmail.com>
+ *  and all contributors signed below.
+ *
+ *  All Rights Reserved.
+ *  Contributor(s): ______________________________________.
+ *
+ */
+
+#include "firebird.h"
+#include "boost/test/unit_test.hpp"
+#include "../FrontendLexer.h"
+#include <variant>
+
+using namespace Firebird;
+
+BOOST_AUTO_TEST_SUITE(ISqlSuite)
+BOOST_AUTO_TEST_SUITE(FrontendLexerSuite)
+
+
+BOOST_AUTO_TEST_SUITE(FrontendLexerTests)
+
+BOOST_AUTO_TEST_CASE(StripCommentsTest)
+{
+	BOOST_TEST(FrontendLexer::stripComments(
+		"/* comment */ select 1 from rdb$database /* comment */") == "select 1 from rdb$database");
+
+	BOOST_TEST(FrontendLexer::stripComments(
+		"-- comment\nselect '123' /* comment */ from rdb$database -- comment") == "select '123' from rdb$database");
+}
+
+BOOST_AUTO_TEST_CASE(GetSingleStatementTest)
+{
+	{	// scope
+		const std::string s1 =
+			"select /* ; */ -- ;\n"
+			"  ';' || q'{;}'\n"
+			"  from rdb$database;";
+
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(FrontendLexer(
+			s1 + "\nselect ...").getSingleStatement(";")).withSemicolon == s1);
+	}
+
+	{	// scope
+		const std::string s1 = "select 1 from rdb$database; -- comment";
+		const std::string s2 = "select 2 from rdb$database;";
+		const std::string s3 = "select 3 from rdb$database;";
+		const std::string s4 = "execute block returns (o1 integer) as begin o1 = 1;";
+		const std::string s5 = "end;";
+		const std::string s6 = "?";
+		const std::string s7 = "? set;";
+
+		FrontendLexer lexer(s1 + "\n" + s2);
+
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s1);
+
+		lexer.appendBuffer(s3 + "\n" + s4 + s5);
+
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s2);
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s3);
+
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s4);
+		lexer.rewind();
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s4 + s5);
+
+		lexer.appendBuffer(s6 + "\n" + s7);
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s6);
+		BOOST_TEST(std::get<FrontendLexer::SingleStatement>(lexer.getSingleStatement(";")).withSemicolon == s7);
+	}
+
+	BOOST_TEST(!std::get<FrontendLexer::IncompleteTokenError>(FrontendLexer(
+		"select 1 from rdb$database").getSingleStatement(";")).insideComment);
+	BOOST_TEST(std::get<FrontendLexer::IncompleteTokenError>(FrontendLexer(
+		"select 1 from rdb$database; /*").getSingleStatement(";")).insideComment);
+}
+
+BOOST_AUTO_TEST_CASE(SkipSingleLineCommentsTest)
+{
+	FrontendLexer lexer(
+		"-- comment 0\r\n"
+		"set -- comment 1\n"
+		"stats -- comment 2\r\n"
+		"- -- comment 3\n"
+		"-- comment 4"
+	);
+
+	BOOST_TEST(lexer.getToken().processedText == "SET");
+	BOOST_TEST(lexer.getToken().processedText == "STATS");
+	BOOST_TEST(lexer.getToken().processedText == "-");
+	BOOST_TEST(lexer.getToken().type == FrontendLexer::Token::TYPE_EOF);
+}
+
+BOOST_AUTO_TEST_CASE(SkipMultiLineCommentsTest)
+{
+	FrontendLexer lexer(
+		"/* comment 0 */\r\n"
+		"set /* comment 1\n"
+		"comment 1 continuation\n"
+		"*/ stats /* comment 2\r\n"
+		"* */ / /* comment 3*/ /* comment 4*/"
+	);
+
+	BOOST_TEST(lexer.getToken().processedText == "SET");
+	BOOST_TEST(lexer.getToken().processedText == "STATS");
+	BOOST_TEST(lexer.getToken().processedText == "/");
+	BOOST_TEST(lexer.getToken().type == FrontendLexer::Token::TYPE_EOF);
+}
+
+BOOST_AUTO_TEST_CASE(ParseStringsTest)
+{
+	FrontendLexer lexer(
+		"'ab''c\"d' "
+		"\"ab''c\"\"d\" "
+		"q'{ab'c\"d}' "
+		"q'(ab'c\"d)' "
+		"q'[ab'c\"d]' "
+		"q'<ab'c\"d>' "
+		"q'!ab'c\"d!' "
+	);
+
+	BOOST_TEST(lexer.getToken().processedText == "ab'c\"d");
+	BOOST_TEST(lexer.getToken().processedText == "ab''c\"d");
+	BOOST_TEST(lexer.getToken().processedText == "ab'c\"d");
+	BOOST_TEST(lexer.getToken().processedText == "ab'c\"d");
+	BOOST_TEST(lexer.getToken().processedText == "ab'c\"d");
+	BOOST_TEST(lexer.getToken().processedText == "ab'c\"d");
+	BOOST_TEST(lexer.getToken().processedText == "ab'c\"d");
+}
+
+BOOST_AUTO_TEST_SUITE_END()	// FrontendLexerTests
+
+
+BOOST_AUTO_TEST_SUITE_END()	// FrontendLexerSuite
+BOOST_AUTO_TEST_SUITE_END()	// ISqlSuite

src/remote/client/interface.cpp

@@ -4131,6 +4131,7 @@ Statement* Attachment::prepare(CheckStatusWrapper* status, ITransaction* apiTra,
 		prepare->p_sqlst_items.cstr_length = (ULONG) items.getCount();
 		prepare->p_sqlst_items.cstr_address = items.begin();
 		prepare->p_sqlst_buffer_length = (ULONG) buffer.getCount();
+		prepare->p_sqlst_flags = flags;
 
 		send_packet(rdb->rdb_port, packet);
 

src/remote/inet.cpp

@@ -716,7 +716,8 @@ rem_port* INET_analyze(ClntAuthBlock* cBlock,
 		REMOTE_PROTOCOL(PROTOCOL_VERSION15, ptype_lazy_send, 6),
 		REMOTE_PROTOCOL(PROTOCOL_VERSION16, ptype_lazy_send, 7),
 		REMOTE_PROTOCOL(PROTOCOL_VERSION17, ptype_lazy_send, 8),
-		REMOTE_PROTOCOL(PROTOCOL_VERSION18, ptype_lazy_send, 9)
+		REMOTE_PROTOCOL(PROTOCOL_VERSION18, ptype_lazy_send, 9),
+		REMOTE_PROTOCOL(PROTOCOL_VERSION19, ptype_lazy_send, 10)
 	};
 	fb_assert(FB_NELEM(protocols_to_try) <= FB_NELEM(cnct->p_cnct_versions));
 	cnct->p_cnct_count = FB_NELEM(protocols_to_try);

src/remote/os/win32/xnet.cpp

@@ -307,7 +307,8 @@ rem_port* XNET_analyze(ClntAuthBlock* cBlock,
 		REMOTE_PROTOCOL(PROTOCOL_VERSION15, ptype_batch_send, 6),
 		REMOTE_PROTOCOL(PROTOCOL_VERSION16, ptype_batch_send, 7),
 		REMOTE_PROTOCOL(PROTOCOL_VERSION17, ptype_batch_send, 8),
-		REMOTE_PROTOCOL(PROTOCOL_VERSION18, ptype_batch_send, 9)
+		REMOTE_PROTOCOL(PROTOCOL_VERSION18, ptype_batch_send, 9),
+		REMOTE_PROTOCOL(PROTOCOL_VERSION19, ptype_lazy_send, 10)
 	};
 	fb_assert(FB_NELEM(protocols_to_try) <= FB_NELEM(cnct->p_cnct_versions));
 	cnct->p_cnct_count = FB_NELEM(protocols_to_try);

src/remote/protocol.cpp

@@ -700,6 +700,10 @@ bool_t xdr_protocol(RemoteXdr* xdrs, PACKET* p)
 		MAP(xdr_long, reinterpret_cast<SLONG&>(prep_stmt->p_sqlst_buffer_length));
 		// p_sqlst_buffer_length was USHORT in older versions
 		fixupLength(xdrs, prep_stmt->p_sqlst_buffer_length);
+
+		if (port->port_protocol >= PROTOCOL_VERSION19)
+			MAP(xdr_short, reinterpret_cast<SSHORT&>(prep_stmt->p_sqlst_flags));
+
 		DEBUG_PRINTSIZE(xdrs, p->p_operation);
 		return P_TRUE(xdrs, p);
 

src/remote/protocol.h

@@ -104,6 +104,11 @@ const USHORT PROTOCOL_VERSION17 = (FB_PROTOCOL_FLAG | 17);
 const USHORT PROTOCOL_VERSION18 = (FB_PROTOCOL_FLAG | 18);
 const USHORT PROTOCOL_FETCH_SCROLL = PROTOCOL_VERSION18;
 
+// Protocol 19:
+//	- supports passing flags to IStatement::prepare
+
+const USHORT PROTOCOL_VERSION19 = (FB_PROTOCOL_FLAG | 19);
+
 // Architecture types
 
 enum P_ARCH
@@ -612,6 +617,7 @@ typedef struct p_sqlst
     USHORT	p_sqlst_messages;			// Number of messages
     CSTRING	p_sqlst_out_blr;			// blr describing output message
     USHORT	p_sqlst_out_message_number;
+	USHORT	p_sqlst_flags;				// prepare flags
 } P_SQLST;
 
 typedef struct p_sqldata

src/remote/server/server.cpp

@@ -1925,7 +1925,7 @@ static bool accept_connection(rem_port* port, P_CNCT* connect, PACKET* send)
 	{
 		if ((protocol->p_cnct_version == PROTOCOL_VERSION10 ||
 			 (protocol->p_cnct_version >= PROTOCOL_VERSION11 &&
-			  protocol->p_cnct_version <= PROTOCOL_VERSION18)) &&
+			  protocol->p_cnct_version <= PROTOCOL_VERSION19)) &&
 			 (protocol->p_cnct_architecture == arch_generic ||
 			  protocol->p_cnct_architecture == ARCHITECTURE) &&
 			protocol->p_cnct_weight >= weight)
@@ -4840,7 +4840,8 @@ ISC_STATUS rem_port::prepare_statement(P_SQLST * prepareL, PACKET* sendL)
 	// stuff isc_info_length in front of info items buffer
 	*info = isc_info_length;
 	memmove(info + 1, prepareL->p_sqlst_items.cstr_address, infoLength++);
-	const unsigned int flags = StatementMetadata::buildInfoFlags(infoLength, info);
+	unsigned flags = StatementMetadata::buildInfoFlags(infoLength, info) |
+		prepareL->p_sqlst_flags;
 
 	ITransaction* iface = NULL;
 	if (transaction)