Restore fix lost in merge: [LY-121929] Asset Processor Error : Sqlite - Failed to prepare statement. (#2202)

Fixed unresolved product dependency query to be able to handle much larger queries (tested with 100,000 products) by storing queried products in a temp table first

Signed-off-by: amzn-mike <[email protected]>

Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.cpp

@@ -850,6 +850,32 @@ namespace AzToolsFramework
                 "SELECT * FROM ProductDependencies WHERE UnresolvedPath LIKE \":%\"";
             static const auto s_queryProductDependencyExclusions = MakeSqlQuery(GET_PRODUCT_DEPENDENCY_EXCLUSIONS, GET_PRODUCT_DEPENDENCY_EXCLUSIONS_STATEMENT, LOG_NAME);
 
+            static const char* CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE =
+                "AssetProcessor::CreateUnresolvedProductDependenciesTempTable";
+            static const char* CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE_STATEMENT =
+                "CREATE TEMPORARY TABLE QueryProductDependenciesUnresolvedAdvanced(search)";
+            static const auto s_createUnresolvedProductDependenciesTempTable = MakeSqlQuery(
+                CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE, CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE_STATEMENT, LOG_NAME);
+
+            static const char* INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES = "AssetProcessor::InsertProductDependencyTempTableValues";
+            static const char* INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES_STATEMENT =
+                "INSERT INTO QueryProductDependenciesUnresolvedAdvanced VALUES (:filename)";
+            static const auto s_queryInsertProductDependencyTempTableValues = MakeSqlQuery(
+                INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES,
+                INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES_STATEMENT,
+                LOG_NAME,
+                SqlParam<const char*>(":filename"));
+
+            static const char* GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE =
+                "AssetProcessor::GetUnresolvedProductDependenciesUsingTempTable";
+            static const char* GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE_STATEMENT =
+                "SELECT * FROM ProductDependencies INNER JOIN QueryProductDependenciesUnresolvedAdvanced "
+                "ON (UnresolvedPath LIKE \"%*%\" AND search LIKE REPLACE(UnresolvedPath, \"*\", \"%\")) OR search = UnresolvedPath";
+            static const auto s_queryGetUnresolvedProductDependenciesUsingTempTable = MakeSqlQuery(
+                GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE,
+                GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE_STATEMENT,
+                LOG_NAME);
+
             // lookup by primary key
             static const char* QUERY_FILE_BY_FILEID = "AzToolsFramework::AssetDatabase::QueryFileByFileID";
             static const char* QUERY_FILE_BY_FILEID_STATEMENT =
@@ -1814,6 +1840,9 @@ namespace AzToolsFramework
             AddStatement(m_databaseConnection, s_queryAllProductdependencies);
             AddStatement(m_databaseConnection, s_queryUnresolvedProductDependencies);
             AddStatement(m_databaseConnection, s_queryProductDependencyExclusions);
+            AddStatement(m_databaseConnection, s_createUnresolvedProductDependenciesTempTable);
+            AddStatement(m_databaseConnection, s_queryInsertProductDependencyTempTableValues);
+            AddStatement(m_databaseConnection, s_queryGetUnresolvedProductDependenciesUsingTempTable);
 
             AddStatement(m_databaseConnection, s_queryFileByFileid);
             AddStatement(m_databaseConnection, s_queryFilesByFileName);
@@ -2480,88 +2509,26 @@ namespace AzToolsFramework
             return s_queryProductDependencyExclusions.BindAndQuery(*m_databaseConnection, handler, &GetProductDependencyResult);
         }
 
-        bool AssetDatabaseConnection::QueryProductDependenciesUnresolvedAdvanced(const AZStd::vector<AZStd::string>& searchPaths, productDependencyAndPathHandler handler)
+        bool AssetDatabaseConnection::QueryProductDependenciesUnresolvedAdvanced(
+            const AZStd::vector<AZStd::string>& searchPaths, productDependencyAndPathHandler handler)
         {
-            AZStd::string sql = R"END(select c.*, b.search FROM (select REPLACE(UnresolvedPath, "*", "%") as search, ProductDependencyID from ProductDependencies where UnresolvedPath LIKE "%*%") as a, ()END";
+            ScopedTransaction transaction(m_databaseConnection);
 
-            bool first = true;
+            bool result = s_createUnresolvedProductDependenciesTempTable.BindAndStep(*m_databaseConnection);
 
-            for(int i = 0; i < searchPaths.size(); ++i)
+            for (auto&& path : searchPaths)
             {
-                if(first)
-                {
-                    sql += "SELECT ? as search ";
-                    first = false;
-                }
-                else
-                {
-                    sql += "UNION SELECT ? ";
-                }
-            }
-
-            sql += R"END() as b
-            INNER JOIN ProductDependencies as c ON c.ProductDependencyID = a.ProductDependencyID
-            WHERE b.search LIKE a.search
-            UNION SELECT p.*, UnresolvedPath
-            FROM ProductDependencies as p
-            WHERE UnresolvedPath != ""
-            )END";
-
-            first = true;
-
-            for (int i = 0; i < searchPaths.size(); ++i)
-            {
-                if(first)
-                {
-                    sql += " AND ";
-                    first = false;
-                }
-                else
-                {
-                    sql += " OR ";
-                }
-
-                sql += "UnresolvedPath = ?";
+                result = s_queryInsertProductDependencyTempTableValues.BindAndStep(*m_databaseConnection, path.c_str()) && result;
             }
 
-            bool result = m_databaseConnection->ExecuteRawSqlQuery(sql, [handler](sqlite3_stmt* statement)
-            {
-                ProductDependencyDatabaseEntry entry;
-
-                entry.m_productDependencyID = SQLite::GetColumnInt64(statement, 0);
-                entry.m_productPK = SQLite::GetColumnInt64(statement, 1);
-                entry.m_dependencySourceGuid = SQLite::GetColumnUuid(statement, 2);
-                entry.m_dependencySubID = GetColumnInt(statement, 3);
-                entry.m_platform = GetColumnText(statement, 4);
-                entry.m_dependencyFlags = GetColumnInt64(statement, 5);
-                entry.m_unresolvedPath = GetColumnText(statement, 6);
-                entry.m_dependencyType = static_cast<ProductDependencyDatabaseEntry::DependencyType>(GetColumnInt(statement, 7));
-                entry.m_fromAssetId = sqlite3_column_int(statement, 8);
-
-                AZStd::string matchedPath = GetColumnText(statement, 9);
+            result = s_queryGetUnresolvedProductDependenciesUsingTempTable.BindAndQuery(
+                         *m_databaseConnection, handler, &GetProductDependencyAndPathResult) &&
+                result;
 
-                handler(entry, matchedPath);
+            result = m_databaseConnection->ExecuteRawSqlQuery("DROP TABLE QueryProductDependenciesUnresolvedAdvanced", nullptr, nullptr) &&
+                result;
 
-                return true;
-            }, [&searchPaths](sqlite3_stmt* statement)
-            {
-                int index = 1;
-
-                for (const auto& path : searchPaths)
-                {
-                    [[maybe_unused]] int res = sqlite3_bind_text(statement, index, path.c_str(), static_cast<int>(path.size()), nullptr);
-                    AZ_Assert(res == SQLITE_OK, "Statement::BindValueText: failed to bind!");
-                    ++index;
-                }
-
-                // Bind the same ones again since we looped this twice when making the query above
-                for (const auto& path : searchPaths)
-                {
-                    [[maybe_unused]] int res = sqlite3_bind_text(statement, index, path.c_str(), static_cast<int>(path.size()), nullptr);
-                    AZ_Assert(res == SQLITE_OK, "Statement::BindValueText: failed to bind!");
-                    ++index;
-                }
-            });
+            transaction.Commit();
 
             return result;
         }
@@ -2876,6 +2843,46 @@ namespace AzToolsFramework
                 return GetResult(callName, statement, handler);
             }
 
+            bool GetProductDependencyAndPathResult(
+                [[maybe_unused]] const char* callName,
+                Statement* statement,
+                AssetDatabaseConnection::productDependencyAndPathHandler handler)
+            {
+                Statement::SqlStatus result = statement->Step();
+
+                ProductDependencyDatabaseEntry productDependency;
+
+                AZStd::string relativeSearchPath;
+                auto boundColumns = CombineColumns(productDependency.GetColumns(), MakeColumns(MakeColumn("search", relativeSearchPath)));
+
+                bool validResult = result == Statement::SqlDone;
+                while (result == Statement::SqlOK)
+                {
+                    if (!boundColumns.Fetch(statement))
+                    {
+                        return false;
+                    }
+
+                    if (handler(productDependency, relativeSearchPath))
+                    {
+                        result = statement->Step();
+                    }
+                    else
+                    {
+                        result = Statement::SqlDone;
+                    }
+                    validResult = true;
+                }
+
+                if (result == Statement::SqlError)
+                {
+                    AZ_Warning(LOG_NAME, false, "Error occurred while stepping %s", callName);
+                    return false;
+                }
+
+                return validResult;
+            }
+
             bool GetMissingProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::missingProductDependencyHandler handler)
             {
                 return GetResult(callName, statement, handler);

Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h

@@ -614,6 +614,7 @@ namespace AzToolsFramework
 
             //! Returns any unresolved dependencies which match (by exact or wildcard match) the input searchPaths
             //! The extra path returned for each row is the searchPath entry that was matched with the returned dependency entry
+            //! @param searchPaths vector of relative paths to search for matches
             bool QueryProductDependenciesUnresolvedAdvanced(const AZStd::vector<AZStd::string>& searchPaths, productDependencyAndPathHandler handler);
 
             bool QueryMissingProductDependencyByProductId(AZ::s64 productId, missingProductDependencyHandler handler);
@@ -668,6 +669,7 @@ namespace AzToolsFramework
             bool GetProductResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productHandler handler, AZ::Uuid builderGuid = AZ::Uuid::CreateNull(), const char* jobKey = nullptr, AssetSystem::JobStatus status = AssetSystem::JobStatus::Any);
             bool GetLegacySubIDsResult(const char* callname, SQLite::Statement* statement, AssetDatabaseConnection::legacySubIDsHandler handler);
             bool GetProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productDependencyHandler handler);
+            bool GetProductDependencyAndPathResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productDependencyAndPathHandler handler);
             bool GetMissingProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::missingProductDependencyHandler handler);
             bool GetCombinedDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::combinedProductDependencyHandler handler);
             bool GetFileResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::fileHandler handler);

Code/Framework/AzToolsFramework/AzToolsFramework/SQLite/SQLiteConnection.cpp

@@ -302,7 +302,10 @@ namespace AzToolsFramework
                 return false;
             }
 
-            bindCallback(statement);
+            if (bindCallback)
+            {
+                bindCallback(statement);
+            }
 
             res = sqlite3_step(statement);
             bool validResult = res == SQLITE_DONE;
@@ -495,7 +498,7 @@ namespace AzToolsFramework
 
             int res = sqlite3_prepare_v2(db, m_parentPrototype->GetSqlText().c_str(), (int)m_parentPrototype->GetSqlText().length() + 1, &m_statement, NULL);
             
-            AZ_Error("SQLiteConnection", res == SQLITE_OK, "Statement::PrepareFirstTime: failed! %s ( prototype is '%s'). Error code returned is %d.", sqlite3_errmsg(db), m_parentPrototype->GetSqlText().c_str(), res);
+            AZ_Assert(res == SQLITE_OK, "Statement::PrepareFirstTime: failed! %s ( prototype is '%s'). Error code returned is %d.", sqlite3_errmsg(db), m_parentPrototype->GetSqlText().c_str(), res);
             return ((res == SQLITE_OK)&&(m_statement));
         }
 

Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp

@@ -2253,6 +2253,63 @@ namespace UnitTests
         EXPECT_EQ(m_errorAbsorber->m_numAssertsAbsorbed, 0);
     }
 
+    TEST_F(AssetDatabaseTest, QueryProductDependenciesUnresolvedAdvanced_HandlesLargeSearch_Success)
+    {
+        CreateCoverageTestData();
+
+        constexpr int NumTestPaths = 10000;
+
+        AZStd::vector<AZStd::string> searchPaths;
+
+        searchPaths.reserve(NumTestPaths);
+
+        for (int i = 0; i < NumTestPaths; ++i)
+        {
+            searchPaths.emplace_back(AZStd::string::format("%d.txt", i));
+        }
+
+        ProductDependencyDatabaseEntry dependency1(m_data->m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", false, "*.txt");
+        ProductDependencyDatabaseEntry dependency2(
+            m_data->m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", false, "default.xml");
+
+        m_data->m_connection.SetProductDependency(dependency1);
+        m_data->m_connection.SetProductDependency(dependency2);
+
+        AZStd::vector<AZStd::string> matches;
+        matches.reserve(NumTestPaths);
+
+        ASSERT_TRUE(m_data->m_connection.QueryProductDependenciesUnresolvedAdvanced(
+            searchPaths,
+            [&matches](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& /*entry*/, const AZStd::string& path)
+            {
+                matches.push_back(path);
+                return true;
+            }));
+
+        ASSERT_EQ(matches.size(), searchPaths.size());
+
+        // Check the first few results match
+        for (int i = 0; i < 10 && i < NumTestPaths; ++i)
+        {
+            ASSERT_STREQ(matches[i].c_str(), searchPaths[i].c_str());
+        }
+
+        matches.clear();
+        searchPaths.clear();
+        searchPaths.push_back("default.xml");
+
+        // Run the query again to make sure a) we can b) we don't get any extra results and c) we can query for exact (non wildcard) matches
+        ASSERT_TRUE(m_data->m_connection.QueryProductDependenciesUnresolvedAdvanced(
+            searchPaths,
+            [&matches](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& /*entry*/, const AZStd::string& path)
+            {
+                matches.push_back(path);
+                return true;
+            }));
+
+        ASSERT_THAT(matches, testing::ElementsAreArray(searchPaths));
+    }
+
     TEST_F(AssetDatabaseTest, QueryCombined_Succeeds)
     {
         // This test specifically checks that the legacy subIds returned by QueryCombined are correctly matched to only the one product that they're associated with