MySQL added to framework next to PostgreSQL ODBC data access with native, reflection free C# compilation (#6749)

* MySQL ODBC implementation added to PostgreSQL ODBC implementation

* Handling of Japanese characters in MariaDB ODBC connector

* Optimal batch updates for MySQL and PostgreSQL

Co-authored-by: LLT21 <[email protected]>

frameworks/CSharp/appmpower/appmpower-odbc-my.dockerfile

@@ -0,0 +1,55 @@
+FROM mcr.microsoft.com/dotnet/sdk:5.0 AS build
+RUN apt-get update
+RUN apt-get -yqq install clang zlib1g-dev libkrb5-dev libtinfo5
+RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
+   libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+   xz-utils tk-dev libffi-dev liblzma-dev pgpool2 vim-tiny
+
+WORKDIR /odbc
+
+RUN curl -L -o unixODBC-2.3.9.tar.gz ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.9.tar.gz
+RUN tar -xvf unixODBC-2.3.9.tar.gz
+
+WORKDIR /odbc/unixODBC-2.3.9
+RUN ./configure --prefix=/usr/local/unixODBC
+RUN make
+RUN make install
+
+ENV PATH=/usr/local/unixODBC/lib:$PATH
+
+WORKDIR /app
+COPY src .
+RUN dotnet publish -c Release -o out -r linux-x64 /p:Database=mysql
+
+# Construct the actual image that will run
+FROM mcr.microsoft.com/dotnet/aspnet:5.0 AS runtime
+
+RUN apt-get update
+# The following installs standard versions unixodbc 2.3.6 and pgsqlodbc 11
+#RUN apt-get install -y unixodbc odbc-postgresql
+# unixodbc still needs to be installed even if compiled locally
+RUN apt-get install -y unixodbc wget curl
+
+WORKDIR /odbc
+
+RUN curl -L -o mariadb-connector-odbc-3.1.13-debian-9-stretch-amd64.tar.gz https://downloads.mariadb.com/Connectors/odbc/connector-odbc-3.1.13/mariadb-connector-odbc-3.1.13-debian-9-stretch-amd64.tar.gz
+RUN tar -xvzf mariadb-connector-odbc-3.1.13-debian-9-stretch-amd64.tar.gz
+RUN cp mariadb-connector-odbc-3.1.13-debian-9-stretch-amd64/lib/mariadb/libm* /usr/lib/
+
+COPY --from=build /usr/local/unixODBC /usr/local/unixODBC
+
+# Check unixODBC version by: 
+# 1. Logging into containter: docker run --rm -it --entrypoint=/bin/bash techempower/tfb.test.appmpower
+# 2. odbcinst  --version
+
+ENV PATH=/usr/local/unixODBC/bin:$PATH
+
+WORKDIR /etc/
+COPY odbcinst.ini .
+
+WORKDIR /app
+COPY --from=build /app/out ./
+
+EXPOSE 8080
+
+ENTRYPOINT ["./appMpower"]

frameworks/CSharp/appmpower/appmpower-odbc-pg.dockerfile

@@ -0,0 +1,79 @@
+FROM mcr.microsoft.com/dotnet/sdk:5.0 AS build
+RUN apt-get update
+RUN apt-get -yqq install clang zlib1g-dev libkrb5-dev libtinfo5
+RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
+   libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+   xz-utils tk-dev libffi-dev liblzma-dev pgpool2 vim-tiny
+
+WORKDIR /odbc
+
+# To compile the latest postgresql odbc driver, postgresql itself needs to be installed
+RUN curl -L -o postgresql-13.3.tar.gz https://ftp.postgresql.org/pub/source/v13.3/postgresql-13.3.tar.gz
+RUN curl -L -o unixODBC-2.3.9.tar.gz ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.9.tar.gz
+RUN curl -L -o psqlodbc-13.01.0000.tar.gz https://ftp.postgresql.org/pub/odbc/versions/src/psqlodbc-13.01.0000.tar.gz
+
+RUN tar -xvf postgresql-13.3.tar.gz
+RUN tar -xvf unixODBC-2.3.9.tar.gz
+RUN tar -xvf psqlodbc-13.01.0000.tar.gz
+
+WORKDIR /odbc/postgresql-13.3
+RUN ./configure
+RUN make
+RUN make install
+
+ENV PATH=/usr/local/pgsql/bin:$PATH
+
+WORKDIR /odbc/unixODBC-2.3.9
+RUN ./configure --prefix=/usr/local/unixODBC
+RUN make
+RUN make install
+
+ENV PATH=/usr/local/unixODBC/lib:$PATH
+
+WORKDIR /odbc/psqlodbc-13.01.0000
+RUN ./configure --with-unixodbc=/usr/local/unixODBC --with-libpq=/usr/local/pgsql --prefix=/usr/local/pgsqlodbc
+RUN make
+RUN make install
+
+WORKDIR /app
+COPY src .
+RUN dotnet publish -c Release -o out -r linux-x64  /p:Database=postgresql
+
+# Construct the actual image that will run
+FROM mcr.microsoft.com/dotnet/aspnet:5.0 AS runtime
+
+RUN apt-get update
+# The following installs standard versions unixodbc 2.3.6 and pgsqlodbc 11
+#RUN apt-get install -y unixodbc odbc-postgresql
+# unixodbc still needs to be installed even if compiled locally
+RUN apt-get install -y unixodbc wget curl libpq-dev build-essential
+
+WORKDIR /odbc
+
+RUN curl -L -o pgpool-II-4.2.3.tar.gz https://www.pgpool.net/mediawiki/download.php?f=pgpool-II-4.2.3.tar.gz
+RUN tar -xvf pgpool-II-4.2.3.tar.gz
+
+WORKDIR /odbc/pgpool-II-4.2.3
+RUN ./configure
+RUN make
+RUN make install
+
+COPY --from=build /usr/local/unixODBC /usr/local/unixODBC
+
+# Check unixODBC version by: 
+# 1. Logging into containter: docker run --rm -it --entrypoint=/bin/bash techempower/tfb.test.appmpower
+# 2. odbcinst  --version
+
+ENV PATH=/usr/local/unixODBC/bin:$PATH
+
+COPY --from=build /usr/local/pgsqlodbc /usr/local/pgsqlodbc
+
+WORKDIR /etc/
+COPY odbcinst.ini .
+
+WORKDIR /app
+COPY --from=build /app/out ./
+
+EXPOSE 8080
+
+ENTRYPOINT ["./appMpower"]

frameworks/CSharp/appmpower/appmpower.dockerfile

@@ -1,39 +1,6 @@
 FROM mcr.microsoft.com/dotnet/sdk:5.0 AS build
 RUN apt-get update
 RUN apt-get -yqq install clang zlib1g-dev libkrb5-dev libtinfo5
-RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
-   libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
-   xz-utils tk-dev libffi-dev liblzma-dev pgpool2 vim-tiny
-
-WORKDIR /odbc
-
-# To compile the latest postgresql odbc driver, postgresql itself needs to be installed
-RUN curl -L -o postgresql-13.3.tar.gz https://ftp.postgresql.org/pub/source/v13.3/postgresql-13.3.tar.gz
-RUN curl -L -o unixODBC-2.3.9.tar.gz ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.9.tar.gz
-RUN curl -L -o psqlodbc-13.01.0000.tar.gz https://ftp.postgresql.org/pub/odbc/versions/src/psqlodbc-13.01.0000.tar.gz
-
-RUN tar -xvf postgresql-13.3.tar.gz
-RUN tar -xvf unixODBC-2.3.9.tar.gz
-RUN tar -xvf psqlodbc-13.01.0000.tar.gz
-
-WORKDIR /odbc/postgresql-13.3
-RUN ./configure
-RUN make
-RUN make install
-
-ENV PATH=/usr/local/pgsql/bin:$PATH
-
-WORKDIR /odbc/unixODBC-2.3.9
-RUN ./configure --prefix=/usr/local/unixODBC
-RUN make
-RUN make install
-
-ENV PATH=/usr/local/unixODBC/lib:$PATH
-
-WORKDIR /odbc/psqlodbc-13.01.0000
-RUN ./configure --with-unixodbc=/usr/local/unixODBC --with-libpq=/usr/local/pgsql --prefix=/usr/local/pgsqlodbc
-RUN make
-RUN make install
 
 WORKDIR /app
 COPY src .
@@ -41,35 +8,7 @@ RUN dotnet publish -c Release -o out -r linux-x64
 
 # Construct the actual image that will run
 FROM mcr.microsoft.com/dotnet/aspnet:5.0 AS runtime
-
 RUN apt-get update
-# The following installs standard versions unixodbc 2.3.6 and pgsqlodbc 11
-#RUN apt-get install -y unixodbc odbc-postgresql
-# unixodbc still needs to be installed even if compiled locally
-RUN apt-get install -y unixodbc wget curl libpq-dev build-essential
-
-WORKDIR /odbc
-
-RUN curl -L -o pgpool-II-4.2.3.tar.gz https://www.pgpool.net/mediawiki/download.php?f=pgpool-II-4.2.3.tar.gz
-RUN tar -xvf pgpool-II-4.2.3.tar.gz
-
-WORKDIR /odbc/pgpool-II-4.2.3
-RUN ./configure
-RUN make
-RUN make install
-
-COPY --from=build /usr/local/unixODBC /usr/local/unixODBC
-
-# Check unixODBC version by: 
-# 1. Logging into containter: docker run --rm -it --entrypoint=/bin/bash techempower/tfb.test.appmpower
-# 2. odbcinst  --version
-
-ENV PATH=/usr/local/unixODBC/bin:$PATH
-
-COPY --from=build /usr/local/pgsqlodbc /usr/local/pgsqlodbc
-
-WORKDIR /etc/
-COPY odbcinst.ini .
 
 WORKDIR /app
 COPY --from=build /app/out ./

frameworks/CSharp/appmpower/benchmark_config.json

@@ -5,6 +5,23 @@
       "default": {
         "plaintext_url": "/plaintext",
         "json_url": "/json",
+        "port": 8080,
+        "approach": "Realistic",
+        "classification": "Platform",
+        "database": "None",
+        "framework": "ASP.NET Core",
+        "language": "C#",
+        "orm": "Raw",
+        "platform": ".NET",
+        "flavor": "CoreCLR",
+        "webserver": "Kestrel",
+        "os": "Linux",
+        "database_os": "Linux",
+        "display_name": "appMpower [Middleware]",
+        "notes": "",
+        "versus": "aspcore-mw"
+      },
+      "odbc-pg": {
         "db_url": "/db",
         "query_url": "/queries?c=",
         "update_url": "/updates?c=",
@@ -21,9 +38,30 @@
         "webserver": "Kestrel",
         "os": "Linux",
         "database_os": "Linux",
-        "display_name": "appMpower",
+        "display_name": "appMpower [Middleware, Odbc, Pg]",
+        "notes": "",
+        "versus": "aspcore-mw-ado-pg"
+      },
+      "odbc-my": {
+        "db_url": "/db",
+        "query_url": "/queries?c=",
+        "update_url": "/updates?c=",
+        "fortune_url": "/fortunes",
+        "port": 8080,
+        "approach": "Realistic",
+        "classification": "Platform",
+        "database": "MySQL",
+        "framework": "ASP.NET Core",
+        "language": "C#",
+        "orm": "Raw",
+        "platform": ".NET",
+        "flavor": "CoreCLR",
+        "webserver": "Kestrel",
+        "os": "Linux",
+        "database_os": "Linux",
+        "display_name": "appMpower [Middleware, Odbc, My]",
         "notes": "",
-        "versus": "aspcore"
+        "versus": "aspcore-mw-ado-my"
       }
     }
   ]

frameworks/CSharp/appmpower/config.toml

@@ -4,16 +4,42 @@ name = "appmpower"
 [main]
 urls.plaintext = "/plaintext"
 urls.json = "/json"
+approach = "Realistic"
+classification = "Micro"
+database = "None"
+database_os = "Linux"
+os = "Linux"
+orm = "Raw"
+platform = ".NET"
+webserver = "kestrel"
+versus = "aspcore-mw"
+
+[odbc-pg]
 urls.db = "/db"
 urls.query = "/queries?c="
 urls.update = "/updates?c="
 urls.fortune = "/fortunes"
 approach = "Realistic"
-classification = "Platform"
+classification = "Micro"
 database = "Postgres"
 database_os = "Linux"
 os = "Linux"
 orm = "Raw"
 platform = ".NET"
-webserver = "kestrel"
-versus = "aspcore"
+webserver = "Kestrel"
+versus = "aspcore-mw-ado-pg"
+
+[odbc-my]
+urls.db = "/db"
+urls.query = "/queries?c="
+urls.update = "/updates?c="
+urls.fortune = "/fortunes"
+approach = "Realistic"
+classification = "Micro"
+database = "MySQL"
+database_os = "Linux"
+os = "Linux"
+orm = "Raw"
+platform = ".NET"
+webserver = "Kestrel"
+versus = "aspcore-mw-ado-my"

frameworks/CSharp/appmpower/odbcinst.ini

@@ -5,6 +5,7 @@ Pooling=0
 
 [ODBC Drivers]
 PostgreSQL = Installed
+MariaDB = Installed
 
 ;
 ;  odbcinst.ini
@@ -18,4 +19,9 @@ Description=ODBC for PostgreSQL
 ;Driver = /usr/lib/x86_64-linux-gnu/odbc/psqlodbcw.so
 Driver =/usr/local/pgsqlodbc/lib/psqlodbcw.so
 Threading = 0
-CPTimeout = 0
+CPTimeout = 0
+
+[MariaDB]
+Description=MariaDB ODBC for MySQL
+Driver = /usr/lib/libmaodbc.so
+Threading   = 0

frameworks/CSharp/appmpower/src/ConnectionStrings.cs

@@ -2,7 +2,10 @@ namespace appMpower
 {
    public static class ConnectionStrings
    {
-      //public const string OdbcConnection = "Driver={PostgreSQL};Server=host.docker.internal;Port=5432;Database=hello_world;Uid=benchmarkdbuser;Pwd=benchmarkdbpass;UseServerSidePrepare=1;Pooling=false";
+#if MYSQL
+      public const string OdbcConnection = "Driver={MariaDB};Server=tfb-database;Database=hello_world;Uid=benchmarkdbuser;Pwd=benchmarkdbpass;Pooling=false;OPTIONS=67108864;FLAG_FORWARD_CURSOR=1"; 
+#else
       public const string OdbcConnection = "Driver={PostgreSQL};Server=tfb-database;Database=hello_world;Uid=benchmarkdbuser;Pwd=benchmarkdbpass;UseServerSidePrepare=1;Pooling=false";
+#endif
    }
 }

frameworks/CSharp/appmpower/src/Db/PooledConnections.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Concurrent;
 using System.Data.Odbc;
 using System.Threading.Tasks;
@@ -9,7 +8,13 @@ namespace appMpower.Db
    {
       private static bool _connectionsCreated = false;
       private static short _createdConnections = 0;
-      private static short _maxConnections = 333;
+
+#if MYSQL
+      private static short _maxConnections = 1250; 
+#else
+      private static short _maxConnections = 500;
+#endif
+
       private static ConcurrentStack<PooledConnection> _stack = new ConcurrentStack<PooledConnection>();
       private static ConcurrentQueue<TaskCompletionSource<PooledConnection>> _waitingQueue = new ConcurrentQueue<TaskCompletionSource<PooledConnection>>();
 
@@ -29,7 +34,6 @@ namespace appMpower.Db
             }
 
             return pooledConnection;
-
          }
          else
          {
@@ -81,4 +85,4 @@ namespace appMpower.Db
          }
       }
    }
-}
+}

frameworks/CSharp/appmpower/src/FortunesView.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.IO.Pipelines;
+using System.Globalization;
 using System.Text;
 using System.Threading.Tasks;
 using System.Web;
@@ -33,7 +34,7 @@ namespace appMpower
 
          foreach (var fortune in fortunes)
          {
-            writer.Append(_fortunesRowStart).Append(fortune.Id).Append(_fortunesColumn).Append(HttpUtility.HtmlEncode(fortune.Message)).Append(_fortunesRowEnd);
+            writer.Append(_fortunesRowStart).Append(fortune.Id.ToString(CultureInfo.InvariantCulture)).Append(_fortunesColumn).Append(HttpUtility.HtmlEncode(fortune.Message)).Append(_fortunesRowEnd);
          }
 
          writer.Append(_fortunesTableEnd);

frameworks/CSharp/appmpower/src/Microsoft/BatchUpdateString.cs

@@ -26,24 +26,19 @@ namespace PlatformBenchmarks
 
          var sb = StringBuilderCache.Acquire();
 
-         //if (DatabaseServer == DatabaseServer.PostgreSql)
-         //{
-         //sb.Append("UPDATE world SET randomNumber = temp.randomNumber FROM (VALUES ");
-         //Enumerable.Range(0, lastIndex).ToList().ForEach(i => sb.Append($"(@Id_{i}, @Random_{i}), "));
-         //sb.Append($"(@Id_{lastIndex}, @Random_{lastIndex}) ORDER BY 1) AS temp(id, randomNumber) WHERE temp.id = world.id");
-         //}
-         //else
-         //{
-         //   Enumerable.Range(0, batchSize).ToList().ForEach(i => sb.Append($"UPDATE world SET randomnumber = @Random_{i} WHERE id = @Id_{i};"));
-         //}
-
-         /* --- this is extremely slow with odbc, probably due to int specification ? 
+         /*
          sb.Append("UPDATE world SET randomNumber = temp.randomNumber FROM (VALUES ");
          Enumerable.Range(0, lastIndex).ToList().ForEach(i => sb.Append("(?::int,?::int),"));
-         //sb.Append("(?::int,?::int) ORDER BY 1) AS temp(id, randomNumber) WHERE temp.id = world.id");
-         sb.Append("(?::int,?::int)) AS temp(id, randomNumber) WHERE temp.id = world.id");
+         sb.Append("(?::int,?::int) ORDER BY 1) AS temp(id, randomNumber) WHERE temp.id = world.id");
+         //sb.Append("(?::int,?::int)) AS temp(id, randomNumber) WHERE temp.id = world.id");
+         */
 
-         /* --- only for alternative update statement - will be used for MySQL */
+#if MYSQL
+         for (int i = 0; i < batchSize; i++)
+         {
+            sb.Append("UPDATE world SET randomNumber=? WHERE id=?;");
+         }
+#else
          sb.Append("UPDATE world SET randomNumber=CASE id ");
 
          for (int i = 0; i < batchSize; i++)
@@ -51,16 +46,15 @@ namespace PlatformBenchmarks
             sb.Append("WHEN ? THEN ? ");
          }
 
-         //Enumerable.Range(0, batchSize).ToList().ForEach(i => sb.Append("WHEN ? THEN ? "));
-         sb.Append("ELSE randomnumber END WHERE id IN (");
+         sb.Append("ELSE randomnumber END WHERE id IN(");
 
          for (int i = 0; i < lastIndex; i++)
          {
             sb.Append("?,");
          }
 
-         //Enumerable.Range(0, lastIndex).ToList().ForEach(i => sb.Append("?, "));
          sb.Append("?)");
+#endif
 
          return _queries[batchSize] = StringBuilderCache.GetStringAndRelease(sb);
       }

frameworks/CSharp/appmpower/src/RawDb.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
-using System.Text;
+using System.Data.Common;
 using System.Threading.Tasks;
 using appMpower.Db;
 
@@ -63,7 +63,13 @@ namespace appMpower
             fortunes.Add(new Fortune
             (
                 id: dataReader.GetInt32(0),
+#if MYSQL
+               //MariaDB ODBC connector does not correctly support Japanese characters in combination with default ADO.NET;
+               //as a solution we custom read this string
+                message: ReadColumn(dataReader, 1)
+#else
                 message: dataReader.GetString(1)
+#endif
             ));
          }
 
@@ -98,8 +104,10 @@ namespace appMpower
 
          var ids = PlatformBenchmarks.BatchUpdateString.Ids;
          var randoms = PlatformBenchmarks.BatchUpdateString.Randoms;
-         // --- only for alternative update statement - will be used for MySQL
+
+#if !MYSQL
          var jds = PlatformBenchmarks.BatchUpdateString.Jds;
+#endif      
 
          for (int i = 0; i < count; i++)
          {
@@ -111,11 +119,12 @@ namespace appMpower
             worlds[i].RandomNumber = randomNumber;
          }
 
-         // --- only for alternative update statement - will be used for MySQL
+#if !MYSQL
          for (int i = 0; i < count; i++)
          {
             updateCommand.CreateParameter(jds[i], DbType.Int32, worlds[i].Id);
          }
+#endif
 
          await updateCommand.ExecuteNonQueryAsync();
 
@@ -204,5 +213,23 @@ namespace appMpower
 
          return worlds;
       }
+
+      public static string ReadColumn(DbDataReader dbDataReader, int column)
+      {
+         long size = dbDataReader.GetBytes(column, 0, null, 0, 0);  //get the length of data
+         byte[] values = new byte[size];
+
+         int bufferSize = 64;
+         long bytesRead = 0;
+         int currentPosition = 0;
+
+         while (bytesRead < size)
+         {
+            bytesRead += dbDataReader.GetBytes(column, currentPosition, values, currentPosition, bufferSize);
+            currentPosition += bufferSize;
+         }
+
+         return System.Text.Encoding.Default.GetString(values);
+      }
    }
 }

frameworks/CSharp/appmpower/src/appMpower.csproj

@@ -24,4 +24,8 @@
     <PackageReference Include="Microsoft.DotNet.ILCompiler" Version="6.0.0-*" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <DefineConstants Condition=" '$(Database)' == 'mysql' ">$(DefineConstants);MYSQL</DefineConstants>
+  </PropertyGroup>
+
 </Project>