* Codepage-aware, allow unicode, refactoring of SQLDB code, add PG connector

git-svn-id: trunk@39381 -

.gitattributes

@@ -3472,6 +3472,7 @@ packages/fpindexer/src/ireaderhtml.pp svneol=native#text/plain
 packages/fpindexer/src/ireaderpas.pp svneol=native#text/plain
 packages/fpindexer/src/ireadertxt.pp svneol=native#text/plain
 packages/fpindexer/src/memindexdb.pp svneol=native#text/plain
+packages/fpindexer/src/pgindexdb.pp svneol=native#text/plain
 packages/fpindexer/src/sqldbindexdb.pp svneol=native#text/plain
 packages/fpindexer/src/sqliteindexdb.pp svneol=native#text/plain
 packages/fpmake.pp svneol=native#text/plain

packages/fpindexer/fpmake.pp

@@ -61,6 +61,9 @@ begin
       T:=P.Targets.AddUnit('fbindexdb.pp',SqliteOSes);
       T.Dependencies.AddUnit('fpindexer');
       
+      T:=P.Targets.AddUnit('pgindexdb.pp',SqldbConnectionOSes);
+      T.Dependencies.AddUnit('fpindexer');
+      
       T:=P.Targets.AddUnit('dbindexer.pp',SqldbConnectionOSes);
       T.Dependencies.AddUnit('fpindexer');
       T.Dependencies.AddUnit('ireadertxt');

packages/fpindexer/src/dbindexer.pp

@@ -395,9 +395,9 @@ Var
 begin
   Result:=0;
   T:='';
-  R:=TIReaderTXT.Create;
+  URL:=TableName+'/'+KeyField.AsString;
+  R:=TIReaderTXT.Create(URL,CP_UTF8);
   try
-    URL:=TableName+'/'+KeyField.AsString;
     For I:=0 to List.Count-1 do
       begin
       F:=TField(List.Objects[i]);

packages/fpindexer/src/fbindexdb.pp

@@ -26,26 +26,17 @@ type
 
   TFBIndexDB = class(TSQLDBIndexDB)
   private
-    FIB: TIBConnection;
     FGenQuery: Array[TIndexTable] of TSQLQuery;
     Function GetGenQuery(ATable : TIndexTable) : TSQLQuery;
-    function GetS(AIndex: integer): string;
-    procedure SetS(AIndex: integer; const AValue: string);
   protected
     procedure InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData); override;
-    function InsertWord(const AWord: string): int64; override;
-    function InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64; override;
-    function InsertLanguage(const ALanguage: string): int64; override;
-    function GetConnection: TSQLConnection; override;
+    function InsertWord(const AWord: UTF8string): int64; override;
+    function InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64; override;
+    function InsertLanguage(const ALanguage: UTF8string): int64; override;
+    function CreateConnection: TSQLConnection; override;
     function GetID(TableType: TIndexTable): int64;
     procedure FinishCreateTable(const TableType: TIndexTable); override;
-  public
-    constructor Create(AOwner: TComponent); override;
-    destructor Destroy; override;
-  published
-    property DatabasePath: string Index 0 read GetS write SetS;
-    property UserName: string Index 1 read GetS write SetS;
-    property Password: string Index 2 read GetS write SetS;
+    procedure FinishDropTable(const TableType: TIndexTable);override;
   end;
 
 implementation
@@ -56,7 +47,6 @@ function TFBIndexDB.GetID(TableType: TIndexTable): int64;
 
 var
   Q: TSQLQuery;
-  S: string;
 
 begin
   Q := GetGenQuery(TableType);
@@ -70,7 +60,7 @@ begin
   end;
 end;
 
-function TFBIndexDB.InsertLanguage(const ALanguage: string): int64;
+function TFBIndexDB.InsertLanguage(const ALanguage: UTF8string): int64;
 var
   Q: TSQLQuery;
 begin
@@ -81,7 +71,7 @@ begin
   Q.ExecSQL;
 end;
 
-function TFBIndexDB.InsertWord(const AWord: string): int64;
+function TFBIndexDB.InsertWord(const AWord: UTF8string): int64;
 var
   Q: TSQLQuery;
 begin
@@ -92,7 +82,7 @@ begin
   Q.ExecSQL;
 end;
 
-function TFBIndexDB.InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+function TFBIndexDB.InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
 var
   Q: TSQLQuery;
 begin
@@ -117,13 +107,19 @@ begin
   Q.ParamByName(GetFieldName(ifMatchesWordID)).AsLargeInt := aWordID;
   Q.ParamByName(GetFieldName(ifMatchesFileID)).AsLargeInt := aFileID;
   Q.ParamByName(GetFieldName(ifMatchesPosition)).AsLargeInt := ASearchData.Position;
-  Q.ParamByName(GetFieldName(ifMatchesContext)).AsString := ASearchData.Context;
+  Q.ParamByName(GetFieldName(ifMatchesContext)).AsString := Copy(ASearchData.Context,1,MaxContextLen);
   Q.ExecSQL;
 end;
 
-function TFBIndexDB.GetConnection: TSQLConnection;
+function TFBIndexDB.CreateConnection: TSQLConnection;
+Var
+  T : TIndexTable;
+
 begin
-  Result := FiB;
+  // So they are recreated
+  for T in TIndexTable do
+    FreeAndNil(FGenQuery[T]);
+  Result := TIBConnection.Create(Self);
 end;
 
 procedure TFBIndexDB.FinishCreateTable(const TableType: TIndexTable);
@@ -131,16 +127,9 @@ begin
   Execute('CREATE GENERATOR ' + DefaultGeneratorNames[TableType], True);
 end;
 
-constructor TFBIndexDB.Create(AOwner: TComponent);
-begin
-  inherited Create(AOwner);
-  FIB := TIBConnection.Create(nil);
-end;
-
-destructor TFBIndexDB.Destroy;
+procedure TFBIndexDB.FinishDropTable(const TableType: TIndexTable);
 begin
-  // Parent destroys FIB.
-  inherited Destroy;
+  Execute('DROP GENERATOR ' + DefaultGeneratorNames[TableType], True);
 end;
 
 function TFBIndexDB.GetGenQuery(ATable: TIndexTable): TSQLQuery;
@@ -153,25 +142,5 @@ begin
   Result:=FGenQuery[ATable];
 end;
 
-function TFBIndexDB.GetS(AIndex: integer): string;
-begin
-  case Aindex of
-    0: Result := FIB.DatabaseName;
-    1: Result := FIB.UserName;
-    2: Result := FIB.Password;
-    3: Result := FIB.HostName;
-  end;
-end;
-
-procedure TFBIndexDB.SetS(AIndex: integer; const AValue: string);
-begin
-  case Aindex of
-    0: FIB.DatabaseName := AValue;
-    1: FIB.UserName := AValue;
-    2: FIB.Password := AValue;
-    3: FIB.HostName := AValue;
-  end;
-end;
-
 end.
 

packages/fpindexer/src/fpmasks.pp

@@ -54,17 +54,17 @@ type
   private
     FMask: TMaskString;
   public
-    constructor Create(const AValue: String);
+    constructor Create(const AValue: UTF8string);
     destructor Destroy; override;
     
-    function Matches(const AFileName: String): Boolean;
+    function Matches(const AFileName: UTF8string): Boolean;
   end;
   
   { TParseStringList }
 
   TParseStringList = class(TStringList)
   public
-    constructor Create(const AText, ASeparators: String);
+    constructor Create(const AText, ASeparators: UTF8string);
   end;
   
   { TMaskList }
@@ -75,21 +75,21 @@ type
     function GetCount: Integer;
     function GetItem(Index: Integer): TMask;
   public
-    constructor Create(const AValue: String; ASeparator: Char = ';');
+    constructor Create(const AValue: UTF8string; ASeparator: Char = ';');
     destructor Destroy; override;
     
-    function Matches(const AFileName: String): Boolean;
+    function Matches(const AFileName: UTF8string): Boolean;
     
     property Count: Integer read GetCount;
     property Items[Index: Integer]: TMask read GetItem;
   end;
 
-function MatchesMask(const FileName, Mask: String): Boolean;
-function MatchesMaskList(const FileName, Mask: String; Separator: Char = ';'): Boolean;
+function MatchesMask(const FileName, Mask: UTF8string): Boolean;
+function MatchesMaskList(const FileName, Mask: UTF8string; Separator: Char = ';'): Boolean;
 
 implementation
 
-function MatchesMask(const FileName, Mask: String): Boolean;
+function MatchesMask(const FileName, Mask: UTF8string): Boolean;
 var
   AMask: TMask;
 begin
@@ -101,7 +101,7 @@ begin
   end;
 end;
 
-function MatchesMaskList(const FileName, Mask: String; Separator: Char): Boolean;
+function MatchesMaskList(const FileName, Mask: UTF8string; Separator: Char): Boolean;
 var
   AMaskList: TMaskList;
 begin
@@ -115,7 +115,7 @@ end;
 
 { TMask }
 
-constructor TMask.Create(const AValue: String);
+constructor TMask.Create(const AValue: UTF8string);
 var
   I: Integer;
   SkipAnyText: Boolean;
@@ -260,10 +260,10 @@ begin
   inherited Destroy;
 end;
 
-function TMask.Matches(const AFileName: String): Boolean;
+function TMask.Matches(const AFileName: UTF8string): Boolean;
 var
   L: Integer;
-  S: String;
+  S: UTF8string;
   
   function MatchToEnd(MaskIndex, CharIndex: Integer): Boolean;
   var
@@ -331,7 +331,7 @@ end;
 
 { TParseStringList }
 
-constructor TParseStringList.Create(const AText, ASeparators: String);
+constructor TParseStringList.Create(const AText, ASeparators: UTF8string);
 var
   I, S: Integer;
 begin
@@ -362,7 +362,7 @@ begin
   Result := FMasks.Count;
 end;
 
-constructor TMaskList.Create(const AValue: String; ASeparator: Char);
+constructor TMaskList.Create(const AValue: UTF8string; ASeparator: Char);
 var
   S: TParseStringList;
   I: Integer;
@@ -385,7 +385,7 @@ begin
   inherited Destroy;
 end;
 
-function TMaskList.Matches(const AFileName: String): Boolean;
+function TMaskList.Matches(const AFileName: UTF8string): Boolean;
 var
   I: Integer;
 begin

packages/fpindexer/src/ireaderhtml.pp

@@ -28,17 +28,17 @@ type
 
   TIReaderHTML = class(TCustomFileReader)
   private
-    sLine: string;
+    sLine: UTF8String;
     StartPos: integer;
     Offset: integer;
     LinePos: integer;
     Tg, Tx: integer;
     FParser: THTMLParser; //our htmlparser class
-    procedure OnTag(NoCaseTag, ActualTag: string);
-    procedure OnText(Text: string);
+    procedure OnTag(NoCaseTag, ActualTag: String);
+    procedure OnText(Text: String);
   protected
-    function GetToken: string; override;
-    function AllowedToken(token: string): boolean; override;
+    function GetToken: UTF8String; override;
+    function AllowedToken(token: UTF8String): boolean; override;
   public
     procedure LoadFromStream(FileStream: TStream); override;
   end;
@@ -47,13 +47,13 @@ implementation
 
 { TIReaderHTML }
 
-procedure TIReaderHTML.OnTag(NoCaseTag, ActualTag: string);
+procedure TIReaderHTML.OnTag(NoCaseTag, ActualTag: String);
 begin
 end;
 
-procedure TIReaderHTML.OnText(Text: string);
+procedure TIReaderHTML.OnText(Text: String);
 var
-  token: string;
+  token: UTF8String;
   s: TSearchWordData;
   i : Integer;
 begin
@@ -78,10 +78,10 @@ begin
     end;
 end;
 
-function TIReaderHTML.GetToken: string;
+function TIReaderHTML.GetToken: UTF8String;
 var
-  s: string;
-  c: string;
+  s: UTF8String;
+  c: UTF8String;
 begin
   Result := '';
 
@@ -126,7 +126,7 @@ begin
   end;
 end;
 
-function TIReaderHTML.AllowedToken(token: string): boolean;
+function TIReaderHTML.AllowedToken(token: UTF8String): boolean;
 begin
   Result := (Length(token) > 1) and
             (token <> 'nbsp') and (token <> 'quot') and (token <> 'apos') and

packages/fpindexer/src/ireaderpas.pp

@@ -28,7 +28,7 @@ type
   TIReaderPAS = class(TIReaderTXT)
   private
   protected
-    function AllowedToken(token: string): boolean; override;
+    function AllowedToken(token: UTF8String): boolean; override;
   public
     procedure LoadFromStream(FileStream: TStream); override;
   end;
@@ -37,7 +37,7 @@ implementation
 
 { TIReaderPAS }
 
-function TIReaderPAS.AllowedToken(token: string): boolean;
+function TIReaderPAS.AllowedToken(token: UTF8String): boolean;
 begin
   Result:=inherited AllowedToken(token);
 end;

packages/fpindexer/src/ireadertxt.pp

@@ -28,7 +28,7 @@ type
   TIReaderTXT = class(TCustomFileReader)
   private
   protected
-    function AllowedToken(token: string): boolean; override;
+    function AllowedToken(token: UTF8String): boolean; override;
   public
     procedure LoadFromStream(FileStream: TStream); override;
   end;
@@ -37,14 +37,14 @@ implementation
 
 { TIReaderTXT }
 
-function TIReaderTXT.AllowedToken(token: string): boolean;
+function TIReaderTXT.AllowedToken(token: UTF8String): boolean;
 begin
   Result := inherited AllowedToken(token) and (Length(token) > 1);
 end;
 
 procedure TIReaderTXT.LoadFromStream(FileStream: TStream);
 var
-  token: string;
+  token: UTF8String;
   p: TSearchWordData;
 begin
   inherited LoadFromStream(FileStream);

packages/fpindexer/src/memindexdb.pp

@@ -28,18 +28,18 @@ Type
 
   TDescrItem = Class(TCollectionItem)
   private
-    FDescription: String;
+    FDescription: UTF8string;
     FTheID: Integer;
   Protected
     Function BlockSize : Integer; virtual;
-    Procedure WriteStringToStream(Astream : TStream; S : String);
+    Procedure WriteStringToStream(Astream : TStream; S : UTF8string);
     Procedure WriteToStream(S : TStream); virtual;
     Procedure WriteRefToStream(AStream : Tstream; AItem : TDescrItem);
-    Function ReadStringFromStream(Astream : TStream) :  String;
+    Function ReadStringFromStream(Astream : TStream) :  UTF8string;
     Function ReadFromStream(S : TStream) : Integer; virtual;
   Published
     // The description
-    Property Description : String Read FDescription Write FDescription;
+    Property Description : UTF8string Read FDescription Write FDescription;
     // ID. Not used during work. Only used when loading/saving,
     // it then equals the Index. (index is slow, uses a linear search)
     Property TheID : Integer Read FTheID Write FTheID;
@@ -61,8 +61,8 @@ Type
     Procedure BeginLoading;
     Procedure EndLoading;
     Procedure AllocateIDS;
-    Function FindDescr(Const ADescription : String) : TDescrItem;
-    Function AddDescr(Const ADescription : String) : TDescrItem;
+    Function FindDescr(Const ADescription : UTF8string) : TDescrItem;
+    Function AddDescr(Const ADescription : UTF8string) : TDescrItem;
     Property Descr [AIndex : Integer] : TDescrItem Read GetD Write SetD; default;
   end;
 
@@ -115,8 +115,8 @@ Type
     FWord: TWordItem;
     FWordID : Integer;
     FURLID : Integer;
-    function GetContext: String;
-    procedure SetContext(AValue: String);
+    function GetContext: UTF8string;
+    procedure SetContext(AValue: UTF8string);
     procedure SetURL(AValue: TURLItem);
     procedure SetWord(AValue: TWordItem);
   protected
@@ -128,7 +128,7 @@ Type
     Property Word : TWordItem Read FWord Write SetWord;
     Property URL : TURLItem Read FURL Write SetURL;
     Property Position : Int64 Read FPosition Write FPosition;
-    Property Context : String Read GetContext Write SetContext;
+    Property Context : UTF8string Read GetContext Write SetContext;
   end;
 
   { TMatches }
@@ -151,7 +151,7 @@ Type
     FLanguages : TDescrCollection;
     FWords : TDescrCollection;
     FMatches : TMatches;
-    procedure GetMatches(AWord: String; SearchOptions: TSearchOptions;  AList: TFPList);
+    procedure GetMatches(AWord: UTF8string; SearchOptions: TSearchOptions;  AList: TFPList);
     procedure IntersectMatches(ListA, ListB: TFPList);
     procedure UnionMatches(ListA, ListB: TFPList);
   protected
@@ -166,7 +166,8 @@ Type
     procedure CommitTrans; override;
     procedure BeginTrans; override;
     procedure CompactDB; override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    Procedure CreateDB; override;
+    procedure DeleteWordsFromFile(URL: UTF8string); override;
     procedure AddSearchData(ASearchData: TSearchWordData); override;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); override;
     procedure CreateIndexerTables; override;
@@ -177,7 +178,7 @@ Type
 
   TFileIndexDB = Class(TMemIndexDB)
   private
-    FFIleName: String;
+    FFIleName: UTF8string;
     FWriteOnCommit: Boolean;
   Protected
     Procedure LoadFromStream; override;
@@ -186,7 +187,7 @@ Type
     procedure Connect; override;
     procedure DisConnect; override;
     procedure CommitTrans; override;
-    Property FileName : String Read FFIleName Write FFileName;
+    Property FileName : UTF8string Read FFIleName Write FFileName;
     Property WriteOnCommit : Boolean Read FWriteOnCommit Write FWriteOnCommit;
   end;
 
@@ -247,7 +248,7 @@ begin
   Result:=Sizeof(Integer)+Length(FDescription)*SizeOf(Char);
 end;
 
-procedure TDescrItem.WriteStringToStream(Astream: TStream; S: String);
+procedure TDescrItem.WriteStringToStream(Astream: TStream; S: UTF8string);
 Var
   L : Integer;
 begin
@@ -277,7 +278,7 @@ begin
   AStream.WriteBuffer(I,SizeOf(I));
 end;
 
-function TDescrItem.ReadStringFromStream(Astream: TStream): String;
+function TDescrItem.ReadStringFromStream(Astream: TStream): UTF8string;
 Var
   L : Integer;
 begin
@@ -440,6 +441,11 @@ begin
   // Do nothing
 end;
 
+procedure TMemIndexDB.CreateDB;
+begin
+  Clear;
+end;
+
 procedure TMemIndexDB.BeginTrans;
 begin
   // Do nothing
@@ -494,7 +500,7 @@ begin
     end;
 end;
 
-procedure TMemIndexDB.DeleteWordsFromFile(URL: string);
+procedure TMemIndexDB.DeleteWordsFromFile(URL: UTF8string);
 begin
  // inherited DeleteWordsFromFile(URL);
 end;
@@ -563,7 +569,7 @@ begin
     end;
 end;
 
-procedure TMemIndexDB.GetMatches(AWord : String; SearchOptions: TSearchOptions; AList : TFPList);
+procedure TMemIndexDB.GetMatches(AWord : UTF8string; SearchOptions: TSearchOptions; AList : TFPList);
 
   Procedure AddMatches(W : TWordItem);
   Var
@@ -608,7 +614,7 @@ procedure TMemIndexDB.FindSearchData(SearchWord: TWordParser;
 
 Var
   L,W : TFPList;
-  S : String;
+  S : UTF8string;
   I : Integer;
   M : TMatch;
   WD : TSearchWordData;
@@ -685,12 +691,12 @@ begin
     FURL.AddMatch(Self);
 end;
 
-function TMatch.GetContext: String;
+function TMatch.GetContext: UTF8string;
 begin
   Result:=Description;
 end;
 
-procedure TMatch.SetContext(AValue: String);
+procedure TMatch.SetContext(AValue: UTF8string);
 begin
   Description:=AValue;
 end;
@@ -807,7 +813,7 @@ begin
     GetD(i).TheID:=I;
 end;
 
-function TDescrCollection.FindDescr(const ADescription: String): TDescrItem;
+function TDescrCollection.FindDescr(const ADescription: UTF8string): TDescrItem;
 begin
 
   If FHash=Nil then
@@ -816,7 +822,7 @@ begin
     Result:=TDescrItem(FHash.Find(ADescription));
 end;
 
-function TDescrCollection.AddDescr(const ADescription: String): TDescrItem;
+function TDescrCollection.AddDescr(const ADescription: UTF8string): TDescrItem;
 begin
   Result:=Add as TDescrItem;
   Result.Description:=ADescription;

packages/fpindexer/src/pgindexdb.pp

@@ -0,0 +1,150 @@
+{
+    This file is part of the Free Component Library (FCL)
+    Copyright (c) 2012 by the Free Pascal development team
+
+    SQLDB Firebird based indexer
+    
+    See the file COPYING.FPC, included in this distribution,
+    for details about the copyright.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+ **********************************************************************}
+unit pgindexdb;
+
+{$mode objfpc}{$H+}
+
+interface
+
+uses
+  Classes, SysUtils, fpIndexer, sqldbIndexDB ,sqldb, pqconnection;
+
+type
+  { TPGIndexDB }
+
+  TPGIndexDB = class(TSQLDBIndexDB)
+  private
+    FGenQuery: Array[TIndexTable] of TSQLQuery;
+    Function GetGenQuery(ATable : TIndexTable) : TSQLQuery;
+  protected
+    procedure InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData); override;
+    function InsertWord(const AWord: UTF8String): int64; override;
+    function InsertURL(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64): int64; override;
+    function InsertLanguage(const ALanguage: UTF8String): int64; override;
+    function CreateConnection: TSQLConnection; override;
+    function GetID(TableType: TIndexTable): int64;
+    procedure FinishCreateTable(const TableType: TIndexTable); override;
+    procedure FinishDropTable(const TableType: TIndexTable); override;
+  end;
+
+implementation
+
+{ TPGIndexDB }
+
+function TPGIndexDB.GetID(TableType: TIndexTable): int64;
+
+var
+  Q: TSQLQuery;
+
+begin
+  Q := GetGenQuery(TableType);
+  Q.Open;
+  try
+    if (Q.EOF and Q.BOF) then
+      raise Exception.CreateFmt('Could not get ID for table %s', [GetTableName(TableType)]);
+    Result := Q.Fields[0].AsLargeInt;
+  finally
+    Q.Close;
+  end;
+end;
+
+function TPGIndexDB.InsertLanguage(const ALanguage: UTF8String): int64;
+var
+  Q: TSQLQuery;
+begin
+  Result := getID(itLanguages);
+  Q := CreateCachedQuery(cqtInsertLanguage, InsertSQL(itLanguages));
+  Q.ParamByName(GetFieldName(ifLanguagesID)).AsLargeInt := Result;
+  Q.ParamByName(GetFieldName(ifLanguagesName)).AsString := ALanguage;
+  Q.ExecSQL;
+end;
+
+function TPGIndexDB.InsertWord(const AWord: UTF8String): int64;
+var
+  Q: TSQLQuery;
+begin
+  Result := getID(itWords);
+  Q := CreateCachedQuery(cqtInsertWord, InsertSQL(itWords));
+  Q.ParamByName(GetFieldName(ifWordsID)).AsLargeInt := Result;
+  Q.ParamByName(GetFieldName(ifWordsWord)).AsString := AWord;
+  Q.ExecSQL;
+end;
+
+function TPGIndexDB.InsertURL(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+var
+  Q: TSQLQuery;
+begin
+  Result := getID(itFiles);
+  Q := CreateCachedQuery(cqtInsertFile, InsertSQL(itFiles));
+  Q.ParamByName(GetFieldName(ifFilesID)).AsLargeInt := Result;
+  Q.ParamByName(GetFieldName(ifFilesURL)).AsString := URL;
+  Q.ParamByName(GetFieldName(ifFilesTimeStamp)).AsDateTime := ATimeStamp;
+  Q.ParamByName(GetFieldName(ifFilesLanguageID)).AsLargeInt := ALanguageID;
+  Q.ParamByName(GetFieldName(ifFilesUpdated)).AsInteger := 0;
+  Q.ParamByName(GetFieldName(ifFilesReindex)).AsInteger := 0;
+  Q.ExecSQL;
+end;
+
+procedure TPGIndexDB.InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData);
+var
+  Q: TSQLQuery;
+begin
+  Q := CreateCachedQuery(cqtInsertMatch, InsertSQL(itMatches));
+  Q.ParamByName(GetFieldName(ifMatchesID)).AsLargeInt := GetID(itMatches);
+  Q.ParamByName(GetFieldName(ifMatchesLanguageID)).AsLargeInt := aLanguageID;
+  Q.ParamByName(GetFieldName(ifMatchesWordID)).AsLargeInt := aWordID;
+  Q.ParamByName(GetFieldName(ifMatchesFileID)).AsLargeInt := aFileID;
+  Q.ParamByName(GetFieldName(ifMatchesPosition)).AsLargeInt := ASearchData.Position;
+  Q.ParamByName(GetFieldName(ifMatchesContext)).AsString := Copy(ASearchData.Context,1,255);
+  Q.ExecSQL;
+end;
+
+function TPGIndexDB.CreateConnection: TSQLConnection;
+
+Var
+  T : TIndexTable;
+
+begin
+  // So they are recreated
+  for T in TIndexTable do
+    FreeAndNil(FGenQuery[T]);
+  Result := TPQConnection.Create(Self);
+end;
+
+procedure TPGIndexDB.FinishCreateTable(const TableType: TIndexTable);
+begin
+  Execute('CREATE SEQUENCE ' + DefaultGeneratorNames[TableType], True);
+end;
+
+procedure TPGIndexDB.FinishDropTable(const TableType: TIndexTable);
+begin
+  Execute('DROP SEQUENCE ' + DefaultGeneratorNames[TableType], True);
+end;
+
+
+function TPGIndexDB.GetGenQuery(ATable: TIndexTable): TSQLQuery;
+begin
+  If (FGenQuery[ATable]=Nil) then
+    begin
+    FGenQuery[ATable]:=CreateQuery(Format('select nextval(''%s'')', [DefaultGeneratorNames[ATable]]));
+    FGenQuery[ATable].ParseSQL:=False;
+    FGenQuery[ATable].Prepare;
+    end;
+  Result:=FGenQuery[ATable];
+end;
+
+
+end.
+

packages/fpindexer/src/sqldbindexdb.pp

@@ -26,10 +26,8 @@ type
   TCachedQueryType = (cqtInsertWord, cqtGetWordID, cqtInsertFile, cqtGetFileID,
     cqtInsertLanguage, cqtGetLanguageID, cqtInsertMatch);
 
-// Interbase specific
-const
-  {$note @MvC, TIndexTable is defined as: itWords, itLanguages, itFiles, itMatches, below order seems to be wrong}
-  DefaultGeneratorNames: array[TIndexTable] of string = ('GEN_WORDS', 'GEN_MATCHES', 'GEN_LANGUAGES', 'GEN_FILES');
+Const
+  DefaultGeneratorNames: array[TIndexTable] of string = ('GEN_WORDS','GEN_LANGUAGES', 'GEN_FILES', 'GEN_MATCHES');
 
 type
 
@@ -38,28 +36,32 @@ type
   TSQLDBIndexDB = class(TSQLIndexDB)
   private
     // SQLDB specific
-    db: TSQLConnection;
+    FDB: TSQLConnection;
     FLastURLID: int64;
     FLastURL: string;
     FLastLanguageID: int64;
     FLastLanguage: string;
     FLastWordID: int64;
     FLastWord: string;
+    FProps : Array [0..3] of UTF8String;
     FQueries: array [TCachedQueryType] of TSQLQuery;
+    function GetS(AIndex: integer): UTF8String;
+    procedure SetS(AIndex: integer; const AValue: UTF8String);
+    Procedure EnsureDB;
   protected
     // SQLDB Specific statements
-    procedure Execute(const sql: string; ignoreErrors: boolean = True); override;
-    function GetLanguageID(const ALanguage: string): int64;
-    function GetWordID(const AWord: string): int64;
-    function GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
-    function CreateQuery(const ASQL: string): TSQLQuery;
-    function CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: string): TSQLQuery;
+    procedure Execute(const sql: UTF8string; ignoreErrors: boolean = True); override;
+    function GetLanguageID(const ALanguage: UTF8string): int64;
+    function GetWordID(const AWord: UTF8String): int64;
+    function GetURLID(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
+    function CreateQuery(const ASQL: UTF8String): TSQLQuery;
+    function CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: UTF8String): TSQLQuery;
     // Connection specific, need to be overridden
-    function GetConnection: TSQLConnection; virtual; abstract;
+    function CreateConnection: TSQLConnection; virtual; abstract;
     procedure InsertMatch(AWordID, aFileID, aLanguageID: int64; const ASearchData: TSearchWordData); virtual; abstract;
-    function InsertWord(const AWord: string): int64; virtual; abstract;
-    function InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64; virtual; abstract;
-    function InsertLanguage(const ALanguage: string): int64; virtual; abstract;
+    function InsertWord(const AWord: UTF8String): int64; virtual; abstract;
+    function InsertURL(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64): int64; virtual; abstract;
+    function InsertLanguage(const ALanguage: UTF8String): int64; virtual; abstract;
   public
     destructor Destroy; override;
     procedure Connect; override;
@@ -70,14 +72,53 @@ type
     procedure CompactDB; override;
     procedure AddSearchData(ASearchData: TSearchWordData); override;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    procedure DeleteWordsFromFile(URL: UTF8String); override;
+    Property NativeConnection : TSQLConnection Read FDB;
+  published
+    property DatabasePath: UTF8String Index 0 read GetS write SetS;
+    property UserName: UTF8String Index 1 read GetS write SetS;
+    property Password: UTF8String Index 2 read GetS write SetS;
+    property HostName : UTF8String Index 3 read GetS write SetS;
   end;
 
 implementation
 
 { TSQLDBIndexDB }
 
-function TSQLDBIndexDB.GetLanguageID(const ALanguage: string): int64;
+function TSQLDBIndexDB.GetS(AIndex: integer): UTF8String;
+begin
+  Result:=FProps[aIndex];
+end;
+
+procedure TSQLDBIndexDB.SetS(AIndex: integer; const AValue: UTF8String);
+begin
+  FProps[aIndex]:=aValue;
+  if Assigned(FDB) then
+    case Aindex of
+      0: FDB.DatabaseName := AValue;
+      1: FDB.UserName := AValue;
+      2: FDB.Password := AValue;
+      3: FDB.HostName := AValue;
+    end;
+end;
+
+procedure TSQLDBIndexDB.EnsureDB;
+begin
+  if FDB=Nil then
+    begin
+    FDB:=CreateConnection;
+    FDB.UserName:=UserName;
+    FDB.Password:=Password;
+    FDB.HostName:=HostName;
+    FDB.DatabaseName:=DatabasePath;
+    end;
+  if FDB.Transaction = nil then
+    FDB.Transaction := TSQLTransaction.Create(FDB);
+  FDB.LogEvents:=LogAllEventsExtra;
+end;
+
+
+function TSQLDBIndexDB.GetLanguageID(const ALanguage: UTF8String): int64;
 var
   Q: TSQLQuery;
 begin
@@ -101,7 +142,7 @@ begin
   end;
 end;
 
-function TSQLDBIndexDB.GetWordID(const AWord: string): int64;
+function TSQLDBIndexDB.GetWordID(const AWord: UTF8String): int64;
 var
   Q: TSQLQuery;
 begin
@@ -125,16 +166,18 @@ begin
   end;
 end;
 
-function TSQLDBIndexDB.CreateQuery(const ASQL: string): TSQLQuery;
+function TSQLDBIndexDB.CreateQuery(const ASQL: UTF8String): TSQLQuery;
 begin
   Result := TSQLQuery.Create(Self);
-  Result.Database := Self.db;
-  Result.Transaction := Self.db.Transaction;
+  Result.Database := Self.FDB;
+  Result.Transaction := Self.FDB.Transaction;
   Result.SQL.Text := ASQL;
+  Result.UsePrimaryKeyAsKey:=False;
+  Result.UniDirectional:=True;
   //Writeln('SQL  :',ASQL);
 end;
 
-function TSQLDBIndexDB.GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64;
+function TSQLDBIndexDB.GetURLID(const URL: UTF8String; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64;
 var
   Q: TSQLQuery;
 begin
@@ -165,7 +208,7 @@ begin
   end;
 end;
 
-function TSQLDBIndexDB.CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: string): TSQLQuery;
+function TSQLDBIndexDB.CreateCachedQuery(QueryType: TCachedQueryType; const ASQL: UTF8String): TSQLQuery;
 begin
   if FQueries[QueryType] = nil then
   begin
@@ -191,7 +234,7 @@ var
   Q: TSQLQuery;
   FN, FP, FD, FW, FC: TField;
   Res: TSearchWordData;
-  S,WW : String;
+  S,WW : UTF8String;
   I,L : Integer;
 
 begin
@@ -233,31 +276,32 @@ begin
   end;
 end;
 
-procedure TSQLDBIndexDB.DeleteWordsFromFile(URL: string);
+procedure TSQLDBIndexDB.DeleteWordsFromFile(URL: UTF8String);
 begin
   inherited DeleteWordsFromFile(URL);
   FLastURL := '';
 end;
 
-procedure TSQLDBIndexDB.Execute(const sql: string; ignoreErrors: boolean = True);
+procedure TSQLDBIndexDB.Execute(const sql: UTF8String; ignoreErrors: boolean = True);
 begin
   if SQL = '' then
     exit;
   try
-    DB.ExecuteDirect(sql);
+    FDB.ExecuteDirect(sql);
   except
-    if not IgnoreErrors then
-      raise;
+    on E : exception do
+      if not IgnoreErrors then
+        raise
+      else
+        Writeln(E.ClassName,' : ',E.Message);
+
   end;
 end;
 
 procedure TSQLDBIndexDB.Connect;
 begin
-  if (DB = nil) then
-    db := GetConnection;
-  if DB.Transaction = nil then
-    DB.Transaction := TSQLTransaction.Create(db);
-  DB.Connected := True;
+  EnsureDB;
+  FDB.Connected := True;
 end;
 
 procedure TSQLDBIndexDB.Disconnect;
@@ -267,14 +311,13 @@ Var
 begin
   For T:=Low(TCachedQueryType) to High(TCachedQueryType) do
     FreeAndNil(FQueries[T]);
-  FreeAndNil(DB);
+  FreeAndNil(FDB);
 end;
 
 procedure TSQLDBIndexDB.CreateDB;
 begin
-  if DB = nil then
-    DB := GetConnection;
-  DB.CreateDB;
+  EnsureDB;
+  FDB.CreateDB;
   Connect;
   CreateIndexerTables;
 end;
@@ -287,12 +330,12 @@ end;
 
 procedure TSQLDBIndexDB.BeginTrans;
 begin
-  DB.Transaction.StartTransaction;
+  FDB.Transaction.StartTransaction;
 end;
 
 procedure TSQLDBIndexDB.CommitTrans;
 begin
-  DB.Transaction.Commit;
+  FDB.Transaction.Commit;
 end;
 
 procedure TSQLDBIndexDB.CompactDB;

packages/fpindexer/src/sqliteindexdb.pp

@@ -24,7 +24,7 @@ uses
 type
   TDatabaseID = record
     ID: int64;
-    Name: string;
+    Name: UTF8string;
   end;
 
   { TSQLiteIndexDB }
@@ -32,24 +32,24 @@ type
   TSQLiteIndexDB = class(TSQLIndexDB)
   private
     db: Psqlite3;
-    FFileName: string;
+    FFileName: UTF8string;
     Frow: integer;
     FSearchClass: TFPSearch;
     LanguageID: TDatabaseID;
-    QueryResult: string;
+    QueryResult: UTF8string;
     SearchWordID: TDatabaseID;
     URLID: TDatabaseID;
     procedure CheckSQLite(Rc: cint; pzErrMsg: PChar);
   protected
     class function AllowForeignKeyInTable: boolean; override;
-    function GetFieldType(FieldType: TIndexField): string; override;
-    function GetLanguageID(const ALanguage: string): int64;
-    function GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
-    function GetWordID(const AWord: string): int64; virtual;
-    function InsertLanguage(const ALanguage: string): int64; virtual;
-    function InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
-    function InsertWord(const AWord: string): int64; virtual;
-    procedure Execute(const sql: string; ignoreErrors: boolean = True); override;
+    function GetFieldType(FieldType: TIndexField): UTF8string; override;
+    function GetLanguageID(const ALanguage: UTF8string): int64;
+    function GetURLID(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean = True): int64; override;
+    function GetWordID(const AWord: UTF8string): int64; virtual;
+    function InsertLanguage(const ALanguage: UTF8string): int64; virtual;
+    function InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+    function InsertWord(const AWord: UTF8string): int64; virtual;
+    procedure Execute(const sql: UTF8string; ignoreErrors: boolean = True); override;
   public
     destructor Destroy; override;
     procedure AddSearchData(ASearchData: TSearchWordData); override;
@@ -58,10 +58,10 @@ type
     procedure CompactDB; override;
     procedure Connect; override;
     procedure CreateDB; override;
-    procedure DeleteWordsFromFile(URL: string); override;
+    procedure DeleteWordsFromFile(URL: UTF8string); override;
     procedure FindSearchData(SearchWord: TWordParser; FPSearch: TFPSearch; SearchOptions: TSearchOptions); override;
   published
-    property FileName: string read FFileName write FFileName;
+    property FileName: UTF8string read FFileName write FFileName;
   end;
 
 implementation
@@ -99,7 +99,7 @@ end;
 
 { TSQLiteIndexDB }
 
-procedure TSQLiteIndexDB.Execute(const sql: string; ignoreErrors: boolean = True);
+procedure TSQLiteIndexDB.Execute(const sql: UTF8string; ignoreErrors: boolean = True);
 var
   pzErrMsg: PChar;
   rc: cint;
@@ -111,9 +111,9 @@ begin
     CheckSQLite(rc, pzErrMsg);
 end;
 
-function TSQLiteIndexDB.GetURLID(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean): int64;
+function TSQLiteIndexDB.GetURLID(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64; DoCreate: boolean): int64;
 var
-  SQL: string;
+  SQL: UTF8string;
 begin
   if (URL = URLID.Name) then
     Result := URLID.ID
@@ -129,9 +129,9 @@ begin
   end;
 end;
 
-function TSQLiteIndexDB.GetLanguageID(const ALanguage: string): int64;
+function TSQLiteIndexDB.GetLanguageID(const ALanguage: UTF8string): int64;
 var
-  SQL: string;
+  SQL: UTF8string;
 begin
   if (ALanguage = LanguageID.Name) then
     Result := LanguageID.ID
@@ -147,9 +147,9 @@ begin
   end;
 end;
 
-function TSQLiteIndexDB.GetWordID(const AWord: string): int64;
+function TSQLiteIndexDB.GetWordID(const AWord: UTF8string): int64;
 var
-  SQL: string;
+  SQL: UTF8string;
 begin
   if (AWord = SearchWordID.Name) then
     Result := SearchWordID.ID
@@ -165,26 +165,26 @@ begin
   end;
 end;
 
-function TSQLiteIndexDB.InsertWord(const AWord: string): int64;
+function TSQLiteIndexDB.InsertWord(const AWord: UTF8string): int64;
 begin
   Execute(Format(InsertSQL(itWords, False), ['Null', QuoteString(AWord)]), False);
   Result := sqlite3_last_insert_rowid(db);
 end;
 
-function TSQLiteIndexDB.InsertURL(const URL: string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
+function TSQLiteIndexDB.InsertURL(const URL: UTF8string; ATimeStamp: TDateTime; ALanguageID: int64): int64;
 begin
   // ifFilesID,ifFilesURL,ifFilesReindex,ifFilesUpdated,ifFilesTimeStamp,ifFilesLanguageID
   Execute(Format(InsertSQL(itFiles, False), ['Null', QuoteString(URL), '0', '0', QuoteString(DateToISO8601(ATimeStamp)), IntToStr(AlanguageID)]), False);
   Result := sqlite3_last_insert_rowid(db);
 end;
 
-function TSQLiteIndexDB.InsertLanguage(const ALanguage: string): int64;
+function TSQLiteIndexDB.InsertLanguage(const ALanguage: UTF8string): int64;
 begin
   Execute(Format(InsertSQL(itLanguages, False), ['Null', QuoteString(ALanguage)]), False);
   Result := sqlite3_last_insert_rowid(db);
 end;
 
-function TSQLiteIndexDB.GetFieldType(FieldType: TIndexField): string;
+function TSQLiteIndexDB.GetFieldType(FieldType: TIndexField): UTF8string;
 begin
   Result := inherited GetFieldType(FieldType);
   if (Result = PrimaryFieldType) then
@@ -196,7 +196,7 @@ begin
   Result := True;
 end;
 
-procedure TSQLiteIndexDB.DeleteWordsFromFile(URL: string);
+procedure TSQLiteIndexDB.DeleteWordsFromFile(URL: UTF8string);
 begin
   inherited DeleteWordsFromFile(URL);
 
@@ -247,7 +247,7 @@ end;
 procedure TSQLiteIndexDB.AddSearchData(ASearchData: TSearchWordData);
 var
   WID, LID, FID: int64;
-  SQL: string;
+  SQL: UTF8string;
 begin
   WID := GetWordID(ASearchData.SearchWord);
   LID := GetLanguageID(ASearchData.Language);
@@ -262,7 +262,7 @@ end;
 
 procedure TSQLiteIndexDB.CheckSQLite(Rc: cint; pzErrMsg: PChar);
 var
-  S: string;
+  S: UTF8string;
 begin
   if (rc <> SQLITE_OK) then
   begin
@@ -276,7 +276,7 @@ procedure TSQLiteIndexDB.FindSearchData(SearchWord: TWordParser; FPSearch: TFPSe
 var
   pzErrMsg: PChar;
   rc: cint;
-  sql: string;
+  sql: UTF8string;
 begin
   FSearchClass := FPSearch;
   Frow := 0;