Improvements on AbstractMem speed

- More speed
- Customize cache values at ini file
- Flush to disk regularly

CHANGELOG.md

@@ -1,9 +1,12 @@
 # Changelog
 
 ## Build 5.4 - (PENDING RELEASE)
-- Added "DATAFOLDER" configuration option at pascalcoin_daemon.ini file (daemon/service) in order to allow customize data folder
 - Added usage of AbstractMem library to allow build a PascalCoin version using virtual memory and efficient caching mechanism
   - Must activate {$DEFINE USE_ABSTRACTMEM} at config.inc file
+- Changes to `pascalcoin_daemon.ini` file:
+  - Added "DATAFOLDER" configuration option at pascalcoin_daemon.ini file (daemon/service) in order to allow customize data folder
+  - Added "ABSTRACTMEM_MAX_CACHE_MB" to customize Maximum megabytes in memory as a cache
+  - Added "ABSTRACTMEM_USE_CACHE_ON_LISTS","ABSTRACTMEM_CACHE_MAX_ACCOUNTS","ABSTRACTMEM_CACHE_MAX_PUBKEYS" in order to customize cache values
 - Improved performance when downloading Safebox (Fresh installation)
 - JSON-RPC changes:  
   - Updated "findaccounts": 

src/core/UAccounts.pas

@@ -320,6 +320,7 @@ Type
     procedure UpdateSafeboxFileName(const ANewSafeboxFileName : String);
     procedure ClearSafeboxfile;
     class Function CopyAbstractMemToSafeBoxStream(ASource : TPCAbstractMem; ADestStream : TStream; AFromBlock, AToBlock : Cardinal; var AErrors : String) : Boolean;
+    property PCAbstractMem : TPCAbstractMem read FPCAbstractMem;
     {$ENDIF}
   End;
 

src/core/UBlockChain.pas

@@ -1050,8 +1050,9 @@ begin
         TPCBankNotify(FNotifyList.Items[i]).NotifyNewBlock;
       end;
     end;
-
-
+    {$IFDEF USE_ABSTRACTMEM}
+    SafeBox.PCAbstractMem.FlushCache;
+    {$ENDIF}
   finally
     FBankLock.Release;
   end;
@@ -1249,12 +1250,20 @@ function TPCBank.LoadBankFromStream(Stream: TStream; useSecureLoad : Boolean; ch
 Var LastReadBlock : TBlockAccount;
   i : Integer;
   auxSB : TPCSafeBox;
+  Lucoaml : boolean;
+  Lmmu : Integer;
 begin
   auxSB := Nil;
   Try
     If useSecureLoad then begin
       // When on secure load will load Stream in a separate SafeBox, changing only real SafeBox if successfully
       auxSB := TPCSafeBox.Create;
+      {$IFDEF USE_ABSTRACTMEM}
+      Lucoaml := Self.SafeBox.PCAbstractMem.UseCacheOnAbstractMemLists;
+      Lmmu := Self.SafeBox.PCAbstractMem.MaxMemUsage;
+      auxSB.PCAbstractMem.UseCacheOnAbstractMemLists := False;
+      auxSB.PCAbstractMem.MaxMemUsage := 100 * 1024 * 1024; // 100 Mb
+      {$ENDIF}
       Result := auxSB.LoadSafeBoxFromStream(Stream,true,checkSafeboxHash,progressNotify,previousCheckedSafebox,LastReadBlock,errors);
       If Not Result then Exit;
     end;
@@ -1262,6 +1271,10 @@ begin
     try
       If Assigned(auxSB) then begin
         SafeBox.CopyFrom(auxSB);
+        {$IFDEF USE_ABSTRACTMEM}
+        Self.SafeBox.PCAbstractMem.UseCacheOnAbstractMemLists := Lucoaml;
+        Self.SafeBox.PCAbstractMem.MaxMemUsage := Lmmu;
+        {$ENDIF}
       end else begin
         Result := SafeBox.LoadSafeBoxFromStream(Stream,False,checkSafeboxHash,progressNotify,previousCheckedSafebox,LastReadBlock,errors);
       end;

src/core/UEPasa.pas

@@ -104,7 +104,7 @@ type
       function IsPayToKey: Boolean; inline;
       function GetRawPayloadBytes(): TArray<Byte>; inline;
       function ToString(): String; overload;
-      function ToString(AOmitExtendedChecksum: Boolean): String; reintroduce; overload;
+      function ToString(AOmitExtendedChecksum: Boolean): String; overload;
 
       property Account: TNullable<UInt32> read GetAccount write SetAccount;
       property AccountChecksum: TNullable<UInt32> read GetAccountChecksum write SetAccountChecksum;

src/core/UNetProtocol.pas

@@ -3038,6 +3038,10 @@ begin
         end;
         sleep(1);
       end;
+      {$IFDEF USE_ABSTRACTMEM}
+      TNode.Node.Bank.SafeBox.PCAbstractMem.FlushCache;
+      {$ENDIF}
+
       FIsDownloadingBlocks := false;
       if ((LOpCount>0) And (FRemoteOperationBlock.block>=TNode.Node.Bank.BlocksCount)) then begin
         Send_GetBlocks(TNode.Node.Bank.BlocksCount,100,c);

src/core/UNode.pas

@@ -258,6 +258,9 @@ begin
       // Does not need to save a FOperations backup because is Sanitized by "TNode.OnBankNewBlock"
       Result := Bank.AddNewBlockChainBlock(NewBlockOperations,TNetData.NetData.NetworkAdjustedTime.GetMaxAllowedTimestampForNewBlock,errors);
       if Result then begin
+        {$IFDEF USE_ABSTRACTMEM}
+        Bank.SafeBox.PCAbstractMem.FlushCache;
+        {$ENDIF}
         if Assigned(SenderConnection) then begin
           FNodeLog.NotifyNewLog(ltupdate,SenderConnection.ClassName,Format(';%d;%s;%s;;%d;%d;%d;%s',[OpBlock.block,sClientRemoteAddr,OpBlock.block_payload.ToPrintable,
             OpBlock.timestamp,UnivDateTimeToUnix(DateTime2UnivDateTime(Now)),UnivDateTimeToUnix(DateTime2UnivDateTime(Now)) - OpBlock.timestamp,IntToHex(OpBlock.compact_target,8)]));
@@ -932,7 +935,7 @@ procedure TNode.GetStoredOperationsFromAccount(AOwnerThread : TPCThread; const O
           last_block_number := block_number;
           l.Clear;
           If not Bank.Storage.LoadBlockChainBlock(opc,block_number) then begin
-            TLog.NewLog(ltdebug,ClassName,'Block '+inttostr(block_number)+' not found. Cannot read operations');
+            {$IFDEF HIGHLOG}TLog.NewLog(ltdebug,ClassName,'Block '+inttostr(block_number)+' not found. Cannot read operations');{$ENDIF}
             exit;
           end;
           opc.OperationsHashTree.GetOperationsAffectingAccount(account_number,l);

src/core/UPCAbstractMem.pas

@@ -77,7 +77,7 @@ type
     FSaveBufferPosition : TAbstractMemPosition;
   protected
   public
-    Constructor Create(AAbstractMem : TAbstractMem; APosition : TAbstractMemPosition; ACurrBlocksCount : Integer);
+    Constructor Create(AAbstractMem : TAbstractMem; APosition : TAbstractMemPosition; ACurrBlocksCount : Integer); reintroduce;
     procedure Flush;
   end;
 
@@ -90,7 +90,7 @@ type
     procedure BCExecute; override;
   public
     Constructor Create(APCAbstractMem : TPCAbstractMem);
-    Destructor Destroy;
+    Destructor Destroy; override;
     procedure Restart;
     property Errors : TStrings read FErrors;
   End;
@@ -98,6 +98,13 @@ type
   TAccountCache = Class(TAVLCache<TAccount>)
   End;
 
+  TPCAbstractMemStats = Record
+    FlushesCount : Integer;
+    FlushesMillis : TTickCount;
+    function ToString : String;
+    procedure Clear;
+  end;
+
   TPCAbstractMem = class
   private
     FFileName : String;
@@ -105,6 +112,8 @@ type
     FCheckingThread : TPCAbstractMemCheckThread;
     FLockAbstractMem : TPCCriticalSection;
 
+    FStats : TPCAbstractMemStats;
+
     FBlocks: TPCAbstractMemListBlocks;
     FAccounts: TPCAbstractMemListAccounts;
     FAccountsNames: TPCAbstractMemListAccountNames;
@@ -113,6 +122,8 @@ type
     FBufferBlocksHash: TPCAbstractMemBytesBuffer32Safebox;
     FAggregatedHashrate : TBigNum;
     FZoneAggregatedHashrate : TAMZone;
+    FUseCacheOnAbstractMemLists: Boolean;
+    FMaxMemUsage: Integer;
 
     function IsChecking : Boolean;
     procedure DoCheck;
@@ -121,6 +132,12 @@ type
     procedure AddBlockInfo(const ABlock : TOperationBlockExt);
     procedure SetBlockInfo(const ABlock : TOperationBlockExt);
     function DoInit(out AIsNewStructure : Boolean) : Boolean;
+    procedure SetMaxMemUsage(const Value: Integer);
+    procedure SetUseCacheOnAbstractMemLists(const Value: Boolean);
+    procedure SetMaxAccountsCache(const Value: Integer);
+    function GetMaxAccountsCache: Integer;
+    function GetMaxAccountKeysCache: Integer;
+    procedure SetMaxAccountKeysCache(const Value: Integer);
   protected
     procedure UpgradeAbstractMemVersion(const ACurrentHeaderVersion : Integer);
   public
@@ -156,6 +173,12 @@ type
     property AccountCache : TAccountCache read FAccountCache;
     property FileName : String read FFileName;
     procedure EraseData;
+    function GetStatsReport(AClearStats : Boolean) : String;
+    //
+    Property UseCacheOnAbstractMemLists : Boolean read FUseCacheOnAbstractMemLists write SetUseCacheOnAbstractMemLists;
+    Property MaxMemUsage : Integer read FMaxMemUsage write SetMaxMemUsage;
+    Property MaxAccountsCache : Integer read GetMaxAccountsCache write SetMaxAccountsCache;
+    Property MaxAccountKeysCache : Integer read GetMaxAccountKeysCache write SetMaxAccountKeysCache;
   end;
 
 implementation
@@ -403,6 +426,8 @@ var LIsNew : Boolean;
 begin
   ASource.FlushCache;
   FAbstractMem.CopyFrom(ASource.FAbstractMem);
+  FUseCacheOnAbstractMemLists := ASource.FUseCacheOnAbstractMemLists;
+  FMaxMemUsage := ASource.FMaxMemUsage;
   DoInit(LIsNew);
 end;
 
@@ -514,13 +539,17 @@ begin
   // Free
   FreeAndNil(FBlocks);
   //
-  FBlocks := TPCAbstractMemListBlocks.Create( FAbstractMem, LZoneBlocks, 10000 );
+  FBlocks := TPCAbstractMemListBlocks.Create( FAbstractMem, LZoneBlocks, 10000, Self.UseCacheOnAbstractMemLists);
   FBlocks.FPCAbstractMem := Self;
-  FAccounts := TPCAbstractMemListAccounts.Create( FAbstractMem, LZoneAccounts, 50000);
+
+  FAccounts := TPCAbstractMemListAccounts.Create( FAbstractMem, LZoneAccounts, 50000, Self.UseCacheOnAbstractMemLists);
   FAccounts.FPCAbstractMem := Self;
-  FAccountsNames := TPCAbstractMemListAccountNames.Create( FAbstractMem, LZoneAccountsNames, 5000 , False);
+
+  FAccountsNames := TPCAbstractMemListAccountNames.Create( FAbstractMem, LZoneAccountsNames, 5000 , False, Self.UseCacheOnAbstractMemLists);
   FAccountsNames.FPCAbstractMem := Self;
-  FAccountKeys := TPCAbstractMemAccountKeys.Create( FAbstractMem, LZoneAccountKeys.position );
+
+  FAccountKeys := TPCAbstractMemAccountKeys.Create( FAbstractMem, LZoneAccountKeys.position, Self.UseCacheOnAbstractMemLists);
+
   // Read AggregatedHashrate
   SetLength(LBuffer,100);
   FAbstractMem.Read(FZoneAggregatedHashrate.position,LBuffer[0],Length(LBuffer));
@@ -554,6 +583,17 @@ constructor TPCAbstractMem.Create(const ASafeboxFileName: string; AReadOnly: boo
 var
   LIsNewStructure : Boolean;
 begin
+  FStats.Clear;
+
+  FUseCacheOnAbstractMemLists := True;
+  FMaxMemUsage := 100 * 1024 * 1024;
+
+  FBlocks := Nil;
+  FAccounts:= Nil;
+  FAccountsNames:= Nil;
+  FAccountKeys:= Nil;
+  FBufferBlocksHash:= Nil;
+
   FCheckingThread := Nil;
   FLockAbstractMem := TPCCriticalSection.Create(Self.ClassName);
   FAccountCache := TAccountCache.Create(10000,_AccountCache_Comparision);
@@ -566,7 +606,7 @@ begin
     FAbstractMem := TMem.Create(0,AReadOnly);
   end;
   if FAbstractMem is TFileMem then begin
-    TFileMem(FAbstractMem).MaxCacheSize := 40 * 1024 * 1024; // 40Mb
+    TFileMem(FAbstractMem).MaxCacheSize := FMaxMemUsage;
     TFileMem(FAbstractMem).MaxCacheDataBlocks := 200000;
   end;
 
@@ -631,8 +671,10 @@ end;
 
 procedure TPCAbstractMem.FlushCache;
 var LBigNum : TBytes;
+  Ltc : TTickCount;
 begin
   if FAbstractMem.ReadOnly then Exit;
+  Ltc := TPlatform.GetTickCount;
   FBlocks.FlushCache;
   FAccounts.FlushCache;
   FAccountsNames.FlushCache;
@@ -643,6 +685,8 @@ begin
   if FAbstractMem is TFileMem then begin
     TFileMem(FAbstractMem).FlushCache;
   end;
+  Inc(FStats.FlushesCount);
+  Inc(Fstats.FlushesMillis, TPlatform.GetElapsedMilliseconds(Ltc) );
 end;
 
 Procedure DoCopyFile(const ASource, ADest : String);
@@ -773,6 +817,33 @@ begin
   end else raise EPCAbstractMem.Create(Format('Cannot set block info %d (current %d blocks)',[ABlock.operationBlock.block,LCount]));
 end;
 
+procedure TPCAbstractMem.SetMaxAccountKeysCache(const Value: Integer);
+begin
+  FAccountKeys.AccountKeyByPositionCache.MaxRegisters := Value;
+end;
+
+procedure TPCAbstractMem.SetMaxAccountsCache(const Value: Integer);
+begin
+  FAccountCache.MaxRegisters := Value;
+end;
+
+procedure TPCAbstractMem.SetMaxMemUsage(const Value: Integer);
+begin
+  FMaxMemUsage := Value;
+  if FAbstractMem is TFileMem then begin
+    TFileMem(FAbstractMem).MaxCacheSize := FMaxMemUsage;
+    TFileMem(FAbstractMem).MaxCacheDataBlocks := 200000;
+  end;
+end;
+
+procedure TPCAbstractMem.SetUseCacheOnAbstractMemLists(const Value: Boolean);
+var Lins : Boolean;
+begin
+  if Value=FUseCacheOnAbstractMemLists then Exit;
+  FUseCacheOnAbstractMemLists := Value;
+  DoInit(Lins);
+end;
+
 procedure TPCAbstractMem.UpdateSafeboxFileName(const ANewSafeboxFileName: String);
 var LReadOnly, Ltmp : Boolean;
 begin
@@ -791,7 +862,7 @@ begin
     FAbstractMem := TMem.Create(0,LReadOnly);
   end;
   if FAbstractMem is TFileMem then begin
-    TFileMem(FAbstractMem).MaxCacheSize := 40 * 1024 * 1024; // 40Mb
+    TFileMem(FAbstractMem).MaxCacheSize := FMaxMemUsage;
     TFileMem(FAbstractMem).MaxCacheDataBlocks := 200000;
   end;
   DoInit(Ltmp);
@@ -864,6 +935,22 @@ begin
   Result := FBlocks.GetItem( ABlockNumber );
 end;
 
+function TPCAbstractMem.GetMaxAccountKeysCache: Integer;
+begin
+  Result := FAccountKeys.AccountKeyByPositionCache.MaxRegisters;
+end;
+
+function TPCAbstractMem.GetMaxAccountsCache: Integer;
+begin
+  Result := FAccountCache.MaxRegisters;
+end;
+
+function TPCAbstractMem.GetStatsReport(AClearStats: Boolean): String;
+begin
+  Result := AbstractMem.GetStatsReport(AClearStats) + #10 + FStats.ToString;
+  if AClearStats then FStats.Clear;
+end;
+
 function TPCAbstractMem.IsChecking: Boolean;
 begin
   Result := Assigned(TPCThread.GetThreadByClass(TPCAbstractMemCheckThread,Nil));
@@ -1168,6 +1255,7 @@ begin
     FPCAbstractMem.FLockAbstractMem.Release;
   end;
   FErrors.Free;
+  inherited Destroy;
 end;
 
 procedure TPCAbstractMemCheckThread.Restart;
@@ -1177,4 +1265,17 @@ begin
 end;
 
 
+{ TPCAbstractMemStats }
+
+procedure TPCAbstractMemStats.Clear;
+begin
+  Self.FlushesCount := 0;
+  Self.FlushesMillis := 0;
+end;
+
+function TPCAbstractMemStats.ToString: String;
+begin
+  Result := Format('PCAbstractMem flushes:%d in %d millis',[Self.FlushesCount,Self.FlushesMillis]);
+end;
+
 end.

src/core/UPCAbstractMemAccountKeys.pas

@@ -38,7 +38,7 @@ type
     procedure SaveTo(const AItem : Cardinal; AIsAddingItem : Boolean; var ABytes : TBytes); override;
     function Compare(const ALeft, ARight : Cardinal) : Integer; override;
   public
-    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone); reintroduce;
+    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; AUseCache : Boolean); reintroduce;
     Function Add(const AItem : Cardinal) : Integer; reintroduce;
     procedure Delete(index : Integer); reintroduce;
   End;
@@ -64,6 +64,7 @@ type
     FPointerToRootPosition : TAbstractMemPosition;
     FRootPosition : TAbstractMemPosition;
     FAccountKeyByPositionCache : TPCAccountKeyByPositionCache;
+    FUseCacheOnAbstractMemLists: Boolean;
   protected
     function GetRoot: TAbstractMemAccountKeyNode; override;
     procedure SetRoot(const Value: TAbstractMemAccountKeyNode); override;
@@ -79,7 +80,7 @@ type
   public
     function IsNil(const ANode : TAbstractMemAccountKeyNode) : Boolean; override;
     function ToString(const ANode: TAbstractMemAccountKeyNode) : String; override;
-    constructor Create(AAbstractMem : TAbstractMem; APointerToRootPosition : TAbstractMemPosition); reintroduce;
+    constructor Create(AAbstractMem : TAbstractMem; APointerToRootPosition : TAbstractMemPosition; AUseCacheOnAbstractMemLists : Boolean); reintroduce;
     destructor Destroy; override;
     //
     function GetKeyAtPosition(APosition : TAbstractMemPosition) : TAccountKey;
@@ -89,6 +90,8 @@ type
     procedure GetAccountsUsingKey(const AAccountKey : TAccountKey; const AList : TList<Cardinal>);
     function GetAccountsUsingThisKey(const AAccountKey : TAccountKey) : TAccountsUsingThisKey;
     procedure FlushCache;
+    property UseCacheOnAbstractMemLists : Boolean read FUseCacheOnAbstractMemLists write FUseCacheOnAbstractMemLists;
+    property AccountKeyByPositionCache : TPCAccountKeyByPositionCache read FAccountKeyByPositionCache;
   end;
 
 
@@ -113,7 +116,7 @@ begin
       _BlackHoleAbstractMem := TMem.Create(0,True);
     end;
     LZone.Clear;
-    _TAccountsUsingThisKey_BlackHole := TAccountsUsingThisKey_BlackHole.Create(_BlackHoleAbstractMem,LZone);
+    _TAccountsUsingThisKey_BlackHole := TAccountsUsingThisKey_BlackHole.Create(_BlackHoleAbstractMem,LZone,True);
   end;
   Result :=  _TAccountsUsingThisKey_BlackHole;
 end;
@@ -225,12 +228,13 @@ begin
   Result := Left.data.position - Right.data.position;
 end;
 
-constructor TPCAbstractMemAccountKeys.Create(AAbstractMem: TAbstractMem; APointerToRootPosition : TAbstractMemPosition);
+constructor TPCAbstractMemAccountKeys.Create(AAbstractMem: TAbstractMem; APointerToRootPosition : TAbstractMemPosition; AUseCacheOnAbstractMemLists : Boolean);
 begin
   FAccountKeysLock := TCriticalSection.Create;
   FAbstractMem := AAbstractMem;
   FPointerToRootPosition := APointerToRootPosition;
   FRootPosition := 0;
+  FUseCacheOnAbstractMemLists := AUseCacheOnAbstractMemLists;
   // Read Root position
   FAbstractMem.Read(FPointerToRootPosition,FRootPosition,4);
   FAccountKeyByPositionCache := TPCAccountKeyByPositionCache.Create(5000,_AccountKeyByPositionCache_Comparision);
@@ -239,8 +243,8 @@ end;
 
 destructor TPCAbstractMemAccountKeys.Destroy;
 begin
-  FAccountKeyByPositionCache.Free;
-  FAccountKeysLock.Free;
+  FreeAndNil(FAccountKeyByPositionCache);
+  FreeAndNil(FAccountKeysLock);
   inherited;
 end;
 
@@ -297,7 +301,7 @@ begin
     LP.Clear;
     LP.position := LNode.myPosition;
     LP.accountKey := AAccountKey;
-    LP.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LZone);
+    LP.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LZone,Self.UseCacheOnAbstractMemLists);
     FAccountKeyByPositionCache.Add(LP); // Add to cache!
   end;
   Result := LP.accountsUsingThisKey;
@@ -330,7 +334,7 @@ begin
     if LNode.accounts_using_this_key_position>0 then begin
       LAccZone.Clear;
       LAccZone.position := LNode.accounts_using_this_key_position;
-      LP.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LAccZone);
+      LP.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LAccZone,Self.UseCacheOnAbstractMemLists);
     end else LP.accountsUsingThisKey := Nil;
     FAccountKeyByPositionCache.Add(LP); // Add to cache!
   end;
@@ -392,7 +396,7 @@ begin
       LNode.accounts_using_this_key_position := LZone.position;
       LNode.WriteToMem( FAbstractMem ); // Save update:
     end else LZone.position := LNode.accounts_using_this_key_position;
-    LAccKeyByPos.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LZone);
+    LAccKeyByPos.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LZone,Self.UseCacheOnAbstractMemLists);
     // Add to cache
     FAccountKeyByPositionCache.Add( LAccKeyByPos );
   end;
@@ -435,7 +439,7 @@ begin
     LAccKeyByPos.accountKey := AAccountKey;
     LZone.Clear;
     LZone.position := LNode.accounts_using_this_key_position;
-    LAccKeyByPos.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LZone);
+    LAccKeyByPos.accountsUsingThisKey := TAccountsUsingThisKey.Create(FAbstractMem,LZone,Self.UseCacheOnAbstractMemLists);
     // Add to cache
     FAccountKeyByPositionCache.Add( LAccKeyByPos );
   end;
@@ -554,9 +558,9 @@ begin
   Result := ALeft - ARight;
 end;
 
-constructor TAccountsUsingThisKey.Create(AAbstractMem: TAbstractMem; const AInitialZone: TAMZone);
+constructor TAccountsUsingThisKey.Create(AAbstractMem: TAbstractMem; const AInitialZone: TAMZone; AUseCache : Boolean);
 begin
-  inherited Create(AAbstractMem,AInitialZone,1000,False);
+  inherited Create(AAbstractMem,AInitialZone,1000,False, AUseCache);
 end;
 
 procedure TAccountsUsingThisKey.Delete(index: Integer);

src/core/UThread.pas

@@ -81,6 +81,7 @@ Type
     constructor Create(CreateSuspended: Boolean);
     destructor Destroy; override;
     Property DebugStep : String read FDebugStep write FDebugStep;
+    Property StartTickCount : TTickCount read FStartTickCount;
     property Terminated;
   End;
 

src/core/upcdaemon.pas

@@ -1,6 +1,6 @@
 unit upcdaemon;
 
-{ Copyright (c) 2016 by Albert Molina
+{ Copyright (c) 2016-2020 by Albert Molina
 
   Distributed under the MIT software license, see the accompanying file LICENSE
   or visit http://www.opensource.org/licenses/mit-license.php.
@@ -48,6 +48,12 @@ Const
   CT_INI_IDENT_MINPENDINGBLOCKSTODOWNLOADCHECKPOINT = 'MINPENDINGBLOCKSTODOWNLOADCHECKPOINT';
   CT_INI_IDENT_PEERCACHE = 'PEERCACHE';
   CT_INI_IDENT_DATA_FOLDER = 'DATAFOLDER';
+  {$IFDEF USE_ABSTRACTMEM}
+  CT_INI_IDENT_ABSTRACTMEM_MAX_CACHE_MB = 'ABSTRACTMEM_MAX_CACHE_MB';
+  CT_INI_IDENT_ABSTRACTMEM_USE_CACHE_ON_LISTS = 'ABSTRACTMEM_USE_CACHE_ON_LISTS';
+  CT_INI_IDENT_ABSTRACTMEM_CACHE_MAX_ACCOUNTS = 'ABSTRACTMEM_CACHE_MAX_ACCOUNTS';
+  CT_INI_IDENT_ABSTRACTMEM_CACHE_MAX_PUBKEYS = 'ABSTRACTMEM_CACHE_MAX_PUBKEYS';
+  {$ENDIF}
 
 Type
   { TPCDaemonThread }
@@ -238,7 +244,11 @@ var
       TLog.NewLog(ltinfo,ClassName,'RPC Miner Server NOT ACTIVE (Ini file is '+CT_INI_IDENT_RPC_SERVERMINER_PORT+'=0)');
     end;
   end;
-
+  {$IFDEF USE_ABSTRACTMEM}
+  var LMaxMemMb : Integer;
+    LUseCacheOnMemLists : Boolean;
+    LCacheMaxAccounts, LCacheMaxPubKeys : Integer;
+  {$ENDIF}
 begin
   FMInerServer := Nil;
   TLog.NewLog(ltinfo,Classname,'START PascalCoin Server');
@@ -254,6 +264,18 @@ begin
       FWalletKeys.WalletFileName := GetDataFolder+PathDelim+'WalletKeys.dat';
       // Creating Node:
       FNode := TNode.Node;
+      {$IFDEF USE_ABSTRACTMEM}
+      LMaxMemMb := FIniFile.ReadInteger(CT_INI_SECTION_GLOBAL,CT_INI_IDENT_ABSTRACTMEM_MAX_CACHE_MB,100);
+      LUseCacheOnMemLists:= FIniFile.ReadBool(CT_INI_SECTION_GLOBAL,CT_INI_IDENT_ABSTRACTMEM_USE_CACHE_ON_LISTS,True);
+      LCacheMaxAccounts := FIniFile.ReadInteger(CT_INI_SECTION_GLOBAL,CT_INI_IDENT_ABSTRACTMEM_CACHE_MAX_ACCOUNTS,10000);
+      LCacheMaxPubKeys := FIniFile.ReadInteger(CT_INI_SECTION_GLOBAL,CT_INI_IDENT_ABSTRACTMEM_CACHE_MAX_PUBKEYS,5000);
+
+      TLog.NewLog(ltinfo,ClassName,Format('Init abstract mem library to %d mb %d accounts and %d Pubkeys and use cache lists: %s',[LMaxMemMb,LCacheMaxAccounts,LCacheMaxPubKeys,LUseCacheOnMemLists.ToString]));
+      FNode.Bank.SafeBox.PCAbstractMem.MaxMemUsage := LMaxMemMb * 1024 * 1024;
+      FNode.Bank.SafeBox.PCAbstractMem.UseCacheOnAbstractMemLists := LUseCacheOnMemLists;
+      FNode.Bank.SafeBox.PCAbstractMem.MaxAccountsCache := LCacheMaxAccounts;
+      FNode.Bank.SafeBox.PCAbstractMem.MaxAccountKeysCache := LCacheMaxPubKeys;
+      {$ENDIF}
       {$IFDEF TESTNET}
       TPCTNetDataExtraMessages.InitNetDataExtraMessages(FNode,TNetData.NetData,FWalletKeys);
       {$ENDIF}

src/libraries/abstractmem/UAVLCache.pas

@@ -87,13 +87,13 @@ type
       function ConsistencyCheck(const AErrors : TStrings): integer; override;
     end;
     var FAVLCacheMem : TAVLCacheMem;
-    FDefaultMax : Integer;
+    FMaxRegisters : Integer;
     FAVLCacheLock : TCriticalSection;
   protected
     procedure BeforeDelete(var AData : T); virtual;
     procedure ConsistencyCheck;
   public
-    Constructor Create(ADefaultMax : Integer; const AOnCompareMethod: TComparison<PAVLCacheMemData>);
+    Constructor Create(ADefaultMaxRegisters : Integer; const AOnCompareMethod: TComparison<PAVLCacheMemData>);
     Destructor Destroy; override;
     //
     function Find(const AData : T; out AFound : T) : Boolean;
@@ -103,6 +103,7 @@ type
     procedure Clear;
     function TreeToString: String;
     function ToString(const AData : T) : String; overload; virtual;
+    property MaxRegisters : Integer read FMaxRegisters write FMaxRegisters;
   End;
 
 implementation
@@ -339,7 +340,7 @@ begin
   P^.data := AData;
   FAVLCacheMem.Add(P);
   FAVLCacheMem.DoMark(P,True);
-  if (FDefaultMax > 0) And (FAVLCacheMem.FCount>FDefaultMax) then begin
+  if (FMaxRegisters > 0) And (FAVLCacheMem.FCount>FMaxRegisters) then begin
     // Dispose cache
     LnToRemove := FAVLCacheMem.FCount SHR 1;
     i := 1;
@@ -395,10 +396,10 @@ begin
   End;
 end;
 
-constructor TAVLCache<T>.Create(ADefaultMax: Integer;  const AOnCompareMethod: TComparison<PAVLCacheMemData>);
+constructor TAVLCache<T>.Create(ADefaultMaxRegisters: Integer;  const AOnCompareMethod: TComparison<PAVLCacheMemData>);
 begin
   FAVLCacheMem := TAVLCacheMem.Create(AOnCompareMethod,False);
-  FDefaultMax := ADefaultMax;
+  FMaxRegisters := ADefaultMaxRegisters;
   FAVLCacheLock := TCriticalSection.Create;
 end;
 

src/libraries/abstractmem/UAbstractMemTList.pas

@@ -59,33 +59,34 @@ type
     FUseCache : Boolean;
     FCacheData : TBytes;
     FCacheUpdated : Boolean;
+    FCacheDataLoaded : Boolean;
 
     function GetPosition(AIndex: Integer): TAbstractMemPosition;
     procedure SetPosition(AIndex: Integer; const Value: TAbstractMemPosition);
 
+    function UseCacheData : Boolean;
     Procedure CheckInitialized;
     procedure GetPointerTo(AIndex : Integer; AAllowIncrease : Boolean; out APreviousBlockPointer, ABlockPointer : TAbstractMemPosition; out AIndexInBlock : Integer);
     procedure AddRange(AIndexStart, AInsertCount : Integer);
     procedure RemoveRange(AIndexStart, ARemoveCount : Integer);
     procedure LoadElements(AIndexStart : Integer; var AElements : TBytes);
     procedure SetUseCache(const Value: Boolean);
+    procedure Initialize(const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer);
   protected
     FAbstractMemTListLock : TCriticalSection;
   public
-    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer); virtual;
+    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer; AUseCache : Boolean); virtual;
     destructor Destroy; override;
 
     procedure FlushCache;
 
-    procedure Initialize(const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer);
-
-    Function Add(const APosition : TAbstractMemPosition) : Integer; //virtual;
+    Function Add(const APosition : TAbstractMemPosition) : Integer;
 
-    Procedure Clear; //virtual;
+    Procedure Clear;
     Procedure Dispose;
 
-    Procedure Delete(index : Integer); //virtual;
-    Procedure Insert(AIndex : Integer; const APosition : TAbstractMemPosition); //virtual;
+    Procedure Delete(index : Integer);
+    Procedure Insert(AIndex : Integer; const APosition : TAbstractMemPosition);
 
     property Position[AIndex : Integer] : TAbstractMemPosition read GetPosition write SetPosition;
 
@@ -101,6 +102,8 @@ type
   private
     FAbstractMem: TAbstractMem;
     function GetInitialZone: TAMZone;
+    function GetUseCache : Boolean;
+    procedure SetUseCache(const Value: Boolean);
   protected
     FList : TAbstractMemTList;
     // POSSIBLE OVERRIDE METHODS
@@ -111,7 +114,7 @@ type
     procedure LoadFrom(const ABytes : TBytes; var AItem : T); virtual; abstract;
     procedure SaveTo(const AItem : T; AIsAddingItem : Boolean; var ABytes : TBytes); virtual; abstract;
   public
-    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer); virtual;
+    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer; AUseCache : Boolean); virtual;
     Destructor Destroy; override;
 
     Function Add(const AItem : T) : Integer; virtual;
@@ -124,6 +127,7 @@ type
     Procedure Dispose;
     property AbstractMem : TAbstractMem read FAbstractMem;
     property InitialiZone : TAMZone read GetInitialZone;
+    property UseCache : Boolean read GetUseCache write SetUseCache;
   End;
 
 
@@ -141,7 +145,7 @@ type
     // ABSTRACT METHODS NEED TO OVERRIDE
     function Compare(const ALeft, ARight : T) : Integer; virtual; abstract;
   public
-    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer; AAllowDuplicates : Boolean); reintroduce;
+    Constructor Create(AAbstractMem : TAbstractMem; const AInitialZone : TAMZone; ADefaultElementsPerBlock : Integer; AAllowDuplicates, AUseCache : Boolean); reintroduce;
     function Find(const AItemToFind : T; out AIndex : Integer) : Boolean;
     Function Add(const AItem : T) : Integer; reintroduce;
     property Item[index : Integer] : T read GetItem;
@@ -182,7 +186,7 @@ var LElements : TBytes;
 begin
   CheckInitialized;
   if (AIndexStart<0) or (AInsertCount<=0) or (AIndexStart>FNextElementPosition) then raise EAbstractMemTList.Create(Format('%s AddRange %d..%d out of range 0..%d',[ClassName,AIndexStart,AIndexStart+AInsertCount,FNextElementPosition-1]));
-  if (FUseCache) then begin
+  if (UseCacheData) then begin
     FCacheUpdated := True;
     SetLength(FCacheData,Length(FCacheData)+(AInsertCount*4));
     Move(FCacheData[AIndexStart*4],FCacheData[(AIndexStart+AInsertCount)*4],Length(FCacheData)-((AIndexStart+AInsertCount)*4));
@@ -243,11 +247,12 @@ begin
   Result := FNextElementPosition;
 end;
 
-constructor TAbstractMemTList.Create(AAbstractMem: TAbstractMem; const AInitialZone: TAMZone; ADefaultElementsPerBlock : Integer);
+constructor TAbstractMemTList.Create(AAbstractMem: TAbstractMem; const AInitialZone: TAMZone; ADefaultElementsPerBlock : Integer; AUseCache : Boolean);
 begin
   SetLength(FCacheData,0);
-  FUseCache := True;
+  FUseCache := AUseCache;
   FCacheUpdated := False;
+  FCacheDataLoaded := False;
 
   FAbstractMem := AAbstractMem;
   FInitialZone.Clear;
@@ -393,7 +398,7 @@ begin
   Result := 0;
   FAbstractMemTListLock.Acquire;
   try
-  if FUseCache then begin
+  if (UseCacheData) then begin
     if (AIndex<0) or (AIndex>=FNextElementPosition) then raise EAbstractMemTList.Create(Format('%s index %d out of range 0..%d',[ClassName,AIndex,FNextElementPosition-1]));
     Move( FCacheData[AIndex*4], Result, 4);
   end else begin
@@ -450,12 +455,6 @@ begin
       FAbstractMem.Write( FInitialZone.position, LBytes[0], Length(LBytes) );
     end;
   end;
-  if (FUseCache) then begin
-    if (FElementsOfEachBlock>0) then begin
-      LoadElements(0,FCacheData);
-    end;
-    FCacheUpdated := False;
-  end;
 end;
 
 procedure TAbstractMemTList.Insert(AIndex: Integer; const APosition: TAbstractMemPosition);
@@ -465,7 +464,7 @@ begin
   FAbstractMemTListLock.Acquire;
   try
   AddRange(AIndex,1);
-  if FUseCache then begin
+  if (UseCacheData) then begin
     Move(APosition, FCacheData[AIndex*4], 4);
     FCacheUpdated := True;
   end else begin
@@ -519,7 +518,7 @@ begin
     else raise EAbstractMemTList.Create(Format('%s remove %d..%d out of range (NO ELEMENTS)',[ClassName,AIndexStart,AIndexStart + ARemoveCount -1]))
   end;
 
-  if FUseCache then begin
+  if (UseCacheData) then begin
     if (AIndexStart+ARemoveCount < FNextElementPosition) then begin
       Move(FCacheData[(AIndexStart + ARemoveCount) *4],
            FCacheData[(AIndexStart) *4],
@@ -584,7 +583,7 @@ var LBlockPointer, LPreviousBlockPointer : TAbstractMemPosition;
 begin
   FAbstractMemTListLock.Acquire;
   try
-  if FUseCache then begin
+  if (UseCacheData) then begin
     Move( Value, FCacheData[AIndex*4], 4);
     FCacheUpdated := True;
   end else begin
@@ -603,7 +602,8 @@ begin
     FlushCache;
     SetLength(FCacheData,0);
   end else begin
-    LoadElements(0,FCacheData);
+    SetLength(FCacheData,0);
+    FCacheDataLoaded := False;
     FCacheUpdated := False;
   end;
   FUseCache := Value;
@@ -614,6 +614,17 @@ begin
   FAbstractMemTListLock.Release;
 end;
 
+function TAbstractMemTList.UseCacheData: Boolean;
+begin
+  if FUseCache then begin
+    Result := True;
+    if Not FCacheDataLoaded then begin
+      FCacheDataLoaded := True;
+      LoadElements(0,FCacheData);
+    end;
+  end else Result := False;
+end;
+
 { TAbstractMemTListBaseAbstract<T> }
 
 function TAbstractMemTListBaseAbstract<T>.Add(const AItem: T): Integer;
@@ -657,10 +668,10 @@ begin
 end;
 
 constructor TAbstractMemTListBaseAbstract<T>.Create(AAbstractMem: TAbstractMem;
-  const AInitialZone: TAMZone; ADefaultElementsPerBlock: Integer);
+  const AInitialZone: TAMZone; ADefaultElementsPerBlock: Integer; AUseCache : Boolean);
 begin
   FAbstractMem := AAbstractMem;
-  FList := TAbstractMemTList.Create(AAbstractMem,AInitialZone,ADefaultElementsPerBlock);
+  FList := TAbstractMemTList.Create(AAbstractMem,AInitialZone,ADefaultElementsPerBlock,AUseCache);
 end;
 
 procedure TAbstractMemTListBaseAbstract<T>.Delete(index: Integer);
@@ -720,6 +731,16 @@ begin
   end;
 end;
 
+function TAbstractMemTListBaseAbstract<T>.GetUseCache: Boolean;
+begin
+  FList.LockList;
+  try
+    Result := FList.UseCache;
+  finally
+    FList.UnlockList;
+  end;
+end;
+
 procedure TAbstractMemTListBaseAbstract<T>.SetItem(index: Integer;
   const AItem: T);
 var
@@ -765,6 +786,16 @@ begin
   end;
 end;
 
+procedure TAbstractMemTListBaseAbstract<T>.SetUseCache(const Value: Boolean);
+begin
+  FList.LockList;
+  try
+    FList.UseCache := Value;
+  finally
+    FList.UnlockList;
+  end;
+end;
+
 function TAbstractMemTListBaseAbstract<T>.ToString(const AItem: T): String;
 begin
   Result := Self.ClassName+'.T '+IntToStr(SizeOf(AItem));
@@ -797,9 +828,9 @@ end;
 
 constructor TAbstractMemOrderedTList<T>.Create(AAbstractMem: TAbstractMem;
   const AInitialZone: TAMZone; ADefaultElementsPerBlock: Integer;
-  AAllowDuplicates: Boolean);
+  AAllowDuplicates, AUseCache: Boolean);
 begin
-  inherited Create(AAbstractMem, AInitialZone, ADefaultElementsPerBlock);
+  inherited Create(AAbstractMem, AInitialZone, ADefaultElementsPerBlock, AUseCache);
   FAllowDuplicates := AAllowDuplicates;
 end;
 

src/pascalcoin_daemon.ini

@@ -1,13 +1,13 @@
 [GLOBAL]
 ;SAVELOGS : Boolean
 ;If 1 (true) logs will be saved to a file at $HOME/PascalCoin
-SAVELOGS=0
+SAVELOGS=1
 ;NODE_PORT : Integer (Default 4004)
 ;Port P2P of PascalCoin
 NODE_PORT=4004
 ;NODE_MAX_CONNECTIONS : Integer (Default 100)
 ;Max node connections P2P
-NODE_MAX_CONNECTIONS=100
+NODE_MAX_CONNECTIONS=200
 ;LOWMEMORY : Boolean
 ;If True, will read/write directly to file storage, using less memory but decreasing speed
 LOWMEMORY=0
@@ -49,3 +49,17 @@ RPC_SERVERMINER_MAX_ZERO_FEE_OPERATIONS=
 ;Allow define folder to store data of PascalCoin (Blockchain, Safebox, WalletKeys file, Temporal files ...)
 ;If empty will use default folder $HOME/PascalCoin (Each OS will assigna a different $HOME folder, AppData for Windows...)
 DATAFOLDER=
+
+;ABSTRACTMEM CACHE VALUES
+;ABSTRACTMEM_MAX_CACHE_MB : Integer
+;Maximum megabytes in memory as a cache - Default 100
+ABSTRACTMEM_MAX_CACHE_MB=
+;ABSTRACTMEM_USE_CACHE_ON_LISTS : Boolean
+;Set to true (1) to allow save lists on cache or false (0) - Default TRUE
+ABSTRACTMEM_USE_CACHE_ON_LISTS=
+;ABSTRACTMEM_CACHE_MAX_ACCOUNTS : Integer
+;Max number of accounts to store at cache - Default 10000
+ABSTRACTMEM_CACHE_MAX_ACCOUNTS=
+;ABSTRACTMEM_CACHE_MAX_PUBKEYS : Integer
+;Max number of public keys to store at cache - Default 5000
+ABSTRACTMEM_CACHE_MAX_PUBKEYS=

src/pascalcoin_daemon.lpi

@@ -53,7 +53,7 @@
     </Target>
     <SearchPaths>
       <IncludeFiles Value="$(ProjOutDir)"/>
-      <OtherUnitFiles Value="core;libraries\synapse;libraries\pascalcoin;libraries\generics.collections;libraries\sphere10;libraries\hashlib4pascal;libraries\paszlib;libraries\cryptolib4pascal;libraries\simplebaselib4pascal;libraries\abstractmem"/>
+      <OtherUnitFiles Value="core;libraries\synapse;libraries\pascalcoin;libraries\generics.collections;libraries\sphere10;libraries\hashlib4pascal;libraries\paszlib;libraries\cryptolib4pascal;libraries\simplebaselib4pascal;libraries\abstractmem;libraries\regex"/>
       <UnitOutputDirectory Value="lib\$(TargetCPU)-$(TargetOS)"/>
     </SearchPaths>
     <CodeGeneration>