Memory improvements and upgrade to v5.7.1

src/config.inc

@@ -141,3 +141,20 @@ ERROR: You must select ONLY ONE option: Use_OpenSSL or Use_CryptoLib4Pascal
 {$ENDIF}
 {$ENDIF}
 
+  {$IFDEF FPC}
+    {$IFDEF CPU32}
+      {$DEFINE IS32BITS}
+      {$UNDEF IS64BITS}
+    {$ELSE}
+      {$UNDEF IS32BITS}
+      {$DEFINE IS64BITS}
+    {$ENDIF}
+  {ELSE}
+    {$IFDEF CPU32BITS}
+      {$DEFINE IS32BITS}
+      {$UNDEF IS64BITS}
+    {$ELSE}
+      {$UNDEF IS32BITS}
+      {$DEFINE IS64BITS}
+    {$ENDIF}
+  {$ENDIF}

src/core/UAbstractMemBlockchainStorage.pas

@@ -215,6 +215,7 @@ type
     Function DoSaveBlockChainExt(Operations : TPCOperationsComp; const AOrphan : String; var AStats : TBlockchainStorageStats) : Boolean;
     Function DoLoadBlockChainExt(Operations : TPCOperationsComp; Block : Cardinal; const AOrphan : String) : Boolean;
     procedure AddMessage(AMessages : TStrings; const AMessage : String; ARaiseAnException : Boolean);
+    procedure OnCacheMemFlushedCache(const ASender : TCacheMem; const AProcessDesc : String; AElapsedMilis: Int64);
   protected
     procedure SetReadOnly(const Value: Boolean); override;
     Function DoGetBlockInformation(const ABlock : Integer; var AOperationBlock : TOperationBlock; var AOperationsCount : Integer; var AVolume : Int64) : Boolean; override;
@@ -230,8 +231,8 @@ type
     Procedure DoEraseStorage; override;
     function CheckBlockConsistency(ARaiseOnError : Boolean; AMessages : TStrings; const ABlockInformation : TBlockInformation; out AOperationsCount, AAffectedAccountsCount : Integer; AThread : TPCThread) : Boolean;
 
-    procedure DoBlockNotFound(ABlock : Integer); virtual;
-    procedure BlockNotFound(ABlock : Integer);
+    procedure DoBlockNotFound(ABlock : Integer; var AFound : Boolean); virtual;
+    procedure BlockNotFound(ABlock : Integer; var AFound : Boolean);
 
     Function DoGetBlockOperations(ABlock, AOpBlockStartIndex, AMaxOperations : Integer; var AOperationBlock : TOperationBlock; var AOperationsCount : Integer; var AVolume : Int64; const AOperationsResumeList:TOperationsResumeList) : Boolean; override;
     Function DoGetAccountOperations(AAccount : Integer; AMaxDepth, AStartOperation, AMaxOperations, ASearchBackwardsStartingAtBlock: Integer; const AOperationsResumeList:TOperationsResumeList): Boolean; override;
@@ -269,7 +270,7 @@ type
     Function DoMoveBlockChain(StartBlock : Cardinal; Const DestOrphan : TOrphan) : Boolean; override;
     Procedure DoDeleteBlockChainBlocks(StartingDeleteBlock : Cardinal); override;
     function DoInitialize : Boolean; override;
-    procedure DoBlockNotFound(ABlock : Integer); override;
+    procedure DoBlockNotFound(ABlock : Integer; var AFound : Boolean); override;
   public
     Constructor Create(AOwner : TComponent); Override;
     Destructor Destroy; Override;
@@ -329,8 +330,9 @@ begin
   FCheckingConsistencyProgress := AMessage;
 end;
 
-procedure TAbstractMemBlockchainStorage.BlockNotFound(ABlock: Integer);
+procedure TAbstractMemBlockchainStorage.BlockNotFound(ABlock: Integer; var AFound : Boolean);
 begin
+  AFound := False;
   if FInBlockNotFound then begin
     TLog.NewLog(ltdebug,ClassName,Format('BlockNotFound cannot save Block:%d because saving block:%d',[ABlock,FInBlockSaving]));
     Exit;
@@ -338,7 +340,7 @@ begin
   FInBlockNotFound := True;
   try
     FInBlockSaving := ABlock;
-    DoBlockNotFound(ABlock);
+    DoBlockNotFound(ABlock,AFound);
   finally
     FInBlockNotFound := False;
   end;
@@ -621,10 +623,10 @@ begin
 
   if (OrphanCompare(AOrphan,Self.Orphan)=0) then begin
     if (FOperationsRawData_By_RightOpHash.Count<>AOperationsFound) then begin
-      AddInfo(True,Format('Found %d operations but stored %d operations',[AOperationsFound,FOperationsRawData_By_RightOpHash.Count]));
+      AddInfo(False,Format('Stored %d operations but only %d on processed blocks',[FOperationsRawData_By_RightOpHash.Count,AOperationsFound]));
     end;
     if FAffectedAccounts_By_Account_Block_OpBlock.Count<>AAffectedAccountsFound then begin
-      AddInfo(True,Format('Stored %d affected accounts but only %d on blocks',[FAffectedAccounts_By_Account_Block_OpBlock.Count,AAffectedAccountsFound]));
+      AddInfo(False,Format('Stored %d affected accounts but only %d on processed blocks',[FAffectedAccounts_By_Account_Block_OpBlock.Count,AAffectedAccountsFound]));
     end;
   end;
   LOrphanInformation.Clear;
@@ -758,7 +760,9 @@ function TAbstractMemBlockchainStorage.DoBlockExists(Block: Cardinal): Boolean;
 var LFoundBlock : TBlockInformation;
 begin
   Result := DoBlockExistsByOrphan(Block,Orphan,LFoundBlock);
-  if Not Result then BlockNotFound(Block);
+  if Not Result then begin
+    BlockNotFound(Block,Result);
+  end;
 end;
 
 function TAbstractMemBlockchainStorage.DoBlockExistsByOrphan(ABlock: Integer;
@@ -772,9 +776,10 @@ begin
   Result := FBlocksInformation_By_OrphanBlock.FindData(LSearch,LDataPos,LBlockInformation);
 end;
 
-procedure TAbstractMemBlockchainStorage.DoBlockNotFound(ABlock: Integer);
+procedure TAbstractMemBlockchainStorage.DoBlockNotFound(ABlock: Integer; var AFound : Boolean);
 begin
   // Nothing to do here
+  AFound := False;
 end;
 
 procedure TAbstractMemBlockchainStorage.DoDeleteBlockChainBlocks(StartingDeleteBlock: Cardinal);
@@ -1009,14 +1014,20 @@ begin
     end;
 
     FFileMem := TFileMem.Create(FStorageFilename,ReadOnly);
-    FFileMem.IncreaseFileBytes := 1 * 1024*1024; // 1Mb each increase
+    FFileMem.IncreaseFileBytes := 10 * 1024*1024; // 10Mb each increase
 
     LCacheMem := FFileMem.LockCache;
     try
       LCacheMem.GridCache := False;
       LCacheMem.DefaultCacheDataBlocksSize := 1024;
-      LCacheMem.MaxCacheSize := 300 * 1024 * 1024; // 300Mb
+      {$IFDEF IS32BITS}
+      LCacheMem.MaxCacheSize := 5 * Int64(100 * 1024 * 1024); // 100Mb * 3 = 500Mb
       LCacheMem.MaxCacheDataBlocks := 150000;
+      {$ELSE}
+      LCacheMem.MaxCacheSize := 10 * Int64(100 * 1024 * 1024); // 100Mb * 10 = 1Gb
+      LCacheMem.MaxCacheDataBlocks := 750000;
+      {$ENDIF};
+      LCacheMem.OnFlushedCache := OnCacheMemFlushedCache;
     finally
       FFileMem.UnlockCache;
     end;
@@ -1122,7 +1133,7 @@ end;
 function TAbstractMemBlockchainStorage.DoLoadBlockChain(Operations: TPCOperationsComp; Block: Cardinal): Boolean;
 begin
   Result := DoLoadBlockChainExt(Operations,Block,Orphan);
-  if Not Result then BlockNotFound(Block);
+  if Not Result then BlockNotFound(Block,Result);
 end;
 
 function TAbstractMemBlockchainStorage.DoLoadBlockChainExt(
@@ -1313,7 +1324,6 @@ begin
   if FAutoFlushCache then begin
     LTC := TPlatform.GetTickCount;
     FileMem.FlushCache;
-    TLog.NewLog(ltdebug,ClassName,Format('Flushed Cache after finalized updating blockchain in %d millis',[TPlatform.GetElapsedMilliseconds(LTC)]));
   end;
 end;
 
@@ -1360,6 +1370,12 @@ begin
   end else Result := LBlockInformationFound.operationBlock.block;
 end;
 
+procedure TAbstractMemBlockchainStorage.OnCacheMemFlushedCache(
+  const ASender: TCacheMem; const AProcessDesc: String; AElapsedMilis: Int64);
+begin
+  TLog.NewLog(ltdebug,ASender.ClassName,Self.ClassName+' '+AProcessDesc)
+end;
+
 class function TAbstractMemBlockchainStorage.OrphanCompare(const ALeft, ARight: String): Integer;
 begin
   Result := BinStrComp(ALeft,ARight);
@@ -1965,15 +1981,16 @@ begin
   inherited;
 end;
 
-procedure TAbstractMemBlockchainStorageSecondary.DoBlockNotFound(ABlock: Integer);
+procedure TAbstractMemBlockchainStorageSecondary.DoBlockNotFound(ABlock: Integer; var AFound : Boolean);
 var LOperationsComp : TPCOperationsComp;
 begin
   inherited;
+  AFound := False;
   if (Assigned(FAuxStorage)) then begin
     LOperationsComp := TPCOperationsComp.Create(Nil);
     Try
       if FAuxStorage.LoadBlockChainBlock(LOperationsComp,ABlock) then begin
-        TLog.NewLog(ltdebug,ClassName,Format('BlockNotFound Migrating block %d with %d operations',[ABlock,LOperationsComp.Count]));
+        AFound := True;
         inherited DoSaveBlockChain(LOperationsComp);
       end;
     Finally
@@ -2078,7 +2095,7 @@ begin
         Inc(FPendingToSave.FAMStorage.FSaveStorageStats.affectedAccountCount);
       end;
     end else begin
-      if nLastBatch>0 then begin
+      if (nLastBatch>0) then begin
         TLog.NewLog(ltdebug,ClassName,Format('Finished %d operations... waiting for more - %s',
             [nLastBatch, FPendingToSave.FAMStorage.FSaveStorageStats.ToString]));
         nLastBatch := 0;
@@ -2137,7 +2154,7 @@ constructor TAbstractMemBlockchainStorage.TPendingToSave.Create(
 begin
   FAMStorage := AStorage;
   FTotal := 0;
-  FMaxPendingsCount := 5000;
+  FMaxPendingsCount := {$IFDEF IS32BITS}10000{$ELSE}50000{$ENDIF};
   FLastLogTC := TPlatform.GetTickCount;
   FOperationsRawData_By_RightOpHash := AAMBTreeTOperationRawDataByRightOpHash;
   FAffectedAccounts_By_Account_Block_OpBlock := AAMBTreeTAffectedAccountByAccountBlockOpBlock;

src/core/UBlockChain.pas

@@ -585,6 +585,7 @@ Type
     Function LoadBankFileInfo(Const AFilename : String; var ASafeBoxHeader : TPCSafeBoxHeader) : Boolean;
     Property Orphan : TOrphan read FOrphan write FOrphan;
     Function SaveBank(forceSave : Boolean) : Boolean;
+    Property IsRestoringFromFile : Boolean read FIsRestoringFromFile;
   End;
 
 Const
@@ -992,7 +993,7 @@ begin
         LProgressEndBlock := Storage.LastBlock - BlocksCount;
         while ((BlocksCount<=max_block)) do begin
           i := BlocksCount;
-          j := i + 99;
+          j := i + 999;
           // Load a batch of TPCOperationsComp;
           try
             LOpsInBlocks := 0;
@@ -1033,8 +1034,7 @@ begin
                 Exit;
               end else begin
                 // To prevent continuous saving...
-                if ((BlocksCount+(CT_BankToDiskEveryNBlocks*2)) >= Storage.LastBlock ) or
-                   ((BlocksCount MOD (CT_BankToDiskEveryNBlocks*10))=0) then begin
+                if ((BlocksCount MOD (CT_BankToDiskEveryNBlocks*10))=0) then begin
                   SaveBank(False);
                 end;
                 if (Assigned(restoreProgressNotify)) And (TPlatform.GetElapsedMilliseconds(tc)>1000) then begin
@@ -3166,7 +3166,7 @@ begin
 end;
 
 procedure TOperationsHashTree.MarkVerifiedECDSASignatures(operationsHashTreeToMark: TOperationsHashTree);
-var i, iPosInMyList, nMarkedAsGood, nAlreadyMarked : Integer;
+var i, iPosInMyList, nMarkedAsGood, nAlreadyMarked, nFound : Integer;
   opToMark, opInMyList : TPCOperation;
   myList, listToMark : TList<Pointer>;
 begin
@@ -3175,6 +3175,7 @@ begin
   If Self=operationsHashTreeToMark then Exit;
   nMarkedAsGood := 0;
   nAlreadyMarked := 0;
+  nFound := 0;
   myList := FHashTreeOperations.LockList;
   try
     if myList.Count<=0 then Exit; // Nothing to search...
@@ -3187,6 +3188,7 @@ begin
           // Check if found
           iPosInMyList := Self.IndexOfOperation(opToMark);
           if (iPosInMyList>=0) then begin
+            inc(nFound);
             opInMyList := POperationHashTreeReg(myList[iPosInMyList])^.Op;
             if (opInMyList.FHasValidSignature) then begin
               if (opToMark.FHasValidSignature) then inc(nAlreadyMarked)
@@ -3200,7 +3202,9 @@ begin
           end;
         end;
       end;
-      TLog.NewLog(ltdebug,ClassName,Format('Marked %d/%d operations as ValidSignature (%d before) from MemPool with %d operations',[nMarkedAsGood,listToMark.Count,nAlreadyMarked,myList.Count]));
+      if (nFound>0) then begin
+        TLog.NewLog(ltdebug,ClassName,Format('Marked %d/%d operations (%d found) as ValidSignature (%d before) from MemPool with %d operations',[nMarkedAsGood,listToMark.Count,nFound,nAlreadyMarked,myList.Count]));
+      end;
     finally
       operationsHashTreeToMark.FHashTreeOperations.UnlockList;
     end;

src/core/UConst.pas

@@ -199,7 +199,7 @@ Const
   CT_OpSubtype_Data_Signer                = 103;
   CT_OpSubtype_Data_Receiver              = 104;
 
-  CT_ClientAppVersion : String = {$IFDEF PRODUCTION}'5.7'{$ELSE}{$IFDEF TESTNET}'TESTNET 5.7'{$ELSE}{$ENDIF}{$ENDIF};
+  CT_ClientAppVersion : String = {$IFDEF PRODUCTION}'5.7.1'{$ELSE}{$IFDEF TESTNET}'TESTNET 5.7.1'{$ELSE}{$ENDIF}{$ENDIF};
 
   CT_Discover_IPs = {$IFDEF PRODUCTION}'bpascal1.dynamic-dns.net;bpascal2.dynamic-dns.net;pascalcoin1.dynamic-dns.net;pascalcoin2.dynamic-dns.net;pascalcoin1.dns1.us;pascalcoin2.dns1.us;pascalcoin1.dns2.us;pascalcoin2.dns2.us'
                     {$ELSE}'pascaltestnet1.dynamic-dns.net;pascaltestnet2.dynamic-dns.net;pascaltestnet1.dns1.us;pascaltestnet2.dns1.us'{$ENDIF};

src/core/UNode.pas

@@ -280,7 +280,9 @@ begin
       Result := Bank.AddNewBlockChainBlock(NewBlockOperations,TNetData.NetData.NetworkAdjustedTime.GetMaxAllowedTimestampForNewBlock,errors);
       if Result then begin
         {$IFDEF USE_ABSTRACTMEM}
-        Bank.SafeBox.PCAbstractMem.FlushCache;
+        If Not FBank.IsRestoringFromFile then begin
+          Bank.SafeBox.PCAbstractMem.FlushCache;
+        end;
         {$ENDIF}
         if Assigned(SenderConnection) then begin
           FNodeLog.NotifyNewLog(ltupdate,SenderConnection.ClassName,Format(';%d;%s;%s;;%d;%d;%d;%s',[OpBlock.block,sClientRemoteAddr,OpBlock.block_payload.ToPrintable,