Check operations on Mempool to avoid signature verification twice

Improvement speed when mempool has a lot of operations, does not need to verify signatures again.
Also some logs reduction

src/core/UBlockChain.pas

@@ -212,6 +212,7 @@ Type
     Property Previous_Signer_updated_block : Cardinal read FPrevious_Signer_updated_block; // deprecated
     Property Previous_Destination_updated_block : Cardinal read FPrevious_Destination_updated_block; // deprecated
     Property Previous_Seller_updated_block : Cardinal read FPrevious_Seller_updated_block; // deprecated
+    function IsValidECDSASignature(const PubKey: TECDSA_Public; current_protocol : Word; const Signature: TECDSA_SIG): Boolean;
   public
     constructor Create; virtual;
     destructor Destroy; override;
@@ -298,6 +299,7 @@ Type
     procedure RemoveByOpReference(const opReference : TOpReference);
     Property OnChanged : TNotifyEvent read FOnChanged write FOnChanged;
     Property Max0feeOperationsBySigner : Integer Read FMax0feeOperationsBySigner write SetMax0feeOperationsBySigner;
+    procedure MarkVerifiedECDSASignatures(operationsHashTreeToMark : TOperationsHashTree);
   End;
 
   { TPCOperationsComp }
@@ -2104,6 +2106,7 @@ begin
     h := FHashTree + op.Sha256;
     P^.Op.FBufferedSha256:=op.FBufferedSha256;
     P^.Op.tag := list.Count;
+    P^.Op.FHasValidSignature := op.FHasValidSignature; // Improvement speed v4.0.2 reusing previously signed value
     // Improvement TOperationsHashTree speed 2.1.6
     // Include to hash tree (Only if CalcNewHashTree=True)
     If (CalcNewHashTree) And (Length(FHashTree)=32) then begin
@@ -2313,6 +2316,46 @@ begin
   Result := true;
 end;
 
+procedure TOperationsHashTree.MarkVerifiedECDSASignatures(operationsHashTreeToMark: TOperationsHashTree);
+var i, iPosInMyList, nMarkedAsGood : Integer;
+  opToMark, opInMyList : TPCOperation;
+  myList, listToMark : TList;
+begin
+  // Introduced on Build 4.0.2 to increase speed
+  // Will search each "operationsHashTreeToMark" operation on my current list. If found, will set same FHasValidSignature in order to mark operation in "operationsHashTreeToMark" as verified
+  If Self=operationsHashTreeToMark then Exit;
+  nMarkedAsGood := 0;
+  myList := FHashTreeOperations.LockList;
+  try
+    if myList.Count<=0 then Exit; // Nothing to search...
+    listToMark := operationsHashTreeToMark.FHashTreeOperations.LockList;
+    Try
+      if listToMark.Count<=0 then Exit; // Nothing to search...
+      for i:=0 to listToMark.Count-1 do begin
+        opToMark := POperationHashTreeReg(listToMark[i])^.Op;
+        if Not opToMark.FHasValidSignature then begin
+          // Check if found
+          iPosInMyList := Self.IndexOfOperation(opToMark);
+          if (iPosInMyList>=0) then begin
+            opInMyList := POperationHashTreeReg(myList[iPosInMyList])^.Op;
+            if (opInMyList.FHasValidSignature) then begin
+              opToMark.FHasValidSignature:=True;
+              inc(nMarkedAsGood);
+            end;
+          end;
+        end;
+      end;
+      if nMarkedAsGood>0 then begin
+        TLog.NewLog(ltdebug,ClassName,Format('Marked %d/%d operations as ValidSignature from MemPool with %d operations',[nMarkedAsGood,listToMark.Count,myList.Count]));
+      end;
+    finally
+      operationsHashTreeToMark.FHashTreeOperations.UnlockList;
+    end;
+  finally
+    FHashTreeOperations.UnlockList;
+  end;
+end;
+
 function TOperationsHashTree.OperationsCount: Integer;
 Var l : TList;
 begin
@@ -2676,6 +2719,16 @@ begin
   //
 end;
 
+function TPCOperation.IsValidECDSASignature(const PubKey: TECDSA_Public; current_protocol: Word; const Signature: TECDSA_SIG): Boolean;
+begin
+  // Will reuse FHasValidSignature if checked previously and was True
+  // Introduced on Build 4.0.2 to increase speed using MEMPOOL verified operations instead of verify again everytime
+  if (Not FHasValidSignature) then begin
+    FHasValidSignature := TCrypto.ECDSAVerify(PubKey,GetDigestToSign(current_protocol),Signature);
+  end;
+  Result := FHasValidSignature;
+end;
+
 function TPCOperation.LoadFromNettransfer(Stream: TStream): Boolean;
 begin
   Result := LoadOpFromStream(Stream, False);

src/core/UFileStorage.pas

@@ -303,7 +303,7 @@ begin
     fs.Position:=0;
     fs.Size:=0;
     OperationsHashTree.SaveOperationsHashTreeToStream(fs,true);
-    TLog.NewLog(ltdebug,ClassName,Format('DoSavePendingBufferOperations operations:%d',[OperationsHashTree.OperationsCount]));
+    {$IFDEF HIGHLOG}TLog.NewLog(ltdebug,ClassName,Format('DoSavePendingBufferOperations operations:%d',[OperationsHashTree.OperationsCount]));{$ENDIF}
   finally
     UnlockBlockChainStream;
   end;

src/core/UNetProtocol.pas

@@ -1653,6 +1653,7 @@ Const CT_LogSender = 'GetNewBlockChainFromClient';
                 IsAScam := true;
                 break;
               end;
+              TNode.Node.MarkVerifiedECDSASignaturesFromMemPool(OpExecute); // Improvement speed v4.0.2
               if Bank.AddNewBlockChainBlock(OpExecute,TNetData.NetData.NetworkAdjustedTime.GetMaxAllowedTimestampForNewBlock,newBlock,errors) then begin
                 inc(i);
               end else begin
@@ -2192,7 +2193,7 @@ begin
     P^.RequestId := request_id;
     P^.SendTime := Now;
     l.Add(P);
-    TLog.NewLog(ltdebug,Classname,'Registering request to '+Sender.ClientRemoteAddr+' Op:'+OperationToText(operation)+' Id:'+inttostr(request_id)+' Total pending:'+Inttostr(l.Count));
+    {$IFDEF HIGHLOG}TLog.NewLog(ltdebug,Classname,'Registering request to '+Sender.ClientRemoteAddr+' Op:'+OperationToText(operation)+' Id:'+inttostr(request_id)+' Total pending:'+Inttostr(l.Count));{$ENDIF}
   Finally
     FRegisteredRequests.UnlockList;
   End;
@@ -2223,11 +2224,13 @@ begin
         l.Delete(i);
         Dispose(P);
         Result := true;
+        {$IFDEF HIGHLOG}
         if Assigned(Sender.FTcpIpClient) then begin
           TLog.NewLog(ltdebug,Classname,'Unregistering request to '+Sender.ClientRemoteAddr+' Op:'+OperationToText(operation)+' Id:'+inttostr(request_id)+' Total pending:'+Inttostr(l.Count));
         end else begin
           TLog.NewLog(ltdebug,Classname,'Unregistering request to (NIL) Op:'+OperationToText(operation)+' Id:'+inttostr(request_id)+' Total pending:'+Inttostr(l.Count));
         end;
+        {$ENDIF}
       end;
     end;
   finally
@@ -2492,6 +2495,7 @@ procedure TNetConnection.DoProcessBuffer;
 Var HeaderData : TNetHeaderData;
   ms : TMemoryStream;
   ops : AnsiString;
+  iPending : Integer;
 begin
   if FDoFinalizeConnection then begin
     TLog.NewLog(ltdebug,Classname,'Executing DoFinalizeConnection at client '+ClientRemoteAddr);
@@ -2509,11 +2513,14 @@ begin
   If ((FLastDataReceivedTS>0) Or ( NOT (Self is TNetServerClient)))
      AND ((FLastDataReceivedTS+(1000*FRandomWaitSecondsSendHello)<TPlatform.GetTickCount) AND (FLastDataSendedTS+(1000*FRandomWaitSecondsSendHello)<TPlatform.GetTickCount)) then begin
      // Build 1.4 -> Changing wait time from 120 secs to a random seconds value
-    If TNetData.NetData.PendingRequest(Self,ops)>=2 then begin
+    iPending := TNetData.NetData.PendingRequest(Self,ops);
+    If iPending>=2 then begin
       TLog.NewLog(ltDebug,Classname,'Pending requests without response... closing connection to '+ClientRemoteAddr+' > '+ops);
       Connected := false;
     end else begin
-      TLog.NewLog(ltDebug,Classname,'Sending Hello to check connection to '+ClientRemoteAddr+' > '+ops);
+      if iPending>0 then begin
+        TLog.NewLog(ltDebug,Classname,'Sending Hello to check connection to '+ClientRemoteAddr+' > '+ops);
+      end;
       Send_Hello(ntp_request,TNetData.NetData.NewRequestId);
     end;
   end else if (Self is TNetServerClient) AND (FLastDataReceivedTS=0) And (FCreatedTime+EncodeTime(0,1,0,0)<Now) then begin
@@ -2807,6 +2814,7 @@ begin
            exit;
         end;
         if (op.OperationBlock.block=TNode.Node.Bank.BlocksCount) then begin
+          TNode.Node.MarkVerifiedECDSASignaturesFromMemPool(op); // Improvement speed v4.0.2
           if (TNode.Node.Bank.AddNewBlockChainBlock(op,TNetData.NetData.NetworkAdjustedTime.GetMaxAllowedTimestampForNewBlock, newBlockAccount,errors)) then begin
             // Ok, one more!
           end else begin
@@ -3401,7 +3409,7 @@ Begin
           ClientAppVersion := other_version;
           if (DataBuffer.Size-DataBuffer.Position>=SizeOf(FRemoteAccumulatedWork)) then begin
             DataBuffer.Read(FRemoteAccumulatedWork,SizeOf(FRemoteAccumulatedWork));
-            TLog.NewLog(ltdebug,ClassName,'Received HELLO with height: '+inttostr(op.OperationBlock.block)+' Accumulated work '+IntToStr(FRemoteAccumulatedWork));
+            TLog.NewLog(ltdebug,ClassName,'Received HELLO with height: '+inttostr(op.OperationBlock.block)+' Accumulated work '+IntToStr(FRemoteAccumulatedWork)+ ' Remote block: '+TPCOperationsComp.OperationBlockToText(FRemoteOperationBlock));
           end;
         end;
         //
@@ -3414,7 +3422,7 @@ Begin
         end;
       end;
 
-      TLog.NewLog(ltdebug,Classname,'Hello received: '+TPCOperationsComp.OperationBlockToText(FRemoteOperationBlock));
+      {$IFDEF HIGHLOG}TLog.NewLog(ltdebug,Classname,'Hello received: '+TPCOperationsComp.OperationBlockToText(FRemoteOperationBlock));{$ENDIF}
       if (HeaderData.header_type in [ntp_request,ntp_response]) then begin
         // Response:
         if (HeaderData.header_type=ntp_request) then begin
@@ -3726,7 +3734,7 @@ begin
               iDebugStep := 500;
               TNetData.NetData.NodeServersAddresses.UpdateNetConnection(Self);
               iDebugStep := 800;
-              TLog.NewLog(ltDebug,Classname,'Received '+CT_NetTransferType[HeaderData.header_type]+' operation:'+TNetData.OperationToText(HeaderData.operation)+' id:'+Inttostr(HeaderData.request_id)+' Buffer size:'+Inttostr(HeaderData.buffer_data_length) );
+              {$IFDEF HIGHLOG}TLog.NewLog(ltDebug,Classname,'Received '+CT_NetTransferType[HeaderData.header_type]+' operation:'+TNetData.OperationToText(HeaderData.operation)+' id:'+Inttostr(HeaderData.request_id)+' Buffer size:'+Inttostr(HeaderData.buffer_data_length) );{$ENDIF}
               if (RequestId=HeaderData.request_id) And (HeaderData.header_type=ntp_response) then begin
                 Result := true;
               end else begin
@@ -4048,10 +4056,10 @@ begin
     Buffer.Position := 0;
     TPCThread.ProtectEnterCriticalSection(Self,FNetLock);
     Try
-      TLog.NewLog(ltDebug,Classname,'Sending: '+CT_NetTransferType[NetTranferType]+' operation:'+
+      {$IFDEF HIGHLOG}TLog.NewLog(ltDebug,Classname,'Sending: '+CT_NetTransferType[NetTranferType]+' operation:'+
         TNetData.OperationToText(operation)+' id:'+Inttostr(request_id)+' errorcode:'+InttoStr(errorcode)+
         ' Size:'+InttoStr(Buffer.Size)+'b '+s+'to '+
-        ClientRemoteAddr);
+        ClientRemoteAddr);{$ENDIF}
       (Client as TBufferedNetTcpIpClient).WriteBufferToSend(Buffer);
       FLastDataSendedTS := TPlatform.GetTickCount;
       FRandomWaitSecondsSendHello := (CT_NewLineSecondsAvg DIV 3) + Random(CT_NewLineSecondsAvg DIV 5);
@@ -4266,7 +4274,7 @@ begin
     end;
     // Checking if operationblock is the same to prevent double messaging...
     If (TPCOperationsComp.EqualsOperationBlock(FRemoteOperationBlock,NewBlock.OperationBlock)) then begin
-      TLog.NewLog(ltDebug,ClassName,'This connection has the same block, does not need to send');
+      {$IFDEF HIGHLOG}TLog.NewLog(ltDebug,ClassName,'This connection has the same block, does not need to send');{$ENDIF}
       exit;
     end;
     if (TNode.Node.Bank.BlocksCount<>NewBlock.OperationBlock.block+1) then begin
@@ -4870,7 +4878,7 @@ begin
     P^.timeOffset := clientTimestamp - UnivDateTimeToUnix(DateTime2UnivDateTime(now));
     if (lastOffset<>P^.timeOffset) then begin
       UpdateMedian(l);
-      TLog.NewLog(ltDebug,ClassName,Format('UpdateIp (%s,%d) - Total:%d/%d Offset:%d',[clientIp,clientTimestamp,l.Count,FTotalCounter,FTimeOffset]));
+      {$IFDEF HIGHLOG}TLog.NewLog(ltDebug,ClassName,Format('UpdateIp (%s,%d) - Total:%d/%d Offset:%d',[clientIp,clientTimestamp,l.Count,FTotalCounter,FTimeOffset]));{$ENDIF}
     end;
   finally
     FTimesList.UnlockList;

src/core/UNode.pas

@@ -107,6 +107,7 @@ Type
     //
     Property BroadcastData : Boolean read FBroadcastData write FBroadcastData;
     Property UpdateBlockchain : Boolean read FUpdateBlockchain write FUpdateBlockchain;
+    procedure MarkVerifiedECDSASignaturesFromMemPool(newOperationsToValidate : TPCOperationsComp);
   End;
 
   TNodeNotifyEvents = Class;
@@ -234,6 +235,7 @@ begin
         errors := 'Duplicated block';
         exit;
       end;
+      MarkVerifiedECDSASignaturesFromMemPool(NewBlockOperations); // Improvement speed v4.0.2
       // Improvement TNode speed 2.1.6
       // Does not need to save a FOperations backup because is Sanitized by "TNode.OnBankNewBlock"
       Result := Bank.AddNewBlockChainBlock(NewBlockOperations,TNetData.NetData.NetworkAdjustedTime.GetMaxAllowedTimestampForNewBlock,newBlockAccount,errors);
@@ -668,6 +670,25 @@ begin
   FLockNodeOperations.Release;
 end;
 
+procedure TNode.MarkVerifiedECDSASignaturesFromMemPool(newOperationsToValidate: TPCOperationsComp);
+begin
+  // Introduced on Build 4.0.2 to increase speed using MEMPOOL verified operations instead of verify again everytime
+  // Will check if "newOperationsToValidate" operations are on MEMPOOL. If found, will set same FHasValidSignature value in order to mark as verified
+  if newOperationsToValidate = FOperations then Exit; // Is the same, do nothing
+  if newOperationsToValidate.OperationBlock.protocol_version <> newOperationsToValidate.OperationBlock.protocol_version then Exit; // Must be same protocol
+  newOperationsToValidate.Lock;
+  try
+    FLockNodeOperations.Acquire;
+    try
+      Operations.OperationsHashTree.MarkVerifiedECDSASignatures(newOperationsToValidate.OperationsHashTree);
+    finally
+      FLockNodeOperations.Release;
+    end;
+  finally
+    newOperationsToValidate.Unlock;
+  end;
+end;
+
 class function TNode.EncodeNodeServerAddressArrayToIpString(
   const NodeServerAddressArray: TNodeServerAddressArray): AnsiString;
 var i : Integer;
@@ -1308,7 +1329,7 @@ begin
     try
       DebugStep := 'Checking connected';
       if Not FNetconnection.Connected then exit;
-      TLog.NewLog(ltdebug,ClassName,'Sending new block found to '+FNetConnection.Client.ClientRemoteAddr);
+      {$IFDEF HIGHLOG}TLog.NewLog(ltdebug,ClassName,'Sending new block found to '+FNetConnection.Client.ClientRemoteAddr);{$ENDIF}
       DebugStep := 'Sending';
       FNetConnection.Send_NewBlockFound(FNewBlockOperations);
       DebugStep := 'Checking connected again';

src/core/UOpTransaction.pas

@@ -541,11 +541,12 @@ begin
     end;
   end;
 
-  If Not TCrypto.ECDSAVerify(account_signer.accountInfo.accountkey,GetDigestToSign(AccountTransaction.FreezedSafeBox.CurrentProtocol),FData.sign) then begin
-    errors := 'Invalid sign';
-    FHasValidSignature := false;
-    exit;
-  end else FHasValidSignature := true;
+  // Check signature
+  If Not IsValidECDSASignature(account_signer.accountInfo.accountkey,AccountTransaction.FreezedSafeBox.CurrentProtocol,FData.sign) then begin
+    errors := 'Invalid ECDSA signature';
+    Exit;
+  end;
+
   FPrevious_Signer_updated_block := account_signer.updated_block;
   FPrevious_Destination_updated_block := account_target.updated_block;
   If (public_key in FData.changes_type) then begin
@@ -735,7 +736,6 @@ function TOpTransaction.DoOperation(AccountPreviousUpdatedBlock : TAccountPrevio
 Var s_new, t_new : Int64;
   totalamount : Cardinal;
   sender,target,seller : TAccount;
-  _h : TRawBytes;
   _IsBuyTransaction :  Boolean;
 begin
   Result := false;
@@ -806,13 +806,11 @@ begin
   end;
 
   // Check signature
-  _h := GetDigestToSign(AccountTransaction.FreezedSafeBox.CurrentProtocol);
-  if (Not TCrypto.ECDSAVerify(sender.accountInfo.accountkey,_h,FData.sign)) then begin
-    errors := 'Invalid sign';
-    FHasValidSignature := false;
+  If Not IsValidECDSASignature(sender.accountInfo.accountkey,AccountTransaction.FreezedSafeBox.CurrentProtocol,FData.sign) then begin
+    errors := 'Invalid ECDSA signature';
     Exit;
-  end else FHasValidSignature := true;
-  //
+  end;
+
   FPrevious_Signer_updated_block := sender.updated_block;
   FPrevious_Destination_updated_block := target.updated_block;
 
@@ -1272,11 +1270,11 @@ begin
     end;
   end;
 
-  If Not TCrypto.ECDSAVerify(account_signer.accountInfo.accountkey,GetDigestToSign(AccountTransaction.FreezedSafeBox.CurrentProtocol),FData.sign) then begin
-    errors := 'Invalid sign';
-    FHasValidSignature := false;
-    exit;
-  end else FHasValidSignature := true;
+  // Check signature
+  If Not IsValidECDSASignature(account_signer.accountInfo.accountkey,AccountTransaction.FreezedSafeBox.CurrentProtocol,FData.sign) then begin
+    errors := 'Invalid ECDSA signature';
+    Exit;
+  end;
 
   FPrevious_Signer_updated_block := account_signer.updated_block;
   FPrevious_Destination_updated_block := account_target.updated_block;
@@ -1756,11 +1754,11 @@ begin
     exit;
   end;
 
-  If Not TCrypto.ECDSAVerify(account_signer.accountInfo.accountkey,GetDigestToSign(AccountTransaction.FreezedSafeBox.CurrentProtocol),FData.sign) then begin
-    errors := 'Invalid sign';
-    FHasValidSignature := false;
-    exit;
-  end else FHasValidSignature := true;
+  // Check signature
+  If Not IsValidECDSASignature(account_signer.accountInfo.accountkey,AccountTransaction.FreezedSafeBox.CurrentProtocol,FData.sign) then begin
+    errors := 'Invalid ECDSA signature';
+    Exit;
+  end;
 
   FPrevious_Signer_updated_block := account_signer.updated_block;
   FPrevious_Destination_updated_block := account_target.updated_block;
@@ -2299,11 +2297,11 @@ begin
     Exit;
   end;
 
-  If Not TCrypto.ECDSAVerify(account_signer.accountInfo.accountkey,GetDigestToSign(AccountTransaction.FreezedSafeBox.CurrentProtocol),FData.sign) then begin
-    errors := 'Invalid sign';
-    FHasValidSignature := false;
+  // Check signature
+  If Not IsValidECDSASignature(account_signer.accountInfo.accountkey,AccountTransaction.FreezedSafeBox.CurrentProtocol,FData.sign) then begin
+    errors := 'Invalid ECDSA signature';
     Exit;
-  end else FHasValidSignature := true;
+  end;
 
   Result := AccountTransaction.TransferAmount(AccountPreviousUpdatedBlock,
     FData.account_sender,FData.account_signer,FData.account_target,

src/core/UTxMultiOperation.pas

@@ -445,8 +445,14 @@ var i : Integer;
   ophtosign : TRawBytes;
 begin
   // Init
-  FHasValidSignature:=False;
   SetLength(errors,0);
+  If FHasValidSignature then begin
+    // Will reuse FHasValidSignature if checked previously and was True
+    // Introduced on Build 4.0.2 to increase speed using MEMPOOL verified operations instead of verify again everytime
+    // Multioperations will not use standard TPCOperation.IsValidECDSASignature call because will need to check more than 1 signature
+    Result := True;
+    Exit;
+  end;
   // Do check it!
   Try
     ophtosign := GetDigestToSign(AccountTransaction.FreezedSafeBox.CurrentProtocol);