Merge upstream

PIP/PIP-0022.md

@@ -0,0 +1,77 @@
+<pre>
+  PIP: PIP-22
+  Title: Continuous Integration
+  Type: Backend | Front-End | Process
+  Impact: None - Other
+  Author: Benjamin Ansbach <i>&lt;[email protected]&gt;</i>
+  Comments-URI: 
+  Status: Draft
+  Created: 2018-07-12
+</pre>
+
+## Summary
+
+PascalCoin builds are created manually. This proposal aims to add a continuous integration implementation for the PascalCoin repository focussing on automated builds of the PascalCoin software. The integration can later be extended to provide other automated tasks like automated testing of the software etc.
+
+## Motivation
+
+It is important to give interested community members as well as non-core developers without background in the Pascal language the possibility to test or use features which are not released for the MAINNET yet (e.g. Wallet optimizations, improved JSON RPC API, ..). This is not possible right now, as one needs to compile the source on it's own.
+
+## Specification
+
+The repository is hosted on github.com. Github provides several integrations from a number of CI-providers. The plan is to use the service of [travis CI](http://www.travis-ci.com) which has an integration service available for github.com and is free for open source projects.
+
+In addition, travis CI is proved to work for the PascalCoin environment as there are already reference implementations for libraries written in pascal. This will ease the first steps.
+
+### Build types
+
+Each build process will compile and pack the project for different targets (windows, linux, osx). 
+
+MAINNET builds are reserved for git tags. 
+
+TESTNET builds are reserved for branches and will be executed with each push to a specific branch (master for now).
+
+Tag-Based MAINNET builds will be pushed to the projects releases page automatically (https://github.com/PascalCoin/PascalCoin/releases).
+
+### Filename structure
+
+The structure of the resulting files can be defined as follows:
+
+`PascalCoin_($branch|$tag)_(TESTNET|MAINNET)_$target[.$target_compress_extension]`
+
+..where 
+
+- `$branch` is the name of the branch
+- `$tag` is the version of the tag
+- `$target` is the compile target
+- `$target_compress_extension` is the extension of the used compression (zip, .tar.gz, ...)
+
+Branch based builds are short-living builds. With each new push to a branch, the development-releases will be overwritten.
+
+### Forks
+
+Forks of the project can use the same implementation, since the configuration for the builds is saved inside of the repository.
+
+### Hosting
+
+Tag-Based releases will continue to be hosted on github.com and will contain a list of sha checksums while branch-based releases will be hosted on https://development.pascalcoin.org
+
+## Rationale
+
+Manual builds are error-prone. Automatic builds of development-releases will drastically improve the quality of the production releases, because users will have the chance to test new implementations or bug-fixes prior a definite production release.
+
+## Backwards Compatibility
+
+This PIP does not have any effect on the compatibility.
+
+## Reference Implementation
+
+None for PascalCoin but for pascal projects with travis. See some of these repositories: https://github.com/Xor-el
+
+## Links
+
+https://travis-ci.org
+
+https://travis-ci.com/plans
+
+https://docs.travis-ci.com/user/deployment/releases/

src/core/UConst.pas

@@ -102,7 +102,7 @@ Const
 
   CT_MagicNetIdentification = {$IFDEF PRODUCTION}$0A043580{$ELSE}$03000040{$ENDIF}; // Unix timestamp 168048000 ... It's Albert birthdate!
 
-  CT_NetProtocol_Version: Word = $0006; // Version 2.1.2 only allows net protocol 6 (Introduced on 2.0.0)
+  CT_NetProtocol_Version: Word = $0007; // Version 3.0.2 only allows net protocol 7 (Introduced on 3.0.0)
   // IMPORTANT NOTE!!!
   // NetProtocol_Available MUST BE always >= NetProtocol_version
   CT_NetProtocol_Available: Word = $0007;  // Remember, >= NetProtocol_version !!!

src/core/UFileStorage.pas

@@ -131,12 +131,13 @@ Var  iBlockHeaders : Integer;
   BlockHeader : TBlockHeader;
 begin
   Result := false;
+  BlockHeader := CT_TBlockHeader_NUL;
+  iBlockHeaders:=0; BlockHeaderFirstBlock:=0;
   stream := LockBlockChainStream;
   try
     if Not GetBlockHeaderFirstBytePosition(stream,Block,False,iBlockHeaders,BlockHeaderFirstBlock) then exit;
     if not StreamReadBlockHeader(stream,iBlockHeaders,BlockHeaderFirstBlock,Block,False,BlockHeader) then exit;
     Result := (BlockHeader.BlockNumber = Block) And
-        (((BlockHeader.BlockNumber MOD CT_GroupBlockSize)=0) OR (BlockHeader.StreamBlockRelStartPos>0)) And
         (BlockHeader.BlockSize>0);
   finally
     UnlockBlockChainStream;

src/core/UNetProtocol.pas

@@ -125,6 +125,7 @@ Type
 
   TOrderedServerAddressListTS = Class
   private
+    FAllowDeleteOnClean: Boolean;
     FNetData : TNetData;
     FCritical : TPCCriticalSection;
     FListByIp : TList;
@@ -149,6 +150,7 @@ Type
     Procedure UpdateNetConnection(netConnection : TNetConnection);
     procedure GetNodeServersToConnnect(maxNodes : Integer; useArray : Boolean; var nsa : TNodeServerAddressArray);
     Function GetValidNodeServers(OnlyWhereIConnected : Boolean; Max : Integer): TNodeServerAddressArray;
+    property AllowDeleteOnClean : Boolean read FAllowDeleteOnClean write FAllowDeleteOnClean;
   End;
 
 
@@ -240,6 +242,8 @@ Type
   TNetData = Class(TComponent)
   private
     FMaxNodeServersAddressesBuffer: Integer;
+    FMaxServersConnected: Integer;
+    FMinServersConnected: Integer;
     FNetDataNotifyEventsThread : TNetDataNotifyEventsThread;
     FNodePrivateKey : TECPrivateKey;
     FNetConnections : TPCThreadList;
@@ -265,6 +269,8 @@ Type
     FNetworkAdjustedTime : TNetworkAdjustedTime;
     Procedure IncStatistics(incActiveConnections,incClientsConnections,incServersConnections,incServersConnectionsWithResponse : Integer; incBytesReceived, incBytesSend : Int64);
     procedure SetMaxNodeServersAddressesBuffer(AValue: Integer);
+    procedure SetMaxServersConnected(AValue: Integer);
+    procedure SetMinServersConnected(AValue: Integer);
     procedure SetNetConnectionsActive(const Value: Boolean);  protected
     procedure Notification(AComponent: TComponent; Operation: TOperation); override;
     Procedure DiscoverServersTerminated(Sender : TObject);
@@ -326,6 +332,8 @@ Type
     Property NetworkAdjustedTime : TNetworkAdjustedTime read FNetworkAdjustedTime;
     Property MaxNodeServersAddressesBuffer : Integer read FMaxNodeServersAddressesBuffer write SetMaxNodeServersAddressesBuffer;
     Property OnProcessReservedAreaMessage : TProcessReservedAreaMessage read FOnProcessReservedAreaMessage write FOnProcessReservedAreaMessage;
+    Property MinServersConnected : Integer read FMinServersConnected write SetMinServersConnected;
+    Property MaxServersConnected : Integer read FMaxServersConnected write SetMaxServersConnected;
   End;
 
   { TNetConnection }
@@ -487,7 +495,11 @@ begin
       // Is an old blacklisted IP? (More than 1 hour)
       If (P^.is_blacklisted) AND
         ((forceCleanAll) OR ((P^.last_connection+(CT_LAST_CONNECTION_MAX_MINUTES)) < (UnivDateTimeToUnix(DateTime2UnivDateTime(now))))) then begin
-        SecuredDeleteFromListByIp(i);
+        if (AllowDeleteOnClean) then begin
+          SecuredDeleteFromListByIp(i);
+        end else begin
+          P^.is_blacklisted:=False;
+        end;
         inc(Result);
       end;
     end;
@@ -502,6 +514,7 @@ var i : Integer;
   nsa : TNodeServerAddress;
   currunixtimestamp : Cardinal;
 begin
+  If Not (FAllowDeleteOnClean) then Exit;
   currunixtimestamp := UnivDateTimeToUnix(DateTime2UnivDateTime(now));
   FCritical.Acquire;
   Try
@@ -575,6 +588,7 @@ begin
   FCritical := TPCCriticalSection.Create(Classname);
   FListByIp := TList.Create;
   FListByNetConnection := TList.Create;
+  FAllowDeleteOnClean := True;
 end;
 
 function TOrderedServerAddressListTS.DeleteNetConnection(netConnection: TNetConnection) : Boolean;
@@ -863,7 +877,8 @@ begin
   Index := L;
 end;
 
-procedure TOrderedServerAddressListTS.SetNodeServerAddress(Const nodeServerAddress: TNodeServerAddress);
+procedure TOrderedServerAddressListTS.SetNodeServerAddress(
+  const nodeServerAddress: TNodeServerAddress);
 Var i : Integer;
   P : PNodeServerAddress;
 begin
@@ -1147,6 +1162,8 @@ begin
   FNetClientsDestroyThread := TNetClientsDestroyThread.Create(Self);
   FNetworkAdjustedTime := TNetworkAdjustedTime.Create;
   FMaxNodeServersAddressesBuffer:=(CT_MAX_NODESERVERS_BUFFER DIV 2);
+  FMinServersConnected:=CT_MinServersConnected;
+  FMaxServersConnected:=CT_MaxServersConnected;
   If Not Assigned(_NetData) then _NetData := Self;
 end;
 
@@ -1267,9 +1284,9 @@ begin
   end;
   FNodeServersAddresses.CleanBlackList(False);
   If NetStatistics.ClientsConnections>0 then begin
-    j := CT_MinServersConnected - NetStatistics.ServersConnectionsWithResponse;
+    j := FMinServersConnected - NetStatistics.ServersConnectionsWithResponse;
   end else begin
-    j := CT_MaxServersConnected - NetStatistics.ServersConnectionsWithResponse;
+    j := FMaxServersConnected - NetStatistics.ServersConnectionsWithResponse;
   end;
   if j<=0 then exit;
   {$IFDEF HIGHLOG}TLog.NewLog(ltDebug,Classname,'Discover servers start process searching up to '+inttostr(j)+' servers');{$ENDIF}
@@ -1578,6 +1595,10 @@ Const CT_LogSender = 'GetNewBlockChainFromClient';
         end else begin
           // Restore a part from disk
           Bank.DiskRestoreFromOperations(start_block-1);
+          if (Bank.BlocksCount<start_block) then begin
+            TLog.NewLog(lterror,CT_LogSender,Format('No blockchain found start block %d, current %d',[start_block-1,Bank.BlocksCount]));
+            start_block := Bank.BlocksCount;
+          end;
           IsUsingSnapshot := False;
         end;
         start := start_block;
@@ -1628,7 +1649,9 @@ Const CT_LogSender = 'GetNewBlockChainFromClient';
           BlocksList.Free;
         end;
         start := Bank.BlocksCount;
-      until (Bank.BlocksCount=Connection.FRemoteOperationBlock.block+1) Or (finished);
+      until (Bank.BlocksCount=Connection.FRemoteOperationBlock.block+1) Or (finished)
+        // Allow to do not download ALL new blockchain in a separate folder, only needed blocks!
+        Or (Bank.SafeBox.WorkSum > (TNode.Node.Bank.SafeBox.WorkSum + $FFFFFFFF) );
       // New Build 1.5 more work vs more high
       // work = SUM(target) of all previous blocks (Int64)
       // -----------------------------
@@ -1647,11 +1670,13 @@ Const CT_LogSender = 'GetNewBlockChainFromClient';
               try
                 for start:=start_c to TNode.Node.Bank.BlocksCount-1 do begin
                   If TNode.Node.Bank.LoadOperations(OpExecute,start) then begin
-                    for i:=0 to OpExecute.Count-1 do begin
-                      // TODO: NEED TO EXCLUDE OPERATIONS ALREADY INCLUDED IN BLOCKCHAIN?
-                      oldBlockchainOperations.AddOperationToHashTree(OpExecute.Operation[i]);
+                    if (OpExecute.Count>0) then begin
+                      for i:=0 to OpExecute.Count-1 do begin
+                        // TODO: NEED TO EXCLUDE OPERATIONS ALREADY INCLUDED IN BLOCKCHAIN?
+                        oldBlockchainOperations.AddOperationToHashTree(OpExecute.Operation[i]);
+                      end;
+                      TLog.NewLog(ltInfo,CT_LogSender,'Recovered '+IntToStr(OpExecute.Count)+' operations from block '+IntToStr(start));
                     end;
-                    TLog.NewLog(ltInfo,CT_LogSender,'Recovered '+IntToStr(OpExecute.Count)+' operations from block '+IntToStr(start));
                   end else begin
                     TLog.NewLog(ltError,CT_LogSender,'Fatal error: Cannot read block '+IntToStr(start));
                   end;
@@ -1792,7 +1817,7 @@ Const CT_LogSender = 'GetNewBlockChainFromClient';
       SetLength(chunks,0);
       try
         // Will obtain chunks of 10000 blocks each
-        for i:=0 to _blockcount DIV 10000 do begin
+        for i:=0 to ((_blockcount-1) DIV 10000) do begin // Bug v3.0.1 and minors
           receiveChunk := TMemoryStream.Create;
           if (Not DownloadSafeBoxChunk(_blockcount,op.initial_safe_box_hash,(i*10000),((i+1)*10000)-1,receiveChunk,safeBoxHeader,errors)) then begin
             receiveChunk.Free;
@@ -1842,6 +1867,7 @@ Const CT_LogSender = 'GetNewBlockChainFromClient';
             If Not IsMyBlockchainValid then begin
               TNode.Node.Bank.Storage.EraseStorage;
             end;
+            TNode.Node.Bank.Storage.SaveBank;
             Connection.Send_GetBlocks(TNode.Node.Bank.BlocksCount,100,request_id);
             Result := true;
           end else begin
@@ -1868,6 +1894,7 @@ begin
     TLog.NewLog(ltdebug,CT_LogSender,'Is discovering servers...');
     exit;
   end;
+  if (Not Assigned(TNode.Node.Bank.StorageClass)) then Exit;
   //
   If FIsGettingNewBlockChainFromClient then begin
     TLog.NewLog(ltdebug,CT_LogSender,'Is getting new blockchain from client...');
@@ -1973,6 +2000,22 @@ begin
   else FMaxNodeServersAddressesBuffer:=AValue;
 end;
 
+procedure TNetData.SetMaxServersConnected(AValue: Integer);
+begin
+  if FMaxServersConnected=AValue then Exit;
+  if AValue<1 then FMaxServersConnected:=1
+  else FMaxServersConnected:=AValue;
+  if FMaxServersConnected<FMinServersConnected then FMinServersConnected:=FMaxServersConnected;
+end;
+
+procedure TNetData.SetMinServersConnected(AValue: Integer);
+begin
+  if FMinServersConnected=AValue then Exit;
+  if AValue<1 then FMinServersConnected:=1
+  else FMinServersConnected:=AValue;
+  if FMaxServersConnected<FMinServersConnected then FMaxServersConnected:=FMinServersConnected;
+end;
+
 class function TNetData.NetData: TNetData;
 begin
   if Not Assigned(_NetData) then begin
@@ -2966,7 +3009,7 @@ begin
       if (b=1) then begin
         DataBuffer.Read(c,SizeOf(c));
         start:=c;
-        max:=c;
+        max:=1; // Bug 3.0.1 (was c instead of fixed 1)
       end else begin
         DataBuffer.Read(c,SizeOf(c));
         start:=c;
@@ -4037,7 +4080,7 @@ begin
           FNetData.FNetStatistics.ServersConnections := newstats.ServersConnections;
           FNetData.FNetStatistics.ServersConnectionsWithResponse := newstats.ServersConnectionsWithResponse;
           // Must stop clients?
-          if (nserverclients>CT_MaxServersConnected) And // This is to ensure there are more serverclients than clients
+          if (nserverclients>FNetData.MaxServersConnected) And // This is to ensure there are more serverclients than clients
              ((nserverclients + nactive + ndeleted)>=FNetData.FMaxConnections) And (Assigned(netserverclientstop)) then begin
             TLog.NewLog(ltinfo,Classname,Format('Sending FinalizeConnection to NodeConnection %s created on %s (working time %s) - NetServerClients:%d Servers_active:%d Servers_deleted:%d',
               [netserverclientstop.Client.ClientRemoteAddr,FormatDateTime('hh:nn:ss',netserverclientstop.CreatedTime),
@@ -4048,7 +4091,7 @@ begin
         finally
           FNetData.FNetConnections.UnlockList;
         end;
-        if (nactive<=CT_MaxServersConnected) And (Not Terminated) then begin
+        if (nactive<=FNetData.MaxServersConnected) And (Not Terminated) then begin
           // Discover
           FNetData.DiscoverServers;
         end;

src/core/UThread.pas

@@ -252,7 +252,7 @@ begin
   if MaxWaitMilliseconds>60000 then MaxWaitMilliseconds := 60000;
   {$IFDEF HIGHLOG}
   lockWatingForCounter := Lock.WaitingForCounter;
-  lockStartedTimestamp := Lock.StartedTimestamp;
+  lockStartedTimestamp := Lock.StartedTickCount;
   lockCurrThread := Lock.CurrentThread;
   {$ENDIF}
   Repeat
@@ -262,7 +262,7 @@ begin
   {$IFDEF HIGHLOG}
   if Not Result then begin
     tc2 := TPlatform.GetTickCount;
-    if lockStartedTimestamp=0 then lockStartedTimestamp := Lock.StartedTimestamp;
+    if lockStartedTimestamp=0 then lockStartedTimestamp := Lock.StartedTickCount;
     if lockStartedTimestamp=0 then tc3 := 0
     else tc3 := tc2-lockStartedTimestamp;
     s := Format('Cannot Protect a critical section %s %s class %s after %d milis locked by %s waiting %d-%d elapsed milis: %d',
@@ -362,18 +362,18 @@ begin
   Repeat
     continue := inherited TryEnter;
     if (Not continue) then begin
-      If (not logged) And ((FStartedTimestamp>0) And ((FStartedTimestamp+1000)<TPlatform.GetTickCount)) then begin
+      If (not logged) And (TPlatform.GetElapsedMilliseconds(startTC)>1000) then begin
         logged := true;
         TLog.NewLog(ltdebug,ClassName,'ALERT Critical section '+IntToHex(PtrInt(Self),8)+' '+Name+
           ' locked by '+IntToHex(FCurrentThread,8)+' waiting '+
-          IntToStr(FWaitingForCounter)+' elapsed milis: '+IntToStr(TPlatform.GetTickCount-FStartedTimestamp) );
+          IntToStr(FWaitingForCounter)+' elapsed milis: '+IntToStr(TPlatform.GetElapsedMilliseconds(startTC)));
         continue := true;
         inherited;
       end else sleep(1);
     end;
   Until continue;
   if (logged) then begin
-    TLog.NewLog(ltdebug,Classname,'ENTER Critical section '+IntToHex(PtrInt(Self),8)+' '+Name+' elapsed milis: '+IntToStr(TPlatform.GetTickCount - startTC) );
+    TLog.NewLog(ltdebug,Classname,'ENTER Critical section '+IntToHex(PtrInt(Self),8)+' '+Name+' elapsed milis: '+IntToStr(TPlatform.GetElapsedMilliseconds(startTC)) );
   end;
   FCounterLock.Acquire;
   try
@@ -382,8 +382,7 @@ begin
     FCounterLock.Release;
   end;
   FCurrentThread := TThread.CurrentThread.ThreadID;
-  FStartedTimestamp := TPlatform.GetTickCount;
-  inherited;
+  FStartedTickCount := TPlatform.GetTickCount;
 end;
 {$ENDIF}
 
@@ -408,7 +407,7 @@ end;
 procedure TPCCriticalSection.Release;
 begin
   FCurrentThread := 0;
-  FStartedTimestamp := 0;
+  FStartedTickCount := 0;
   inherited;
 end;
 
@@ -422,7 +421,7 @@ begin
   end;
   If inherited TryEnter then begin
     FCurrentThread := TThread.CurrentThread.ThreadID;
-    FStartedTimestamp := TPlatform.GetTickCount;
+    FStartedTickCount := TPlatform.GetTickCount;
     Result := true;
   end else Result := false;
   FCounterLock.Acquire;