PIP-0009: memory transform improvement, implementation & unit test updates

PIP/PIP-0009.md

@@ -86,7 +86,7 @@ A low-memory, GPU and ASIC-resistant hash algorithm called **Random Hash** is pr
             if Round < 1 or Round > N then
                 Error 'Round must be between 0 and N inclusive'
 
-            let roundOutputs = new List of ByteArray;
+            let roundOutputs = new List of ByteArray
             
             if Round = 1 then
                 let seed = Checksum(blockHeader)      // can hash blockHeader first, but not required
@@ -118,7 +118,7 @@ A low-memory, GPU and ASIC-resistant hash algorithm called **Random Hash** is pr
         begin
             let seed = Checksum(input)
             let gen = RandomNumberGenerator(seed)
-            let size = Length(input) + ExpansionFactor*M;
+            let size = Length(input) + ExpansionFactor*M
             let output = input.Clone
             let bytesToAdd = size - Length(input)
             while bytesToAdd > 0 do
@@ -166,21 +166,182 @@ A low-memory, GPU and ASIC-resistant hash algorithm called **Random Hash** is pr
 
         Function RandomNumberGenerator(seed : DWord) : TMersenneTwister
             // standard Mersenne Twister random number generator (or other suitably chosen algorithm)
-        end;
+        end
 ```
 
 #### Memory transform methods
 
 These methods are iteratively and randomly applied to a hash output in order to rapidly expand it for compression in the next round. **Note**: the length of the output is always the same as the length of the input.
 ```
-     - Method 1: No-Op         (e.g. input = 123456   output = 123456)
-     - Method 2: Swap-LR       (e.g. input = 123456   output = 456123)   
-     - Method 3: Reverse       (e.g. input = 123456   output = 654321)  
-     - Method 4: L-Interleave  (e.g. input = 123456   output = 142536)
-     - Method 5: R-Interleave  (e.g. input = 123456   output = 415263)
-     - Method 6: L-XOR         (e.g. input = 123456   output = XOR(1,2), XOR(3,4), XOR(5,6), XOR(1,6), XOR(2,5), XOR(3,4)
-     - Method 7: ROL-ladder    (e.g. input = ABCDEF   output = ROL(A, 0), ROL(B, 1), ... , ROL(F, 5)
-     - Method 8: ROR-ladder    (e.g. input = ABCDEF   output = ROR(A, 0), ROR(B, 1), ... , ROR(F, 5)
+     - Memory Transform 1: XorShift32    (e.g. input = 1234567  output = select random from input using XorShift32 RNG
+     - Memory Transform 2: Swap-LR       (e.g. input = 1234567  output = 5674123)   
+     - Memory Transform 3: Reverse       (e.g. input = 1234567  output = 7654321)  
+     - Memory Transform 4: L-Interleave  (e.g. input = 1234567  output = 1526354)
+     - Memory Transform 5: R-Interleave  (e.g. input = 1234567  output = 5162734)
+     - Memory Transform 6: L-XOR         (e.g. input = 1234567  output = XOR(1,2), XOR(3,4), XOR(5,6), 7, XOR(1,7), XOR(2,6), XOR(3,5)
+     - Memory Transform 7: ROL-ladder    (e.g. input = ABCDEF   output = ROL(A, LENGTH - 0), ROL(B, LENGTH - 1), ... , ROL(F, LENGTH - 5)
+     - Memory Transform 8: ROR-ladder    (e.g. input = ABCDEF   output = ROR(A, LENGTH - 0), ROR(B, LENGTH - 1), ... , ROR(F, LENGTH - 5)
+```
+
+Formal definitions are as follows:
+
+##### Memory Transform 1
+
+This selects random bytes for AChunk using XorShift32 RNG. The initial seed is the CHECKSUM of AChunk. If CHECKSUM is 0 then 1 is used instead.
+
+```
+    function MemTransform1(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength : UInt32
+      LState : UInt32
+
+      function XorShift32 : UInt32 inline
+      begin
+        LState := LState XOR (LState SHL 13)
+        LState := LState XOR (LState SHR 17)
+        LState := LState XOR (LState SHL 5)
+        Result := LState
+      end
+
+    begin
+      // Seed XorShift32 with non-zero seed (checksum of input or 1)
+      LState := Checksum(AChunk)
+      if LState = 0 then
+        LState := 1
+
+      // Select random bytes from input using XorShift32 RNG
+      LChunkLength := Length(AChunk)
+      SetLength(Result, LChunkLength)
+      for i := 0 to High(AChunk) do
+        Result[i] := AChunk[XorShift32 MOD LChunkLength]
+    end   
+```
+
+##### Memory Transform 2
+
+```
+    function MemTransform2(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength, LPivot, LOdd: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      LPivot := LChunkLength SHR 1
+      LOdd := LChunkLength MOD 2
+      SetLength(Result, LChunkLength)
+      Move(AChunk[LPivot + LOdd], Result[0], LPivot)
+      Move(AChunk[0], Result[LPivot + LOdd], LPivot)
+      // Set middle-byte for odd-length arrays
+      if LOdd = 1 then
+        Result[LPivot] := AChunk[LPivot]
+    end
+```
+
+##### Memory Transform 3
+
+```pascal
+    function MemTransform3(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      SetLength(Result, LChunkLength)
+      for i := 0 to High(AChunk) do
+        Result[i] := AChunk[LChunkLength - i - 1]
+    end
+```
+
+##### Memory Transform 4
+
+```pascal
+    function MemTransform4(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength, LPivot, LOdd: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      LPivot := LChunkLength SHR 1
+      LOdd := LChunkLength MOD 2
+      SetLength(Result, LChunkLength)
+      for i := 0 to LPivot - 1 do
+      begin
+        Result[(i * 2)] := AChunk[i]
+        Result[(i * 2) + 1] := AChunk[i + LPivot + LOdd]
+      end
+      // Set final byte for odd-lengths
+      if LOdd = 1 THEN
+        Result[High(Result)] := AChunk[LPivot]
+    end
+```
+
+##### Memory Transform 5
+
+```pascal
+    function MemTransform5(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength, LPivot, LOdd: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      LPivot := LChunkLength SHR 1
+      LOdd := LChunkLength MOD 2
+      SetLength(Result, LChunkLength)
+      for i := 0 to LPivot - 1 do
+      begin
+        Result[(i * 2)] := AChunk[i + LPivot + LOdd]
+        Result[(i * 2) + 1] := AChunk[i]
+      end
+      // Set final byte for odd-lengths
+      if LOdd = 1 THEN
+        Result[High(Result)] := AChunk[LPivot]
+    end
+```
+
+##### Memory Transform 6
+
+```pascal
+    function MemTransform6(const AChunk: TBytes): TBytes
+    var
+      i, LChunkLength, LPivot, LOdd: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      LPivot := LChunkLength SHR 1
+      LOdd := LChunkLength MOD 2
+      SetLength(Result, LChunkLength)
+      for i := 0 to Pred(LPivot) do
+      begin
+        Result[i] := AChunk[(i * 2)] xor AChunk[(i * 2) + 1]
+        Result[i + LPivot + LOdd] := AChunk[i] xor AChunk[LChunkLength - i - 1]
+      end
+      // Set middle-byte for odd-lengths
+      if LOdd = 1 THEN
+        Result[LPivot] := AChunk[High(AChunk)]
+    end
+```
+
+##### Memory Transform 7
+
+```pascal
+    function MemTransform7(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      SetLength(Result, LChunkLength)
+      for i := 0 to High(AChunk) do
+        Result[i] := TBits.RotateLeft8(AChunk[i], LChunkLength - i)
+    end
+```
+
+##### Memory Transform 8
+
+```pascal
+    function MemTransform8(AChunk: TBytes): TBytes
+    var
+      i, LChunkLength: Int32
+    begin
+      LChunkLength := Length(AChunk)
+      SetLength(Result, LChunkLength)
+      for i := 0 to High(AChunk) do
+        Result[i] := TBits.RotateRight8(AChunk[i], LChunkLength - i)
+    end 
 ```
 
 ### RandomHash Analysis
@@ -355,6 +516,7 @@ A reference implementation of RandomHash can be found [here][3].
 ## Acknowledgements
 
 Refinements to improve GPU-hardness were provided by Ian Muldoon.
+Improvements to memory tansform randomness provided by Polyminer.
  
 ## Links
 

src/core/URandomHash.pas

@@ -269,7 +269,7 @@ begin
  // includes the nonce at fixed offset 4.
 
  // Clone the original header
- Result := System.Copy(ABlockHeader);
+ Result := Copy(ABlockHeader);
 
  // If digest not big enough to contain a nonce, just return the clone
  if Length(ABlockHeader) < (LNonceOffset + 4) then
@@ -292,7 +292,7 @@ var
   i: Int32;
 begin
   FMurmurHash3_x86_32.Initialize;
-  for i := System.Low(AInput) to System.High(AInput) do
+  for i := Low(AInput) to High(AInput) do
   begin
     FMurmurHash3_x86_32.TransformBytes(AInput[i]);
   end;
@@ -307,38 +307,59 @@ var
   LGen: TMersenne32;
   LDisposables : TDisposables;
 begin
-  System.SetLength(Result, 100);
+  SetLength(Result, 100);
   LSeed := Checksum(AInputs);
   LGen := LDisposables.AddObject( TMersenne32.Create( LSeed ) ) as TMersenne32;
   for i := 0 to 99 do
   begin
-    LSource := AInputs[LGen.NextUInt32 mod System.Length(AInputs)];
-    Result[i] := LSource[LGen.NextUInt32 mod System.Length(LSource)];
+    LSource := AInputs[LGen.NextUInt32 mod Length(AInputs)];
+    Result[i] := LSource[LGen.NextUInt32 mod Length(LSource)];
   end;
 end;
 
 function TRandomHash.ContencateByteArrays(const AChunk1, AChunk2: TBytes): TBytes;
 begin
-  System.SetLength(Result, System.Length(AChunk1) + System.Length(AChunk2));
-  System.Move(AChunk1[0], Result[0], System.Length(AChunk1));
-  System.Move(AChunk2[0], Result[System.Length(AChunk1)], System.Length(AChunk2));
+  SetLength(Result, Length(AChunk1) + Length(AChunk2));
+  Move(AChunk1[0], Result[0], Length(AChunk1));
+  Move(AChunk2[0], Result[Length(AChunk1)], Length(AChunk2));
 end;
 
 function TRandomHash.MemTransform1(const AChunk: TBytes): TBytes;
+var
+  i, LChunkLength : UInt32;
+  LState : UInt32;
+
+  function XorShift32 : UInt32; inline;
+  begin
+    LState := LState XOR (LState SHL 13);
+    LState := LState XOR (LState SHR 17);
+    LState := LState XOR (LState SHL 5);
+    Result := LState;
+  end;
+
 begin
-  Result := AChunk;
+  // Seed XorShift32 with non-zero seed (checksum of input or 1)
+  LState := Checksum(AChunk);
+  if LState = 0 then
+    LState := 1;
+
+  // Select random bytes from input using XorShift32 RNG
+  LChunkLength := Length(AChunk);
+  SetLength(Result, LChunkLength);
+  for i := 0 to High(AChunk) do
+    Result[i] := AChunk[XorShift32 MOD LChunkLength];
 end;
 
 function TRandomHash.MemTransform2(const AChunk: TBytes): TBytes;
 var
   i, LChunkLength, LPivot, LOdd: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
+  LChunkLength := Length(AChunk);
   LPivot := LChunkLength SHR 1;
   LOdd := LChunkLength MOD 2;
-  System.SetLength(Result, LChunkLength);
-  System.Move(AChunk[LPivot + LOdd], Result[0], LPivot);
-  System.Move(AChunk[0], Result[LPivot + LOdd], LPivot);
+  SetLength(Result, LChunkLength);
+  Move(AChunk[LPivot + LOdd], Result[0], LPivot);
+  Move(AChunk[0], Result[LPivot + LOdd], LPivot);
   // Set middle-byte for odd-length arrays
   if LOdd = 1 then
     Result[LPivot] := AChunk[LPivot];
@@ -348,8 +369,8 @@ function TRandomHash.MemTransform3(const AChunk: TBytes): TBytes;
 var
   i, LChunkLength: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
-  System.SetLength(Result, LChunkLength);
+  LChunkLength := Length(AChunk);
+  SetLength(Result, LChunkLength);
   for i := 0 to High(AChunk) do
     Result[i] := AChunk[LChunkLength - i - 1];
 end;
@@ -358,11 +379,11 @@ function TRandomHash.MemTransform4(const AChunk: TBytes): TBytes;
 var
   i, LChunkLength, LPivot, LOdd: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
+  LChunkLength := Length(AChunk);
   LPivot := LChunkLength SHR 1;
   LOdd := LChunkLength MOD 2;
-  System.SetLength(Result, LChunkLength);
-  for i := 0 to System.Pred(LPivot) do
+  SetLength(Result, LChunkLength);
+  for i := 0 to Pred(LPivot) do
   begin
     Result[(i * 2)] := AChunk[i];
     Result[(i * 2) + 1] := AChunk[i + LPivot + LOdd];
@@ -376,11 +397,11 @@ function TRandomHash.MemTransform5(const AChunk: TBytes): TBytes;
 var
   i, LChunkLength, LPivot, LOdd: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
+  LChunkLength := Length(AChunk);
   LPivot := LChunkLength SHR 1;
   LOdd := LChunkLength MOD 2;
-  System.SetLength(Result, LChunkLength);
-  for i := System.Low(AChunk) to System.Pred(LPivot) do
+  SetLength(Result, LChunkLength);
+  for i := Low(AChunk) to Pred(LPivot) do
   begin
     Result[(i * 2)] := AChunk[i + LPivot + LOdd];
     Result[(i * 2) + 1] := AChunk[i];
@@ -392,41 +413,40 @@ end;
 
 function TRandomHash.MemTransform6(const AChunk: TBytes): TBytes;
 var
-  i, LChunkLength, LPivot, LOffset: Int32;
+  i, LChunkLength, LPivot, LOdd: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
+  LChunkLength := Length(AChunk);
   LPivot := LChunkLength SHR 1;
-  LOffset := LChunkLength MOD 2;
-  System.SetLength(Result, LChunkLength);
-  for i := 0 to System.Pred(LPivot) do
+  LOdd := LChunkLength MOD 2;
+  SetLength(Result, LChunkLength);
+  for i := 0 to Pred(LPivot) do
   begin
     Result[i] := AChunk[(i * 2)] xor AChunk[(i * 2) + 1];
-    Result[i + LPivot + LOffset] := AChunk[i] xor AChunk[LChunkLength - i - 1];
+    Result[i + LPivot + LOdd] := AChunk[i] xor AChunk[LChunkLength - i - 1];
   end;
   // Set middle-byte for odd-lengths
-  if LOffset = 1 THEN
-    Result[LPivot] := AChunk[LPivot];
-
+  if LOdd = 1 THEN
+    Result[LPivot] := AChunk[High(AChunk)];
 end;
 
 function TRandomHash.MemTransform7(const AChunk: TBytes): TBytes;
 var
   i, LChunkLength: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
-  System.SetLength(Result, LChunkLength);
-  for i := 0 to System.High(AChunk) do
-    Result[i] := TBits.RotateLeft8(AChunk[i], i);
+  LChunkLength := Length(AChunk);
+  SetLength(Result, LChunkLength);
+  for i := 0 to High(AChunk) do
+    Result[i] := TBits.RotateLeft8(AChunk[i], LChunkLength - i);
 end;
 
 function TRandomHash.MemTransform8(const AChunk: TBytes): TBytes;
 var
   i, LChunkLength: Int32;
 begin
-  LChunkLength := System.Length(AChunk);
-  System.SetLength(Result, LChunkLength);
-  for i := 0 to System.High(AChunk) do
-    Result[i] := TBits.RotateRight8(AChunk[i], i);
+  LChunkLength := Length(AChunk);
+  SetLength(Result, LChunkLength);
+  for i := 0 to High(AChunk) do
+    Result[i] := TBits.RotateRight8(AChunk[i], LChunkLength - i);
 end;
 
 function TRandomHash.Expand(const AInput: TBytes; AExpansionFactor: Int32): TBytes;
@@ -439,15 +459,15 @@ var
 begin
   LSeed := Checksum(AInput);
   LGen := LDisposables.AddObject( TMersenne32.Create (LSeed) ) as TMersenne32;
-  LSize := System.Length(AInput) + (AExpansionFactor * M);
-  LOutput := System.Copy(AInput);
-  LBytesToAdd := LSize - System.Length(AInput);
+  LSize := Length(AInput) + (AExpansionFactor * M);
+  LOutput := Copy(AInput);
+  LBytesToAdd := LSize - Length(AInput);
 
   while LBytesToAdd > 0 do
   begin
-    LNextChunk := System.Copy(LOutput);
-    if System.Length(LNextChunk) > LBytesToAdd then
-      System.SetLength(LNextChunk, LBytesToAdd);
+    LNextChunk := Copy(LOutput);
+    if Length(LNextChunk) > LBytesToAdd then
+      SetLength(LNextChunk, LBytesToAdd);
 
     LRandom := LGen.NextUInt32;
     case LRandom mod 8 of
@@ -460,7 +480,7 @@ begin
       6: LOutput := ContencateByteArrays(LOutput, MemTransform7(LNextChunk));
       7: LOutput := ContencateByteArrays(LOutput, MemTransform8(LNextChunk));
     end;
-    LBytesToAdd := LBytesToAdd - System.Length(LNextChunk);
+    LBytesToAdd := LBytesToAdd - Length(LNextChunk);
   end;
   Result := LOutput;
 end;
@@ -477,7 +497,7 @@ var
   i: Int32;
 begin
   Fmt[0] := ASeed;
-  for i := 1 to System.Pred(N) do
+  for i := 1 to Pred(N) do
     Fmt[i] := F * (Fmt[i - 1] xor (Fmt[i - 1] shr 30)) + i;
   FIndex := N;
 end;

src/tests/PascalCoinUnitTests.lpi

@@ -101,13 +101,6 @@
         <OptimizationLevel Value="4"/>
       </Optimizations>
     </CodeGeneration>
-    <Linking>
-      <Options>
-        <Win32>
-          <GraphicApplication Value="True"/>
-        </Win32>
-      </Options>
-    </Linking>
     <Other>
       <CustomOptions Value="-dUNITTESTS"/>
     </Other>