AbstractMem v1.5 - Allow 32 or 64bits (More than 4Gb limit)

src/libraries/abstractmem/ConfigAbstractMem.inc

@@ -57,12 +57,18 @@
   - Added TAbstractMemZoneInfo that allows to Analyze TAbstractMem using CheckConsistency and returns all blocks information
   - Added TAVLCacheStats that allows to obtain stats from any TAVLCache<T> object
   
-  Version 1.4 -
+  Version 1.4 - Oct 2021
   - Fixed but to allow files up to 4 Gb. Limitation is due to internal 4 bytes positioning. 4 bytes = 4 Gb maximum size
 
+  Version 1.5 - Nov 2021
+  - Allows 32 or 64 bits file (More than 4Gb limitation)
+  - Allows select Max allowed block size by setting units size:
+    - 32 bits: Fixed 1 unit = 4 bytes: Max size = ((2^24)*4) = 67108864 bytes = 64Mb
+    - 64 bits: Customizable: 1 unit from 4..256 bytes (multiple of 4): Max size = from ((2^32)*4) 16Gb ... ((2^32)*256) = 1099511627776 = 1Tb
+
 }
 
 const
-  CT_ABSTRACTMEM_VERSION = 1.4; // Each revision should increase this version...
+  CT_ABSTRACTMEM_VERSION = 1.5; // Each revision should increase this version...
 
 

src/libraries/abstractmem/UAbstractBTree.pas

@@ -145,25 +145,25 @@ type
     procedure Unlock;
   End;
 
-  TMemoryBTree<TData> = Class( TAbstractBTree<TAbstractMemPosition,TData> )
+  TMemoryBTree<TData> = Class( TAbstractBTree<Integer,TData> )
   private
-    FBuffer : TList<TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode> ;
+    FBuffer : TList<TAbstractBTree<Integer,TData>.TAbstractBTreeNode> ;
     Froot : Integer;
     FDisposed : Integer;
     FDisposedMinPos : Integer;
   protected
-    function GetRoot: TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode; override;
-    procedure SetRoot(var Value: TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode); override;
-    function NewNode : TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode; override;
-    procedure DisposeNode(var ANode : TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode); override;
-    procedure SetNil(var AIdentify : TAbstractMemPosition); override;
-    procedure SaveNode(var ANode : TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode); override;
-    procedure CheckConsistencyFinalized(ADatas : TList<TData>; AIdents : TOrderedList<TAbstractMemPosition>; Alevels, ANodesCount, AItemsCount : Integer); override;
+    function GetRoot: TAbstractBTree<Integer,TData>.TAbstractBTreeNode; override;
+    procedure SetRoot(var Value: TAbstractBTree<Integer,TData>.TAbstractBTreeNode); override;
+    function NewNode : TAbstractBTree<Integer,TData>.TAbstractBTreeNode; override;
+    procedure DisposeNode(var ANode : TAbstractBTree<Integer,TData>.TAbstractBTreeNode); override;
+    procedure SetNil(var AIdentify : Integer); override;
+    procedure SaveNode(var ANode : TAbstractBTree<Integer,TData>.TAbstractBTreeNode); override;
+    procedure CheckConsistencyFinalized(ADatas : TList<TData>; AIdents : TOrderedList<Integer>; Alevels, ANodesCount, AItemsCount : Integer); override;
   public
-    function IsNil(const AIdentify : TAbstractMemPosition) : Boolean; override;
+    function IsNil(const AIdentify : Integer) : Boolean; override;
     constructor Create(const AOnCompareDataMethod: TComparison<TData>; AAllowDuplicates : Boolean; AOrder : Integer);
     destructor Destroy; override;
-    function GetNode(AIdentify : TAbstractMemPosition) : TAbstractBTree<TAbstractMemPosition,TData>.TAbstractBTreeNode; override;
+    function GetNode(AIdentify : Integer) : TAbstractBTree<Integer,TData>.TAbstractBTreeNode; override;
     property Count;
   End;
 
@@ -1379,7 +1379,7 @@ end;
 
 { TMemoryBTree<TData> }
 
-procedure TMemoryBTree<TData>.CheckConsistencyFinalized(ADatas: TList<TData>; AIdents: TOrderedList<TAbstractMemPosition>; Alevels, ANodesCount, AItemsCount: Integer);
+procedure TMemoryBTree<TData>.CheckConsistencyFinalized(ADatas: TList<TData>; AIdents: TOrderedList<Integer>; Alevels, ANodesCount, AItemsCount: Integer);
 var i,iPos,nDisposed, LDisposedMinPos : Integer;
 begin
   inherited;
@@ -1405,7 +1405,7 @@ constructor TMemoryBTree<TData>.Create(const AOnCompareDataMethod: TComparison<T
 begin
   FBuffer := TList<TAbstractBTreeNode>.Create;
   Froot := -1;
-  inherited Create(TComparison_TAbstractMemPosition,AOnCompareDataMethod,AAllowDuplicates,AOrder);
+  inherited Create(TComparison_Integer,AOnCompareDataMethod,AAllowDuplicates,AOrder);
   FCount := 0;
   FDisposed := 0;
   FDisposedMinPos := -1;
@@ -1418,7 +1418,7 @@ begin
   inherited;
 end;
 
-procedure TMemoryBTree<TData>.DisposeNode(var ANode: TAbstractBTree<TAbstractMemPosition, TData>.TAbstractBTreeNode);
+procedure TMemoryBTree<TData>.DisposeNode(var ANode: TAbstractBTree<Integer, TData>.TAbstractBTreeNode);
 var Lpos : Integer;
 begin
   Lpos := ANode.identify;
@@ -1429,13 +1429,13 @@ begin
   if (FDisposedMinPos<0) or (FDisposedMinPos>Lpos) then FDisposedMinPos := Lpos;
 end;
 
-function TMemoryBTree<TData>.GetNode(AIdentify: TAbstractMemPosition): TAbstractBTree<TAbstractMemPosition, TData>.TAbstractBTreeNode;
+function TMemoryBTree<TData>.GetNode(AIdentify: Integer): TAbstractBTree<Integer, TData>.TAbstractBTreeNode;
 begin
   Result := FBuffer[AIdentify];
   if (Result.identify<>AIdentify) then raise EAbstractBTree.Create(Format('Found %d Identify instead of %d',[Result.identify,AIdentify]));
 end;
 
-function TMemoryBTree<TData>.GetRoot: TAbstractBTree<TAbstractMemPosition, TData>.TAbstractBTreeNode;
+function TMemoryBTree<TData>.GetRoot: TAbstractBTree<Integer, TData>.TAbstractBTreeNode;
 begin
   if (Froot<0) then begin
     ClearNode(Result);
@@ -1444,12 +1444,12 @@ begin
   Result := GetNode(Froot);
 end;
 
-function TMemoryBTree<TData>.IsNil(const AIdentify: TAbstractMemPosition): Boolean;
+function TMemoryBTree<TData>.IsNil(const AIdentify: Integer): Boolean;
 begin
   Result := AIdentify<0;
 end;
 
-function TMemoryBTree<TData>.NewNode: TAbstractBTree<TAbstractMemPosition, TData>.TAbstractBTreeNode;
+function TMemoryBTree<TData>.NewNode: TAbstractBTree<Integer, TData>.TAbstractBTreeNode;
 begin
   ClearNode(Result);
   if (FDisposed > 0) And (FDisposed > (Count DIV 5)) then begin // 20% max disposed nodes
@@ -1469,7 +1469,7 @@ begin
   FBuffer.Insert(Result.identify,Result);
 end;
 
-procedure TMemoryBTree<TData>.SaveNode(var ANode: TAbstractBTree<TAbstractMemPosition, TData>.TAbstractBTreeNode);
+procedure TMemoryBTree<TData>.SaveNode(var ANode: TAbstractBTree<Integer, TData>.TAbstractBTreeNode);
 begin
   if (ANode.identify<0) then begin
     raise EAbstractBTree.Create('Save undefined node '+ToString(ANode));
@@ -1481,12 +1481,12 @@ begin
   end;
 end;
 
-procedure TMemoryBTree<TData>.SetNil(var AIdentify: TAbstractMemPosition);
+procedure TMemoryBTree<TData>.SetNil(var AIdentify: Integer);
 begin
   AIdentify := -1;
 end;
 
-procedure TMemoryBTree<TData>.SetRoot(var Value: TAbstractBTree<TAbstractMemPosition, TData>.TAbstractBTreeNode);
+procedure TMemoryBTree<TData>.SetRoot(var Value: TAbstractBTree<Integer, TData>.TAbstractBTreeNode);
 begin
   Froot := Value.identify;
 end;

src/libraries/abstractmem/UFileMem.pas

@@ -81,7 +81,7 @@ type
   public
     Constructor Create(const AFileName : String; AReadOnly : Boolean); reintroduce;
     Destructor Destroy; override;
-    function New(AMemSize : Integer) : TAMZone; override;
+    function New(AMemSize : TAbstractMemSize) : TAMZone; override;
     function Write(const APosition : Int64; const ABuffer; ASize : Integer) : Integer; overload; override;
     function Read(const APosition : Int64; var ABuffer; ASize : Integer) : Integer; overload; override;
     {$IFDEF ABSTRACTMEM_TESTING_MODE}
@@ -291,7 +291,7 @@ begin
   Result := FCache;
 end;
 
-function TFileMem.New(AMemSize: Integer): TAMZone;
+function TFileMem.New(AMemSize: TAbstractMemSize): TAMZone;
 var LBuffer : TBytes;
 begin
   Result := inherited New(AMemSize);

src/libraries/abstractmem/UOrderedList.pas

@@ -42,10 +42,7 @@ uses
 
 {$I ./ConfigAbstractMem.inc }
 
-
 type
-  TAbstractMemPosition = Int64;
-
   {$IFDEF FPC}
   TComparison<T> = function(const Left, Right: T): Integer;
   {$ENDIF}
@@ -241,3 +238,4 @@ end.
 
 
 
+

src/libraries/abstractmem/tests/src/UAbstractBTree.Tests.pas

@@ -17,12 +17,12 @@ uses
 
 type
 
-  TIntegerBTree = Class( TMemoryBTree<TAbstractMemPosition> )
+  TIntegerBTree = Class( TMemoryBTree<Integer> )
   private
   protected
   public
     constructor Create(AAllowDuplicates : Boolean; AOrder : Integer);
-    function NodeDataToString(const AData : TAbstractMemPosition) : String; override;
+    function NodeDataToString(const AData : Integer) : String; override;
   End;
 
 
@@ -51,10 +51,10 @@ implementation
 
 constructor TIntegerBTree.Create(AAllowDuplicates: Boolean; AOrder: Integer);
 begin
-  inherited Create(TComparison_TAbstractMemPosition,AAllowDuplicates,AOrder);
+  inherited Create(TComparison_Integer,AAllowDuplicates,AOrder);
 end;
 
-function TIntegerBTree.NodeDataToString(const AData: TAbstractMemPosition): String;
+function TIntegerBTree.NodeDataToString(const AData: Integer): String;
 begin
   Result := AData.ToString;
 end;
@@ -200,7 +200,7 @@ end;
 procedure TestTAbstractBTree.TestPrecessorSuccessor;
 var Lbt : TIntegerBTree;
   Lorder : Integer;
-  i, intValue, valMin, valMax, Lregs : TAbstractMemPosition;
+  i, intValue, valMin, valMax, Lregs : Integer;
 
 begin
   for Lorder := 3 to 7 do begin
@@ -245,7 +245,7 @@ end;
 procedure TestTAbstractBTree.TestPrecessorSuccessor_Duplicates;
 var Lbt : TIntegerBTree;
   Lorder,
-  i, intValue, valMin, valMax, Lregs : TAbstractMemPosition;
+  i, intValue, valMin, valMax, Lregs : Integer;
 begin
   for Lorder := 3 to 7 do begin
     Lbt := TIntegerBTree.Create(True,Lorder);
@@ -285,27 +285,17 @@ end;
 
 procedure TestTAbstractBTree.Test_duplicate;
 var Lbt : TIntegerBTree;
-  Lorder, i, intValue : TAbstractMemPosition;
-  LLastTree, LCurrentTree : String;
+  Lorder, i, intValue : Integer;
+  LLastTree,LCurrentTree : String;
 
   procedure DoInsert(AValue : Integer);
   begin
     Lbt.Add(AValue);
-    {
-    LCurrentTree := Lbt.BTreeToString;
-    Lbt.CheckConsistency;
-    LLastTree := LCurrentTree;
-    }
   end;
 
   procedure DoDelete(AValue : Integer);
   begin
     Lbt.Delete(AValue);
-    {
-    LCurrentTree := Lbt.BTreeToString;
-    Lbt.CheckConsistency;
-    LLastTree := LCurrentTree;
-    }
   end;
 
 begin
@@ -348,7 +338,7 @@ end;
 
 procedure TestTAbstractBTree.TestDelete;
 var Lbt : TIntegerBTree;
-  Lorder, i, intValue : TAbstractMemPosition;
+  Lorder, i, intValue : Integer;
   LLastTree, LCurrentTree : String;
 
   procedure DoDelete(AValue : Integer);

src/libraries/abstractmem/tests/src/UAbstractMem.Tests.pas

@@ -23,6 +23,7 @@ interface
    public
      procedure SetUp; override;
      procedure TearDown; override;
+     procedure Test_MemLeaks(AAbstractMem: TAbstractMem);
    published
      procedure Test_ClearContent;
      procedure Test_MemLeaksReuse;
@@ -39,55 +40,59 @@ begin
 end;
 
 procedure TestTAbstractMem.Test_ClearContent;
-var Lfm : TFileMem;
+var LAM : TAbstractMem;
 begin
-  Lfm := TFileMem.Create(ExtractFileDir(ParamStr(0))+PathDelim+'test1.am',False);
+  LAM := TMem.Create(0,False);
   try
-    Lfm.ClearContent; // Init
+    LAM.ClearContent(False,4); // Init
+    LAM.ClearContent(True,16); // Init
+    LAM.CheckConsistency;
   finally
-    Lfm.Free;
+    LAM.Free;
   end;
 end;
 
 
-procedure TestTAbstractMem.Test_MemLeaksReuse;
-var LAM : TAbstractMem;
+procedure TestTAbstractMem.Test_MemLeaks(AAbstractMem: TAbstractMem);
+var
   LAMs : TList<TAMZone>;
   i,j, loops : Integer;
-  LStrings : TStrings;
-  LAbstractMemZoneInfoList : TList<TAbstractMemZoneInfo>;
-  LTotalUsedSize, LTotalUsedBlocksCount, LTotalLeaksSize, LTotalLeaksBlocksCount : TAbstractMemPosition;
 begin
-  LAM := TMem.Create(0,False);
-  try
-    LAMs := TList<TAMZone>.Create;
-    Try
-      for loops := 1 to 2 do begin
+  LAMs := TList<TAMZone>.Create;
+  Try
+    for loops := 1 to 2 do begin
 
       LAMs.Clear;
 
       for j := 1 to 10000 do begin
-        LAMs.Add( LAM.New(Random(1000)+10) );
+        LAMs.Add( AAbstractMem.New(Random(1000)+10) );
       end;
 
       //
       for i := 0 to LAMs.Count-1 do begin
-        LAM.Dispose( LAMs.Items[i] );
+        AAbstractMem.Dispose( LAMs.Items[i] );
       end;
 
-      end;
+      AAbstractMem.CheckConsistency;
+    end;
 
-      LStrings := TStringList.Create;
-      LAbstractMemZoneInfoList := TList<TAbstractMemZoneInfo>.Create;
-      try
-        if Not LAM.CheckConsistency(LStrings,LAbstractMemZoneInfoList,LTotalUsedSize, LTotalUsedBlocksCount, LTotalLeaksSize, LTotalLeaksBlocksCount) then raise Exception.Create(LStrings.Text);
-      finally
-        LAbstractMemZoneInfoList.Free;
-        LStrings.Free;
-      end;
-    Finally
-      LAMs.Free;
-    End;
+  Finally
+    LAMs.Free;
+  End;
+end;
+
+procedure TestTAbstractMem.Test_MemLeaksReuse;
+var LAM : TAbstractMem;
+begin
+  LAM := TMem.Create(0,False);
+  try
+    LAM.Initialize(False,4);
+    Test_MemLeaks(LAM);
+    LAM.Initialize(True,4);
+    Test_MemLeaks(LAM);
+    LAM.Initialize(True,16);
+    Test_MemLeaks(LAM);
+    LAM.Initialize(True,64);
   finally
     LAM.Free;
   end;

src/libraries/abstractmem/tests/src/UAbstractMemBTree.Tests.pas

@@ -138,7 +138,7 @@ var Lbt : TAbstractMemBTreeExampleInteger;
   Lzone : TAMZone;
   intValue, nRounds, nAdds, nDeletes, i : Integer;
   j : TAbstractMemPosition;
-  Lnode : TIntegerBTree.TAbstractBTreeNode;
+  Lnode : TAbstractMemBTreeExampleInteger.TAbstractBTreeNode;
   Lmem : TAbstractMem;
   LCurr : String;
 begin
@@ -217,7 +217,7 @@ procedure TestTAbstractMemBTree.TestInfinite_String(AOrder: Integer; AAllowDupli
 var Lbt : TAbstractMemBTreeExampleString;
   Lzone : TAMZone;
   intValue, nRounds, nAdds, nDeletes, i : Integer;
-  Lnode : TIntegerBTree.TAbstractBTreeNode;
+  Lnode : TAbstractMemBTreeExampleString.TAbstractBTreeNode;
   Lmem : TAbstractMem;
   LCurr : String;
   LCurrData : String;

src/libraries/abstractmem/tests/src/UFileMem.Tests.pas

@@ -12,6 +12,7 @@ interface
    fpcunit, testutils, testregistry,
    {$ELSE}
    TestFramework,
+   System.IOUtils,
    {$ENDIF}
    {$IFNDEF FPC}System.Generics.Collections,System.Generics.Defaults,{$ELSE}Generics.Collections,Generics.Defaults,{$ENDIF}
    UFileMem, UAbstractMem, UCacheMem, UOrderedList;
@@ -70,9 +71,11 @@ end;
 
 function TestTFileMem.GetFullFileName(AFileName: String): String;
 begin
-//  Result := 'C:\Users\Albert\Desktop\TEMP\'+AFileName; // XXXXXXXXXXXXXXXX
-  Result := ExtractFileDir(ParamStr(0))+PathDelim+AFileName;
-// XXXXXXXXXXXXXXXX
+  {$IFDEF FPC}
+  Result := GetTempDir+PathDelim+AFileName;
+  {$ELSE}
+  Result := System.IOUtils.TPath.GetTempPath+PathDelim+AFileName;
+  {$ENDIF}
 end;
 
 procedure TestTFileMem.SetUp;
@@ -170,7 +173,7 @@ begin
   Lfm := TFileMem.Create(GetFullFileName('test_FileMem_Aux.am'),False);
   Lfs := TStringList.Create;
   try
-    Lfm.ClearContent;
+    Lfm.ClearContent(False,4);
 //    Lfm.UseCache := AUseCache;
     if AUseCache then begin
       Lfm.MaxCacheSize := 1024 * 1024 * 2; // 2 Mb