* prefetch heap status in sysfreemem

  * re-layouted code in sysfreemem_fixed so that the most likely case is on
    the fall-through path
  * moved dec(poc^.used) in sysfreemem_fixed earlier so that it doesn't
    cause a dependency stall for the "if poc^.used<=0" test later

git-svn-id: trunk@15515 -

rtl/inc/heap.inc

@@ -1059,34 +1059,42 @@ var
   chunksize: ptruint;
   poc: poschunk;
   pmc_next: pmemchunk_fixed;
+  pocfreelists: pfreelists;
 begin
   poc := poschunk(pointer(pmc)-(pmc^.size shr fixedoffsetshift));
+  { start memory access to poc^.freelists already }
+  pocfreelists := poc^.freelists;
   chunksize := pmc^.size and fixedsizemask;
-  if loc_freelists <> poc^.freelists then
-  begin
-    { deallocated in wrong thread! add to to-be-freed list of correct thread }
-    waitfree_fixed(pmc, poc);
-    exit(chunksize);
-  end;
+  if loc_freelists = pocfreelists then
+    begin
+      { decrease used blocks count (well in advance of poc^.used check below,
+        to avoid stalling due to a dependency) }
+      dec(poc^.used);
 
-  dec(loc_freelists^.internal_status.currheapused, chunksize);
-  { insert the block in its freelist }
-  chunkindex := chunksize shr blockshift;
-  pmc_next := loc_freelists^.fixedlists[chunkindex];
-  pmc^.prev_fixed := nil;
-  pmc^.next_fixed := pmc_next;
-  if assigned(pmc_next) then
-    pmc_next^.prev_fixed := pmc;
-  loc_freelists^.fixedlists[chunkindex] := pmc;
-  { decrease used blocks count }
-  dec(poc^.used);
-  if poc^.used <= 0 then
+      { insert the block in its freelist }
+      chunkindex := chunksize shr blockshift;
+      pmc_next := loc_freelists^.fixedlists[chunkindex];
+      pmc^.prev_fixed := nil;
+      pmc^.next_fixed := pmc_next;
+      if assigned(pmc_next) then
+        pmc_next^.prev_fixed := pmc;
+      loc_freelists^.fixedlists[chunkindex] := pmc;
+
+      dec(loc_freelists^.internal_status.currheapused, chunksize);
+
+      if poc^.used <= 0 then
+        begin
+          { decrease used blocks count }
+          if poc^.used<0 then
+            HandleError(204);
+          { osblock can be freed? }
+          append_to_oslist(poc);
+        end;
+    end
+  else
     begin
-      { decrease used blocks count }
-      if poc^.used<0 then
-        HandleError(204);
-      { osblock can be freed? }
-      append_to_oslist(poc);
+      { deallocated in wrong thread! add to to-be-freed list of correct thread }
+      waitfree_fixed(pmc, poc);
     end;
   result := chunksize;
 end;
@@ -1103,13 +1111,13 @@ begin
     exit(chunksize);
   end;
 
-  dec(loc_freelists^.internal_status.currheapused, chunksize);
   { insert the block in it's freelist }
   pmcv^.size := pmcv^.size and (not usedflag);
   append_to_list_var(pmcv);
   pmcv := try_concat_free_chunk(pmcv);
   if (pmcv^.size and (firstblockflag or lastblockflag)) = (firstblockflag or lastblockflag) then
     append_to_oslist_var(pmcv);
+  dec(loc_freelists^.internal_status.currheapused, chunksize);
   result := chunksize;
 end;
 
@@ -1138,6 +1146,7 @@ begin
 {$endif}
   { loc_freelists is a threadvar, so it can be worth it to prefetch }
   loc_freelists := @freelists;
+  prefetch(loc_freelists^.internal_status.currheapused);
   { check if this is a fixed- or var-sized chunk }
   if (pmc^.size and fixedsizeflag) = 0 then
     result := sysfreemem_var(loc_freelists, pmemchunk_var(p-sizeof(tmemchunk_var_hdr)))