Make sure it always downloads archives just once and not a second time on retries because of error with other archive.

Also fixes "SetDownloadTemporaryFileCredentials('', '');" in TDownloadWizardPage.Download not being reached on download errors.

Projects/Src/Setup.Install.pas

@@ -37,7 +37,10 @@ function DownloadFile(const Url, CustomUserName, CustomPassword: String;
   const OnSimpleDownloadProgress: TOnSimpleDownloadProgress;
   const OnSimpleDownloadProgressParam: Integer64): Int64;
 function DownloadTemporaryFile(const Url, BaseName: String;
-  [ref] const Verification: TSetupFileVerification; const OnDownloadProgress: TOnDownloadProgress): Int64;
+  [ref] const Verification: TSetupFileVerification; const OnDownloadProgress: TOnDownloadProgress): Int64; overload;
+function DownloadTemporaryFile(const Url, BaseName: String;
+  [ref] const Verification: TSetupFileVerification; const OnDownloadProgress: TOnDownloadProgress;
+  out DestFile: String): Int64; overload;
 function DownloadTemporaryFileSize(const Url: String): Int64;
 function DownloadTemporaryFileDate(const Url: String): String;
 procedure SetDownloadTemporaryFileCredentials(const User, Pass: String);
@@ -3901,9 +3904,10 @@ begin
 end;
 
 function DownloadTemporaryFile(const Url, BaseName: String;
-  [ref] const Verification: TSetupFileVerification; const OnDownloadProgress: TOnDownloadProgress): Int64;
+  [ref] const Verification: TSetupFileVerification; const OnDownloadProgress: TOnDownloadProgress;
+  out DestFile: String): Int64;
 var
-  DestFile, TempFile: String;
+  TempFile: String;
   TempF: TFile;
   HandleStream: THandleStream;
   TempFileLeftOver: Boolean;
@@ -4063,6 +4067,13 @@ begin
   end;
 end;
 
+function DownloadTemporaryFile(const Url, BaseName: String;
+  [ref] const Verification: TSetupFileVerification; const OnDownloadProgress: TOnDownloadProgress): Int64;
+begin
+  var DestFile: String;
+  Result := DownloadTemporaryFile(Url, BaseName, Verification, OnDownloadProgress, DestFile);
+end;
+
 procedure DownloadTemporaryFileSizeAndDate(const Url: String; var FileSize: Int64; var FileDate: String);
 var
   HTTPClient: THTTPClient;

Projects/Src/Setup.ScriptDlg.pas

@@ -175,9 +175,13 @@ type
   TDownloadFile = class
     Url, BaseName, UserName, Password: String;
     Verification: TSetupFileVerification;
+    Data: NativeUInt;
   end;
   TDownloadFiles = TObjectList<TDownloadFile>;
 
+  TDownloadFileCompleted = reference to procedure(const DownloadedFile: TDownloadFile;
+    const DestFile: String; var Remove: Boolean);
+
   TDownloadWizardPage = class(TOutputProgressWizardPage)
     private
       FFiles: TDownloadFiles;
@@ -186,8 +190,8 @@ type
       FAbortButton: TNewButton;
       FShowProgressControlsOnNextProgress, FAbortedByUser: Boolean;
       function DoAdd(const Url, BaseName, RequiredSHA256OfFile: String;
-        const UserName: String = ''; const Password: String = '';
-        const ISSigVerify: Boolean = False; const ISSigAllowedKeys: AnsiString = ''): Integer;
+        const UserName, Password: String; const ISSigVerify: Boolean;
+        const ISSigAllowedKeys: AnsiString; const Data: NativeUInt): Integer;
       procedure AbortButtonClick(Sender: TObject);
       function InternalOnDownloadProgress(const Url, BaseName: string; const Progress, ProgressMax: Int64): Boolean;
       procedure ShowProgressControls(const AVisible: Boolean);
@@ -198,13 +202,14 @@ type
       function Add(const Url, BaseName, RequiredSHA256OfFile: String): Integer;
       function AddWithISSigVerify(const Url, ISSigUrl, BaseName: String;
         const AllowedKeysRuntimeIDs: TStringList): Integer;
-      function AddEx(const Url, BaseName, RequiredSHA256OfFile, UserName, Password: String): Integer;
+      function AddEx(const Url, BaseName, RequiredSHA256OfFile, UserName, Password: String;
+        const Data: NativeUInt): Integer;
       function AddExWithISSigVerify(const Url, ISSigUrl, BaseName, UserName, Password: String;
-        const AllowedKeysRuntimeIDs: TStringList): Integer; overload;
+        const AllowedKeysRuntimeIDs: TStringList; const Data: NativeUInt): Integer; overload;
       function AddExWithISSigVerify(const Url, ISSigUrl, BaseName, UserName, Password: String;
-        const ISSigAllowedKeys: AnsiString): Integer; overload;
+        const ISSigAllowedKeys: AnsiString; const Data: NativeUInt): Integer; overload;
       procedure Clear;
-      function Download: Int64;
+      function Download(const OnDownloadFileCompleted: TDownloadFileCompleted): Int64;
       property OnDownloadProgress: TOnDownloadProgress write FOnDownloadProgress;
       procedure Show; override;
     published
@@ -1059,7 +1064,7 @@ begin
 end;
 
 function TDownloadWizardPage.DoAdd(const Url, BaseName, RequiredSHA256OfFile, UserName, Password: String;
-  const ISSigVerify: Boolean; const ISSigAllowedKeys: AnsiString): Integer;
+  const ISSigVerify: Boolean; const ISSigAllowedKeys: AnsiString; const Data: NativeUInt): Integer;
 begin
   var F := TDownloadFile.Create;
   F.Url := Url;
@@ -1074,6 +1079,7 @@ begin
     F.Verification.Typ := fvISSig;
     F.Verification.ISSigAllowedKeys := ISSigAllowedKeys
   end;
+  F.Data := Data;
   Result := FFiles.Add(F);
 end;
 
@@ -1102,33 +1108,34 @@ end;
 
 function TDownloadWizardPage.Add(const Url, BaseName, RequiredSHA256OfFile: String): Integer;
 begin
-  Result := DoAdd(Url, BaseName, RequiredSHA256OfFile);
+  Result := DoAdd(Url, BaseName, RequiredSHA256OfFile, '', '', False, '', 0);
 end;
 
 function TDownloadWizardPage.AddWithISSigVerify(const Url, ISSigUrl, BaseName: String;
   const AllowedKeysRuntimeIDs: TStringList): Integer;
 begin
-  Result := AddExWithISSigVerify(Url, ISSigUrl, BaseName, '', '', AllowedKeysRuntimeIDs);
+  Result := AddExWithISSigVerify(Url, ISSigUrl, BaseName, '', '', AllowedKeysRuntimeIDs, 0);
 end;
 
-function TDownloadWizardPage.AddEx(const Url, BaseName, RequiredSHA256OfFile, UserName, Password: String): Integer;
+function TDownloadWizardPage.AddEx(const Url, BaseName, RequiredSHA256OfFile, UserName, Password: String;
+  const Data: NativeUInt): Integer;
 begin
-  Result := DoAdd(Url, BaseName, RequiredSHA256OfFile, UserName, Password);
+  Result := DoAdd(Url, BaseName, RequiredSHA256OfFile, UserName, Password, False, '', Data);
 end;
 
 function TDownloadWizardPage.AddExWithISSigVerify(const Url, ISSigUrl, BaseName, UserName,
-  Password: String; const AllowedKeysRuntimeIDs: TStringList): Integer;
+  Password: String; const AllowedKeysRuntimeIDs: TStringList; const Data: NativeUInt): Integer;
 begin
   const ISSigAllowedKeys = ConvertAllowedKeysRuntimeIDsToISSigAllowedKeys(AllowedKeysRuntimeIDs);
-  Result := AddExWithISSigVerify(Url, ISSigUrl, BaseName, UserName, Password, ISSigAllowedKeys);
+  Result := AddExWithISSigVerify(Url, ISSigUrl, BaseName, UserName, Password, ISSigAllowedKeys, 0);
 end;
 
 function TDownloadWizardPage.AddExWithISSigVerify(const Url, ISSigUrl, BaseName, UserName,
-  Password: String; const ISSigAllowedKeys: AnsiString): Integer;
+  Password: String; const ISSigAllowedKeys: AnsiString; const Data: NativeUInt): Integer;
 begin
   { Also see Setup.ScriptFunc DownloadTemporaryFileWithISSigVerify }
-  DoAdd(GetISSigUrl(Url, ISSigUrl), BaseName + ISSigExt, '', UserName, Password, False, '');
-  Result := DoAdd(Url, BaseName, '', UserName, Password, True, ISSigAllowedKeys);
+  DoAdd(GetISSigUrl(Url, ISSigUrl), BaseName + ISSigExt, '', UserName, Password, False, '', 0);
+  Result := DoAdd(Url, BaseName, '', UserName, Password, True, ISSigAllowedKeys, Data);
 end;
 
 procedure TDownloadWizardPage.Clear;
@@ -1136,17 +1143,30 @@ begin
   FFiles.Clear;
 end;
 
-function TDownloadWizardPage.Download: Int64;
+function TDownloadWizardPage.Download(const OnDownloadFileCompleted: TDownloadFileCompleted): Int64;
 begin
   FAbortedByUser := False;
 
   Result := 0;
-  for var F in FFiles do begin
-    { Don't need to set DownloadTemporaryFileOrExtractArchiveProcessMessages before downloading since we already process messages ourselves }
-    SetDownloadTemporaryFileCredentials(F.UserName, F.Password);
-    Result := Result + DownloadTemporaryFile(F.Url, F.BaseName, F.Verification, InternalOnDownloadProgress);
+
+  try
+    for var I := 0 to FFiles.Count-1 do begin
+      { Don't need to set DownloadTemporaryFileOrExtractArchiveProcessMessages before downloading since we already process messages ourselves }
+      const F = FFiles[I];
+      SetDownloadTemporaryFileCredentials(F.UserName, F.Password);
+      var DestFile: String;
+      Result := Result + DownloadTemporaryFile(F.Url, F.BaseName, F.Verification, InternalOnDownloadProgress, DestFile);
+      if Assigned(OnDownloadFileCompleted) then begin
+        var Remove := False;
+        OnDownloadFileCompleted(F, DestFile, Remove);
+        if Remove then
+          FFiles[I] := nil;
+      end;
+    end;
+  finally
+    SetDownloadTemporaryFileCredentials('', '');
+    FFiles.Pack;
   end;
-  SetDownloadTemporaryFileCredentials('', '');
 end;
 
 {--- Extraction ---}

Projects/Src/Setup.WizardForm.pas

@@ -1858,75 +1858,68 @@ function TWizardForm.PrepareToInstall(const WizardComponents, WizardTasks: TStri
   begin
     var DownloadPage: TDownloadWizardPage := nil;
 
-    const ArchivesToDownload = TDictionary<Integer, String>.Create;
-    try
-      for var I := 0 to Entries[seFile].Count-1 do begin
-        const FileEntry: PSetupFileEntry = Entries[seFile][I];
-        if (foDownload in FileEntry.Options) and (foExtractArchive in FileEntry.Options) and
-           ShouldProcessFileEntry(SelectedComponents, SelectedComponents, FileEntry, False) then begin
-          if DownloadPage = nil then
-            DownloadPage := GetClearedDownloadArchivesPage;
-          if not(foCustomDestName in FileEntry.Options) then
-            InternalError('Expected CustomDestName flag');
-          { Prepare }
-          const TempDir = AddBackslash(TempInstallDir);
-          const DestDir = GenerateUniqueName(False, TempDir + '_isetup', '.tmp');
-          const DestFile = AddBackslash(DestDir) + PathExtractName(FileEntry.DestName);
-          const BaseName = Copy(DestFile, Length(TempDir)+1, MaxInt);
-           { Add to ArchivesToDownload }
-          ArchivesToDownload.Add(I, DestFile);
-          { Add to DownloadPage }
-          const Url = ExpandConst(FileEntry.SourceFilename);
-          const UserName = ExpandConst(FileEntry.DownloadUserName);
-          const Password = ExpandConst(FileEntry.DownloadPassword);
-          if FileEntry.Verification.Typ = fvISSig then begin
-            const ISSigUrl = GetISSigUrl(Url, ExpandConst(FileEntry.DownloadISSigSource));
-            DownloadPage.AddExWithISSigVerify(Url, ISSigUrl, BaseName, UserName, Password,
-              FileEntry.Verification.ISSigAllowedKeys)
-          end else begin
-            var RequiredSHA256OfFile: String;
-            if FileEntry.Verification.Typ = fvHash then
-              RequiredSHA256OfFile := SHA256DigestToString(FileEntry.Verification.Hash)
-            else
-              RequiredSHA256OfFile := '';
-            DownloadPage.AddEx(Url, BaseName, RequiredSHA256OfFile, UserName, Password);
-          end;
+    for var I := 0 to Entries[seFile].Count-1 do begin
+      const FileEntry: PSetupFileEntry = Entries[seFile][I];
+      if (foDownload in FileEntry.Options) and (foExtractArchive in FileEntry.Options) and
+         ShouldProcessFileEntry(SelectedComponents, SelectedComponents, FileEntry, False) then begin
+        if DownloadPage = nil then
+          DownloadPage := GetClearedDownloadArchivesPage;
+        if not(foCustomDestName in FileEntry.Options) then
+          InternalError('Expected CustomDestName flag');
+        { Prepare }
+        const TempDir = AddBackslash(TempInstallDir);
+        const DestDir = GenerateUniqueName(False, TempDir + '_isetup', '.tmp');
+        const DestFile = AddBackslash(DestDir) + PathExtractName(FileEntry.DestName);
+        const BaseName = Copy(DestFile, Length(TempDir)+1, MaxInt);
+        { Add to DownloadPage }
+        const Url = ExpandConst(FileEntry.SourceFilename);
+        const UserName = ExpandConst(FileEntry.DownloadUserName);
+        const Password = ExpandConst(FileEntry.DownloadPassword);
+        if FileEntry.Verification.Typ = fvISSig then begin
+          const ISSigUrl = GetISSigUrl(Url, ExpandConst(FileEntry.DownloadISSigSource));
+          DownloadPage.AddExWithISSigVerify(Url, ISSigUrl, BaseName, UserName, Password,
+            FileEntry.Verification.ISSigAllowedKeys, I)
+        end else begin
+          var RequiredSHA256OfFile: String;
+          if FileEntry.Verification.Typ = fvHash then
+            RequiredSHA256OfFile := SHA256DigestToString(FileEntry.Verification.Hash)
+          else
+            RequiredSHA256OfFile := '';
+          DownloadPage.AddEx(Url, BaseName, RequiredSHA256OfFile, UserName, Password, I);
         end;
       end;
+    end;
 
-      if DownloadPage <> nil then begin
-        DownloadPage.Show;
-        try
-          var Failed: String;
-          repeat
-            Failed := '';
-            try
-              DownloadPage.Download;  { Does not throw EAbort }
-            except
-              Failed := GetExceptMessage;
-            end;
-          until (Failed = '') or (AskRetryDownloadArchivesToExtract(Failed) = IDCANCEL);
-          if Failed <> '' then
-            raise Exception.Create(Failed);
-
-          for var A in ArchivesToDownload do begin
-            with PSetupFileEntry(Entries[seFile][A.Key])^ do begin
-              {!!!} //make it do this per file immediately after file's download success - so that on retries it skips files which succeeded
-              SourceFilename := A.Value;
-              { Remove Download flag since download has been done, and remove CustomDestName flag
-                since ExtractArchive flag doesn't like that }
-              Options := Options - [foDownload, foCustomDestName];
-              { DestName should now not include a filename, see TSetupCompiler.EnumFilesProc.ProcessFileList }
-              DestName := PathExtractPath(DestName);
-              Verification.Typ := fvNone;
-            end;
+    if DownloadPage <> nil then begin
+      DownloadPage.Show;
+      try
+        var Failed: String;
+        repeat
+          Failed := '';
+          try
+            DownloadPage.Download(procedure(const DownloadedFile: TDownloadFile; const DestFile: String; var Remove: Boolean)
+              begin
+                const FileEntry: PSetupFileEntry = Entries[seFile][DownloadedFile.Data];
+                FileEntry.SourceFilename := DestFile;
+                { Remove Download flag since download has been done, and remove CustomDestName flag
+                  since ExtractArchive flag doesn't like that }
+                FileEntry.Options := FileEntry.Options - [foDownload, foCustomDestName];
+                { DestName should now not include a filename, see TSetupCompiler.EnumFilesProc.ProcessFileList }
+                FileEntry.DestName := PathExtractPath(FileEntry.DestName);
+                FileEntry.Verification.Typ := fvNone;
+                { Tell DownloadPage to not download this file again on retry. Without this it would
+                  redownload files that don't use verification. }
+                Remove := True;
+              end);
+          except
+            Failed := GetExceptMessage;
           end;
-        finally
-          DownloadPage.Hide;
-        end;
+        until (Failed = '') or (AskRetryDownloadArchivesToExtract(Failed) = IDCANCEL);
+        if Failed <> '' then
+          raise Exception.Create(Failed);
+      finally
+        DownloadPage.Hide;
       end;
-    finally
-      ArchivesToDownload.Free;
     end;
   end;