ArchiveReaderTest.cpp 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973
  1. /*
  2. * Copyright (c) Contributors to the Open 3D Engine Project.
  3. * For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. *
  5. * SPDX-License-Identifier: Apache-2.0 OR MIT
  6. *
  7. */
  8. #include <AzCore/UnitTest/TestTypes.h>
  9. #include <AzCore/IO/ByteContainerStream.h>
  10. #include <AzCore/std/ranges/ranges_algorithm.h>
  11. #include <Archive/Clients/ArchiveReaderAPI.h>
  12. #include <Archive/Tools/ArchiveWriterAPI.h>
  13. #include <Compression/CompressionLZ4API.h>
  14. // Archive Gem private implementation includes
  15. #include <Clients/ArchiveReaderFactory.h>
  16. #include <Tools/ArchiveWriterFactory.h>
  17. namespace Archive::Test
  18. {
  19. // Note the ArchiveReader unit test are placed in the Archive.Editor.Tests
  20. // Tools module to have to the ArchiveWriter which is used to create
  21. // test archives to be read
  22. // In theory if all archives were written to a file on disk
  23. // the test could be placed in the Archive.Tests client module
  24. // The ArchiveEditorTestEnvironment is tracking memory
  25. // via the GemTestEnvironment::SetupEnvironment function
  26. // so the LeakDetectionFixture should not be used
  27. class ArchiveReaderFixture
  28. : public ::testing::Test
  29. {
  30. public:
  31. ArchiveReaderFixture()
  32. {
  33. m_archiveReaderFactory = AZStd::make_unique<ArchiveReaderFactory>();
  34. AZ::Interface<IArchiveReaderFactory>::Register(m_archiveReaderFactory.get());
  35. m_archiveWriterFactory = AZStd::make_unique<ArchiveWriterFactory>();
  36. AZ::Interface<IArchiveWriterFactory>::Register(m_archiveWriterFactory.get());
  37. }
  38. ~ArchiveReaderFixture()
  39. {
  40. AZ::Interface<IArchiveWriterFactory>::Unregister(m_archiveWriterFactory.get());
  41. AZ::Interface<IArchiveReaderFactory>::Unregister(m_archiveReaderFactory.get());
  42. }
  43. protected:
  44. // Create and register an Archive Reader and Archive Writer factory
  45. // to allow IArchiveReader and IArchiveWriter instances to be created
  46. // The Archive Writer is needed to create the Archives in order to test the
  47. // Archive Reader code
  48. AZStd::unique_ptr<IArchiveReaderFactory> m_archiveReaderFactory;
  49. AZStd::unique_ptr<IArchiveWriterFactory> m_archiveWriterFactory;
  50. };
  51. TEST_F(ArchiveReaderFixture, CreateArchiveReader_Succeeds)
  52. {
  53. {
  54. auto createArchiveReaderResult = CreateArchiveReader();
  55. ASSERT_TRUE(createArchiveReaderResult);
  56. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  57. ASSERT_NE(nullptr, archiveReader);
  58. }
  59. {
  60. auto createArchiveReaderResult = CreateArchiveReader(ArchiveReaderSettings{});
  61. ASSERT_TRUE(createArchiveReaderResult);
  62. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  63. ASSERT_NE(nullptr, archiveReader);
  64. }
  65. }
  66. TEST_F(ArchiveReaderFixture, MountingEmptyFile_Fails)
  67. {
  68. AZStd::vector<AZStd::byte> archiveBuffer;
  69. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  70. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  71. IArchiveReader::ArchiveStreamPtr archiveStreamPtr(&archiveStream, { false });
  72. ArchiveReaderSettings readerSettings;
  73. bool mountErrorOccurred{};
  74. readerSettings.m_errorCallback = [&mountErrorOccurred](const ArchiveReaderError&)
  75. {
  76. mountErrorOccurred = true;
  77. };
  78. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveStreamPtr),
  79. readerSettings);
  80. ASSERT_TRUE(createArchiveReaderResult);
  81. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  82. EXPECT_TRUE(mountErrorOccurred);
  83. EXPECT_FALSE(archiveReader->IsMounted());
  84. // reset the mountErrorOccurred boolean
  85. mountErrorOccurred = false;
  86. // now try explicitly mounting an archive using the MountArchive method
  87. archiveStreamPtr.reset(&archiveStream);
  88. EXPECT_FALSE(archiveReader->MountArchive(AZStd::move(archiveStreamPtr)));
  89. EXPECT_TRUE(mountErrorOccurred);
  90. EXPECT_FALSE(archiveReader->IsMounted());
  91. }
  92. TEST_F(ArchiveReaderFixture, MountingFailsForInvalidArchive)
  93. {
  94. AZStd::vector<AZStd::byte> archiveBuffer;
  95. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  96. // Write random data to the archiveStream
  97. constexpr AZStd::string_view testData = "The slow gray fox hid under the hyperactive cat";
  98. archiveStream.Write(testData.size(), testData.data());
  99. archiveStream.Seek(0, AZ::IO::GenericStream::SeekMode::ST_SEEK_BEGIN);
  100. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  101. IArchiveReader::ArchiveStreamPtr archiveStreamPtr(&archiveStream, { false });
  102. ArchiveReaderSettings readerSettings;
  103. bool mountErrorOccurred{};
  104. readerSettings.m_errorCallback = [&mountErrorOccurred](const ArchiveReaderError&)
  105. {
  106. mountErrorOccurred = true;
  107. };
  108. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveStreamPtr),
  109. readerSettings);
  110. ASSERT_TRUE(createArchiveReaderResult);
  111. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  112. EXPECT_TRUE(mountErrorOccurred);
  113. EXPECT_FALSE(archiveReader->IsMounted());
  114. // reset the mountErrorOccurred boolean
  115. mountErrorOccurred = false;
  116. // now try explicitly mounting an archive using the MountArchive method
  117. archiveStreamPtr.reset(&archiveStream);
  118. EXPECT_FALSE(archiveReader->MountArchive(AZStd::move(archiveStreamPtr)));
  119. EXPECT_TRUE(mountErrorOccurred);
  120. EXPECT_FALSE(archiveReader->IsMounted());
  121. }
  122. TEST_F(ArchiveReaderFixture, DefaultArchive_CreatedFromWriter_CanBeMounted)
  123. {
  124. AZStd::vector<AZStd::byte> archiveBuffer;
  125. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  126. {
  127. // Create an empty archive with no files in it
  128. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  129. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  130. ASSERT_TRUE(createArchiveWriterResult);
  131. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  132. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  133. ASSERT_TRUE(commitResult);
  134. }
  135. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  136. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  137. ArchiveReaderSettings readerSettings;
  138. bool mountErrorOccurred{};
  139. readerSettings.m_errorCallback = [&mountErrorOccurred](const ArchiveReaderError&)
  140. {
  141. mountErrorOccurred = true;
  142. };
  143. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr),
  144. readerSettings);
  145. ASSERT_TRUE(createArchiveReaderResult);
  146. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  147. // No error should occur and the archive should have been successfully mounted
  148. EXPECT_FALSE(mountErrorOccurred);
  149. EXPECT_TRUE(archiveReader->IsMounted());
  150. // Unmount the archive
  151. archiveReader->UnmountArchive();
  152. EXPECT_FALSE(archiveReader->IsMounted());
  153. // reset the mountErrorOccurred boolean
  154. mountErrorOccurred = false;
  155. // now try explicitly mounting an archive using the MountArchive method
  156. archiveReaderStreamPtr.reset(&archiveStream);
  157. EXPECT_TRUE(archiveReader->MountArchive(AZStd::move(archiveReaderStreamPtr)));
  158. EXPECT_FALSE(mountErrorOccurred);
  159. EXPECT_TRUE(archiveReader->IsMounted());
  160. }
  161. TEST_F(ArchiveReaderFixture, ListFileInArchive_ForExistingFile_Succeeds)
  162. {
  163. AZStd::vector<AZStd::byte> archiveBuffer;
  164. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  165. AZStd::string_view fooFileData;
  166. AZStd::string_view levelPrefabFileData;
  167. {
  168. // Create an archive with a several files in it
  169. // At least one of the files are compressed
  170. // and at one is not compressed
  171. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  172. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  173. ASSERT_TRUE(createArchiveWriterResult);
  174. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  175. ArchiveWriterFileSettings fileSettings;
  176. // Write an uncompressed file with the contents of hello world
  177. AZStd::string_view fileData = "Hello World";
  178. fooFileData = fileData;
  179. fileSettings.m_relativeFilePath = "foo.txt";
  180. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  181. // Write a compressed file this time
  182. fileData = "My Prefab Data in an Archive";
  183. levelPrefabFileData = fileData;
  184. fileSettings.m_compressionAlgorithm = CompressionLZ4::GetLZ4CompressionAlgorithmId();
  185. fileSettings.m_relativeFilePath = "subdirectory/Level.prefab";
  186. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  187. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  188. ASSERT_TRUE(commitResult);
  189. }
  190. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  191. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  192. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  193. ASSERT_TRUE(createArchiveReaderResult);
  194. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  195. // No error should occur and the archive should have been successfully mounted
  196. EXPECT_TRUE(archiveReader->IsMounted());
  197. // The fooFileToken ArchiveFileToken instance will be used to validate
  198. // the overload of ListFileInArchive which accepts an ArchiveFileToken
  199. ArchiveFileToken fooFileToken = InvalidArchiveFileToken;
  200. {
  201. // Lookup the foo.txt file
  202. constexpr AZStd::string_view fooPath = "foo.txt";
  203. ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(fooPath);
  204. EXPECT_TRUE(archiveListFileResult);
  205. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  206. // Store of the file token for later lookup
  207. fooFileToken = archiveListFileResult.m_filePathToken;
  208. EXPECT_EQ(fooPath, archiveListFileResult.m_relativeFilePath);
  209. EXPECT_EQ(Compression::Uncompressed, archiveListFileResult.m_compressionAlgorithm);
  210. EXPECT_EQ(fooFileData.size(), archiveListFileResult.m_uncompressedSize);
  211. // As the file is not compressed, the ArchiveListFileResult compressed member is not check
  212. // The file is expected to have been written at offset 512-byte and aligned up to the next multiple of 512
  213. // which is 1024 since the file is <= 512 bytes in size
  214. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  215. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  216. }
  217. {
  218. // Lookup the subdirectory/level.prefab file
  219. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  220. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  221. AZ::IO::Path prefabPathLower = "subdirectory/Level.prefab";
  222. AZStd::to_lower(prefabPathLower.Native());
  223. ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(prefabPathLower);
  224. EXPECT_TRUE(archiveListFileResult);
  225. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  226. EXPECT_EQ(prefabPathLower, archiveListFileResult.m_relativeFilePath);
  227. EXPECT_EQ(CompressionLZ4::GetLZ4CompressionAlgorithmId(), archiveListFileResult.m_compressionAlgorithm);
  228. // The file should have been compressed
  229. // Just validate that its size > 0
  230. EXPECT_GT(archiveListFileResult.m_compressedSize, 0);
  231. EXPECT_EQ(levelPrefabFileData.size(), archiveListFileResult.m_uncompressedSize);
  232. // Since this is the second file written to the archive
  233. // the start offset should be at the next 512-byte offset a
  234. // which is 1024, as foo.txt was written at offset 512
  235. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment * 2;
  236. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  237. }
  238. {
  239. // This time lookup the foo.txt file using the ArchiveFileToken
  240. constexpr AZStd::string_view fooPath = "foo.txt";
  241. ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(fooFileToken);
  242. EXPECT_TRUE(archiveListFileResult);
  243. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  244. EXPECT_EQ(fooPath, archiveListFileResult.m_relativeFilePath);
  245. EXPECT_EQ(Compression::Uncompressed, archiveListFileResult.m_compressionAlgorithm);
  246. EXPECT_EQ(fooFileData.size(), archiveListFileResult.m_uncompressedSize);
  247. // As the file is not compressed, the ArchiveListFileResult compressed member is not check
  248. // The file is expected to have been written at offset 512-byte and aligned up to the next multiple of 512
  249. // which is 1024 since the file is <= 512 bytes in size
  250. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  251. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  252. }
  253. // Finally validate the ContainsFile function succeeds
  254. EXPECT_TRUE(archiveReader->ContainsFile("foo.txt"));
  255. }
  256. TEST_F(ArchiveReaderFixture, ListFileInArchive_ForFileNotInArchive_Fails)
  257. {
  258. AZStd::vector<AZStd::byte> archiveBuffer;
  259. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  260. AZStd::string_view fooFileData;
  261. {
  262. // Create an archive with files in it
  263. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  264. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  265. ASSERT_TRUE(createArchiveWriterResult);
  266. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  267. ArchiveWriterFileSettings fileSettings;
  268. // Write an uncompressed file with the contents of hello world
  269. AZStd::string_view fileData = "Hello World";
  270. fooFileData = fileData;
  271. fileSettings.m_relativeFilePath = "foo.txt";
  272. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  273. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  274. ASSERT_TRUE(commitResult);
  275. }
  276. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  277. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  278. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  279. ASSERT_TRUE(createArchiveReaderResult);
  280. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  281. // No error should occur and the archive should have been successfully mounted
  282. EXPECT_TRUE(archiveReader->IsMounted());
  283. {
  284. // Lookup the non-existent/foo.txt
  285. constexpr AZStd::string_view nonExistentPath = "non-existent/foo.txt";
  286. ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(nonExistentPath);
  287. EXPECT_FALSE(archiveListFileResult);
  288. EXPECT_FALSE(archiveListFileResult.m_resultOutcome.has_value());
  289. // Also the ContainsFile function should fail as well
  290. EXPECT_FALSE(archiveReader->ContainsFile(nonExistentPath));
  291. }
  292. }
  293. TEST_F(ArchiveReaderFixture, EnumerateFilesInArchive_Visits_EachFileInTheArchive)
  294. {
  295. AZStd::vector<AZStd::byte> archiveBuffer;
  296. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  297. AZStd::string_view fooFileData;
  298. AZStd::string_view levelPrefabFileData;
  299. AZStd::string_view barFileData;
  300. {
  301. // Create an archive with a several files in it
  302. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  303. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  304. ASSERT_TRUE(createArchiveWriterResult);
  305. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  306. ArchiveWriterFileSettings fileSettings;
  307. // Write an uncompressed file with the contents of hello world
  308. AZStd::string_view fileData = "Hello World";
  309. fooFileData = fileData;
  310. fileSettings.m_relativeFilePath = "foo.txt";
  311. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  312. // Write a compressed file this time
  313. fileData = "My Prefab Data in an Archive";
  314. levelPrefabFileData = fileData;
  315. fileSettings.m_compressionAlgorithm = CompressionLZ4::GetLZ4CompressionAlgorithmId();
  316. fileSettings.m_relativeFilePath = "subdirectory/Level.prefab";
  317. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  318. // Write a second text file
  319. fileData = "Box Box, Box Box";
  320. barFileData = fileData;
  321. // Don't compress the file this time
  322. fileSettings.m_compressionAlgorithm = Compression::Uncompressed;
  323. fileSettings.m_relativeFilePath = "subdirectory/bar.txt";
  324. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  325. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  326. ASSERT_TRUE(commitResult);
  327. }
  328. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  329. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  330. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  331. ASSERT_TRUE(createArchiveReaderResult);
  332. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  333. // No error should occur and the archive should have been successfully mounted
  334. EXPECT_TRUE(archiveReader->IsMounted());
  335. AZStd::vector<ArchiveListFileResult> filesInArchive;
  336. auto GetListResultsForFiles = [&filesInArchive](ArchiveListFileResult listFileResult) -> bool
  337. {
  338. filesInArchive.emplace_back(AZStd::move(listFileResult));
  339. // A return of true causes enumeration to continue
  340. return true;
  341. };
  342. EXPECT_TRUE(archiveReader->EnumerateFilesInArchive(GetListResultsForFiles));
  343. // The vector should have 3 entries in it
  344. ASSERT_EQ(3, filesInArchive.size());
  345. size_t fileIndex{};
  346. {
  347. // Validate the first entry in the vector
  348. // Lookup the foo.txt file
  349. constexpr AZStd::string_view fooPath = "foo.txt";
  350. const ArchiveListFileResult archiveListFileResult = filesInArchive[fileIndex++];
  351. EXPECT_TRUE(archiveListFileResult);
  352. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  353. EXPECT_EQ(fooPath, archiveListFileResult.m_relativeFilePath);
  354. EXPECT_EQ(Compression::Uncompressed, archiveListFileResult.m_compressionAlgorithm);
  355. EXPECT_EQ(fooFileData.size(), archiveListFileResult.m_uncompressedSize);
  356. // As the file is not compressed, the ArchiveListFileResult compressed member is not check
  357. // The file is expected to have been written at offset 512-byte and aligned up to the next multiple of 512
  358. // which is 1024 since the file is <= 512 bytes in size
  359. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  360. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  361. }
  362. {
  363. // Validate the second entry in the vector
  364. // Lookup the subdirectory/level.prefab file
  365. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  366. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  367. AZ::IO::Path prefabPathLower = "subdirectory/Level.prefab";
  368. AZStd::to_lower(prefabPathLower.Native());
  369. const ArchiveListFileResult archiveListFileResult = filesInArchive[fileIndex++];
  370. EXPECT_TRUE(archiveListFileResult);
  371. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  372. EXPECT_EQ(prefabPathLower, archiveListFileResult.m_relativeFilePath);
  373. EXPECT_EQ(CompressionLZ4::GetLZ4CompressionAlgorithmId(), archiveListFileResult.m_compressionAlgorithm);
  374. // The file should have been compressed
  375. // Just validate that its size > 0
  376. EXPECT_GT(archiveListFileResult.m_compressedSize, 0);
  377. EXPECT_EQ(levelPrefabFileData.size(), archiveListFileResult.m_uncompressedSize);
  378. // Since this is the second file written to the archive
  379. // the start offset should be at the next 512-byte offset a
  380. // which is 1024, as foo.txt was written at offset 512
  381. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment * 2;
  382. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  383. }
  384. {
  385. // Validate the third entry in the vector
  386. // Lookup the subdirectory/bar.txt file
  387. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  388. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  389. AZ::IO::PathView barPath = "subdirectory/bar.txt";
  390. const ArchiveListFileResult archiveListFileResult = filesInArchive[fileIndex++];
  391. EXPECT_TRUE(archiveListFileResult);
  392. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  393. EXPECT_EQ(barPath, archiveListFileResult.m_relativeFilePath);
  394. EXPECT_EQ(Compression::Uncompressed, archiveListFileResult.m_compressionAlgorithm);
  395. EXPECT_EQ(barFileData.size(), archiveListFileResult.m_uncompressedSize);
  396. // Since this is the third file written to the archive
  397. // and the first two files should have had an uncompressed size
  398. // and compressed size that is < 512-bytes
  399. // the start offset for bar.txt should be at 512 * 3 = 1536
  400. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment * 3;
  401. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  402. }
  403. }
  404. TEST_F(ArchiveReaderFixture, EnumerateFilesInArchive_CanFilterFiles_Succeeds)
  405. {
  406. AZStd::vector<AZStd::byte> archiveBuffer;
  407. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  408. AZStd::string_view fooFileData;
  409. AZStd::string_view levelPrefabFileData;
  410. AZStd::string_view barFileData;
  411. {
  412. // Create an archive with a several files in it
  413. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  414. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  415. ASSERT_TRUE(createArchiveWriterResult);
  416. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  417. ArchiveWriterFileSettings fileSettings;
  418. // Write an uncompressed file with the contents of hello world
  419. AZStd::string_view fileData = "Hello World";
  420. fooFileData = fileData;
  421. fileSettings.m_relativeFilePath = "foo.txt";
  422. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  423. // Write a compressed file this time
  424. fileData = "My Prefab Data in an Archive";
  425. levelPrefabFileData = fileData;
  426. fileSettings.m_compressionAlgorithm = CompressionLZ4::GetLZ4CompressionAlgorithmId();
  427. fileSettings.m_relativeFilePath = "subdirectory/Level.prefab";
  428. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  429. // Write a second text file
  430. fileData = "Box Box, Box Box";
  431. barFileData = fileData;
  432. // Don't compress the file this time
  433. fileSettings.m_compressionAlgorithm = Compression::Uncompressed;
  434. fileSettings.m_relativeFilePath = "subdirectory/bar.txt";
  435. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  436. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  437. ASSERT_TRUE(commitResult);
  438. }
  439. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  440. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  441. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  442. ASSERT_TRUE(createArchiveReaderResult);
  443. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  444. // No error should occur and the archive should have been successfully mounted
  445. EXPECT_TRUE(archiveReader->IsMounted());
  446. AZStd::vector<ArchiveListFileResult> filesInArchive;
  447. //!!! This time only add .txt files to the list file vector
  448. auto FilterListResultsForFiles = [&filesInArchive](ArchiveListFileResult listFileResult) -> bool
  449. {
  450. if (listFileResult.m_relativeFilePath.Match("*.txt"))
  451. {
  452. filesInArchive.emplace_back(AZStd::move(listFileResult));
  453. }
  454. return true;
  455. };
  456. EXPECT_TRUE(archiveReader->EnumerateFilesInArchive(FilterListResultsForFiles));
  457. // The vector should have 2 entries in it
  458. // as there are only two txt files in it
  459. ASSERT_EQ(2, filesInArchive.size());
  460. size_t fileIndex{};
  461. {
  462. // Validate the first entry in the vector
  463. // Lookup the foo.txt file
  464. constexpr AZStd::string_view fooPath = "foo.txt";
  465. const ArchiveListFileResult archiveListFileResult = filesInArchive[fileIndex++];
  466. EXPECT_TRUE(archiveListFileResult);
  467. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  468. EXPECT_EQ(fooPath, archiveListFileResult.m_relativeFilePath);
  469. EXPECT_EQ(Compression::Uncompressed, archiveListFileResult.m_compressionAlgorithm);
  470. EXPECT_EQ(fooFileData.size(), archiveListFileResult.m_uncompressedSize);
  471. // As the file is not compressed, the ArchiveListFileResult compressed member is not check
  472. // The file is expected to have been written at offset 512-byte and aligned up to the next multiple of 512
  473. // which is 1024 since the file is <= 512 bytes in size
  474. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  475. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  476. }
  477. {
  478. // Validate the next entry in the vector
  479. // Lookup the subdirectory/bar.txt file
  480. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  481. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  482. AZ::IO::PathView barPath = "subdirectory/bar.txt";
  483. const ArchiveListFileResult archiveListFileResult = filesInArchive[fileIndex++];
  484. EXPECT_TRUE(archiveListFileResult);
  485. EXPECT_NE(InvalidArchiveFileToken, archiveListFileResult.m_filePathToken);
  486. EXPECT_EQ(barPath, archiveListFileResult.m_relativeFilePath);
  487. EXPECT_EQ(Compression::Uncompressed, archiveListFileResult.m_compressionAlgorithm);
  488. EXPECT_EQ(barFileData.size(), archiveListFileResult.m_uncompressedSize);
  489. // Since this is the third file written to the archive
  490. // and the first two files should have had an uncompressed size
  491. // and compressed size that is < 512-bytes
  492. // the start offset for bar.txt should be at 512 * 3 = 1536
  493. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment * 3;
  494. EXPECT_EQ(expectedFileOffset, archiveListFileResult.m_offset);
  495. }
  496. }
  497. TEST_F(ArchiveReaderFixture, ExtractFileFromArchive_ForExistingFile_Succeeds)
  498. {
  499. AZStd::vector<AZStd::byte> archiveBuffer;
  500. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  501. AZStd::string_view fooFileData;
  502. AZStd::string_view levelPrefabFileData;
  503. AZStd::string_view barFileData;
  504. {
  505. // Create an archive with a several files in it
  506. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  507. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  508. ASSERT_TRUE(createArchiveWriterResult);
  509. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  510. ArchiveWriterFileSettings fileSettings;
  511. // Write an uncompressed file with the contents of hello world
  512. AZStd::string_view fileData = "Hello World";
  513. fooFileData = fileData;
  514. fileSettings.m_relativeFilePath = "foo.txt";
  515. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  516. // Write a compressed file this time
  517. fileData = "My Prefab Data in an Archive";
  518. levelPrefabFileData = fileData;
  519. fileSettings.m_compressionAlgorithm = CompressionLZ4::GetLZ4CompressionAlgorithmId();
  520. fileSettings.m_relativeFilePath = "subdirectory/Level.prefab";
  521. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  522. // Write a second text file
  523. fileData = "Box Box, Box Box";
  524. barFileData = fileData;
  525. // Don't compress the file this time
  526. fileSettings.m_compressionAlgorithm = Compression::Uncompressed;
  527. fileSettings.m_relativeFilePath = "subdirectory/bar.txt";
  528. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(fileData)), fileSettings));
  529. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  530. ASSERT_TRUE(commitResult);
  531. }
  532. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  533. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  534. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  535. ASSERT_TRUE(createArchiveReaderResult);
  536. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  537. // No error should occur and the archive should have been successfully mounted
  538. EXPECT_TRUE(archiveReader->IsMounted());
  539. {
  540. // Extract foo.txt
  541. constexpr AZStd::string_view fooPath = "foo.txt";
  542. // Use ArchiveListFileResult to lookup the file metadata to determine
  543. // its uncompressed size
  544. const ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(fooPath);
  545. ASSERT_TRUE(archiveListFileResult);
  546. AZStd::vector<AZStd::byte> fileBuffer;
  547. // Resize the buffer to exact size needed
  548. fileBuffer.resize_no_construct(archiveListFileResult.m_uncompressedSize);
  549. // Populate settings structure needed to extract the file
  550. ArchiveReaderFileSettings fileSettings;
  551. fileSettings.m_filePathIdentifier = fooPath;
  552. const ArchiveExtractFileResult archiveExtractFileResult = archiveReader->ExtractFileFromArchive(
  553. fileBuffer, fileSettings);
  554. ASSERT_TRUE(archiveExtractFileResult);
  555. EXPECT_NE(InvalidArchiveFileToken, archiveExtractFileResult.m_filePathToken);
  556. EXPECT_EQ(fooPath, archiveExtractFileResult.m_relativeFilePath);
  557. EXPECT_EQ(Compression::Uncompressed, archiveExtractFileResult.m_compressionAlgorithm);
  558. EXPECT_EQ(fooFileData.size(), archiveExtractFileResult.m_uncompressedSize);
  559. // As the file is not compressed, the ArchiveListFileResult compressed member is not check
  560. // The file is expected to have been written at offset 512-byte and aligned up to the next multiple of 512
  561. // which is 1024 since the file is <= 512 bytes in size
  562. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  563. EXPECT_EQ(expectedFileOffset, archiveExtractFileResult.m_offset);
  564. // Now check the file span from the archiveExtractFileResult to get the exact foo data
  565. // Reinterpret the byte data in file span as text
  566. AZStd::string_view textFileSpan(reinterpret_cast<const char*>(archiveExtractFileResult.m_fileSpan.data()),
  567. archiveExtractFileResult.m_fileSpan.size());
  568. EXPECT_THAT(textFileSpan, ::testing::ContainerEq(fooFileData));
  569. // Validate the CRC32 of the entire file contents
  570. // NOTE: This only works on when extracting the entire uncompressed file
  571. // If the file was extracted with the ArchiveReader::m_decompressFile option set to false
  572. // and the file was compressed, then the CRC does not apply
  573. // Also if a partial read of the file was done, the Crc32 also does not apply
  574. EXPECT_EQ(archiveExtractFileResult.m_crc32, AZ::Crc32(archiveExtractFileResult.m_fileSpan));
  575. }
  576. {
  577. // Extract subdirectory/Level.prefab
  578. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  579. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  580. AZ::IO::Path prefabPathLower = "subdirectory/Level.prefab";
  581. AZStd::to_lower(prefabPathLower.Native());
  582. // Use ArchiveListFileResult to lookup the file metadata to uncompressed size
  583. const ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(prefabPathLower);
  584. ASSERT_TRUE(archiveListFileResult);
  585. AZStd::vector<AZStd::byte> fileBuffer;
  586. // Resize the buffer to exact size needed
  587. fileBuffer.resize_no_construct(archiveListFileResult.m_uncompressedSize);
  588. // Populate settings structure needed to extract the file
  589. ArchiveReaderFileSettings fileSettings;
  590. // Use the file path token this time to extract the file
  591. fileSettings.m_filePathIdentifier = archiveListFileResult.m_filePathToken;
  592. const ArchiveExtractFileResult archiveExtractFileResult = archiveReader->ExtractFileFromArchive(
  593. fileBuffer, fileSettings);
  594. ASSERT_TRUE(archiveExtractFileResult);
  595. EXPECT_NE(InvalidArchiveFileToken, archiveExtractFileResult.m_filePathToken);
  596. EXPECT_EQ(prefabPathLower, archiveExtractFileResult.m_relativeFilePath);
  597. EXPECT_EQ(CompressionLZ4::GetLZ4CompressionAlgorithmId(), archiveExtractFileResult.m_compressionAlgorithm);
  598. // The file should have been compressed
  599. // Just validate that its size > 0
  600. EXPECT_GT(archiveExtractFileResult.m_compressedSize, 0);
  601. EXPECT_EQ(levelPrefabFileData.size(), archiveExtractFileResult.m_uncompressedSize);
  602. // The subdirectory/level.prefab is the second file within the archive
  603. // So it its start offset should be at 1024 since the foo.txt start offset was at 512
  604. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment * 2;
  605. EXPECT_EQ(expectedFileOffset, archiveExtractFileResult.m_offset);
  606. // Now check the file span from the archiveExtractFileResult to get the exact
  607. // view of the file data for the level.prefab after running it through decompression
  608. AZStd::string_view textFileSpan(reinterpret_cast<const char*>(archiveExtractFileResult.m_fileSpan.data()),
  609. archiveExtractFileResult.m_fileSpan.size());
  610. EXPECT_THAT(textFileSpan, ::testing::ContainerEq(levelPrefabFileData));
  611. // Validate the CRC32 of the entire file contents
  612. // NOTE: This only works on when extracting the entire uncompressed file
  613. // If the file was extracted with the ArchiveReader::m_decompressFile option set to false
  614. // and the file was compressed, then the CRC does not apply
  615. // Also if a partial read of the file was done, the Crc32 also does not apply
  616. EXPECT_EQ(archiveExtractFileResult.m_crc32, AZ::Crc32(archiveExtractFileResult.m_fileSpan));
  617. }
  618. }
  619. //! This test validates the setting the ArchiveReaderFileSettings::m_decompressFile option to `false`
  620. //! will extract the compressed file WITHOUT decompressing it.
  621. TEST_F(ArchiveReaderFixture, ExtractFileFromArchive_ExtractionOfFile_ThatSkipsDecompressed_Succeeds)
  622. {
  623. AZStd::vector<AZStd::byte> archiveBuffer;
  624. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  625. AZStd::string_view levelPrefabFileData = "My Prefab Data in an Archive";
  626. AZStd::vector<AZStd::byte> expectedCompressedFileData;
  627. auto compressionRegistrar = Compression::CompressionRegistrar::Get();
  628. ASSERT_NE(nullptr, compressionRegistrar);
  629. auto lz4Compressor = compressionRegistrar->FindCompressionInterface(CompressionLZ4::GetLZ4CompressionAlgorithmId());
  630. ASSERT_NE(nullptr, lz4Compressor);
  631. // Resize the output buffer to be large enough to contain the compressed data
  632. expectedCompressedFileData.resize_no_construct(lz4Compressor->CompressBound(levelPrefabFileData.size()));
  633. // Compress the test data into the byte buffer
  634. Compression::CompressionResultData compressionResult = lz4Compressor->CompressBlock(expectedCompressedFileData,
  635. AZStd::as_bytes(AZStd::span(levelPrefabFileData)));
  636. ASSERT_TRUE(compressionResult);
  637. AZStd::span<AZStd::byte> actualCompressedDataSpan = compressionResult.m_compressedBuffer;
  638. {
  639. // Create an archive with a several files in it
  640. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  641. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  642. ASSERT_TRUE(createArchiveWriterResult);
  643. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  644. ArchiveWriterFileSettings fileSettings;
  645. // Write a compressed file this time
  646. fileSettings.m_compressionAlgorithm = CompressionLZ4::GetLZ4CompressionAlgorithmId();
  647. fileSettings.m_relativeFilePath = "subdirectory/Level.prefab";
  648. EXPECT_TRUE(archiveWriter->AddFileToArchive(AZStd::as_bytes(AZStd::span(levelPrefabFileData)), fileSettings));
  649. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  650. ASSERT_TRUE(commitResult);
  651. }
  652. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  653. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  654. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  655. ASSERT_TRUE(createArchiveReaderResult);
  656. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  657. // No error should occur and the archive should have been successfully mounted
  658. EXPECT_TRUE(archiveReader->IsMounted());
  659. {
  660. // Extract subdirectory/Level.prefab
  661. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  662. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  663. AZ::IO::Path prefabPathLower = "subdirectory/Level.prefab";
  664. AZStd::to_lower(prefabPathLower.Native());
  665. // Use ArchiveListFileResult to lookup the file metadata
  666. const ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(prefabPathLower);
  667. ASSERT_TRUE(archiveListFileResult);
  668. AZStd::vector<AZStd::byte> fileBuffer;
  669. // As the buffer is pointing to compressed data this time
  670. // It is resized to be large enough to store the compressed data
  671. fileBuffer.resize_no_construct(archiveListFileResult.m_compressedSize);
  672. // Populate settings structure needed to extract the file
  673. ArchiveReaderFileSettings fileSettings;
  674. // Use the file path token this time to extract the file
  675. fileSettings.m_filePathIdentifier = archiveListFileResult.m_filePathToken;
  676. // Skip Decompression of the file content
  677. fileSettings.m_decompressFile = false;
  678. const ArchiveExtractFileResult archiveExtractFileResult = archiveReader->ExtractFileFromArchive(
  679. fileBuffer, fileSettings);
  680. ASSERT_TRUE(archiveExtractFileResult);
  681. EXPECT_NE(InvalidArchiveFileToken, archiveExtractFileResult.m_filePathToken);
  682. EXPECT_EQ(prefabPathLower, archiveExtractFileResult.m_relativeFilePath);
  683. EXPECT_EQ(CompressionLZ4::GetLZ4CompressionAlgorithmId(), archiveExtractFileResult.m_compressionAlgorithm);
  684. // The file should have been compressed
  685. // Just validate that its size > 0
  686. EXPECT_GT(archiveExtractFileResult.m_compressedSize, 0);
  687. EXPECT_EQ(levelPrefabFileData.size(), archiveExtractFileResult.m_uncompressedSize);
  688. // The subdirectory/level.prefab is the only file within the archive
  689. // So it its start offset should be at 512
  690. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  691. EXPECT_EQ(expectedFileOffset, archiveExtractFileResult.m_offset);
  692. // The file span should still be compressed at this point
  693. AZStd::span<AZStd::byte> compressedFileSpan = archiveExtractFileResult.m_fileSpan;
  694. EXPECT_TRUE(AZStd::ranges::equal(compressedFileSpan, actualCompressedDataSpan));
  695. }
  696. }
  697. TEST_F(ArchiveReaderFixture, ExtractFileFromArchive_PartialReadOfCompressedFile_Across3Blocks_Succeeds)
  698. {
  699. // Verify that the ArchiveReaderFileSettings can read content from a compressed file
  700. // where the data is are in different 2 MiB blocks of the file when uncompressed
  701. // The format of the data will be as follows
  702. // First 2-MiB Block to be compressed:
  703. // (2-MiB - 1) bytes of '\0'
  704. // 1 byte of 'H'
  705. // Second 2-MiB Block to be compressed:
  706. // 10 bytes that are 'e', 'l', 'l' 'o', ' ', 'W', 'o', 'r', 'l', 'd'
  707. // (2-MiB - 10) byte of '\0'
  708. // Final 2-MiB Block to be compressed (partial:
  709. // 1 byte 'A'
  710. // 7 bytes of "rchive"
  711. //
  712. // The test will attempt to start reading at offset (2MiB -1)
  713. // and will attempt to read (2 MiB + 2) bytes
  714. // This should result internally in 3 compressed blocks being read
  715. // and decompressed
  716. // What should be returned is the value of
  717. // "Hello World" + (2-MiB - 10) of '\0' + 'A'
  718. //
  719. // This will store to the expected uncompressed that is being tested
  720. constexpr AZStd::string_view firstBlockEnd = "H";
  721. constexpr AZStd::string_view secondBlockBegin = "ello World";
  722. constexpr AZStd::string_view finalBlockBegin = "Archive";
  723. AZStd::vector<AZStd::byte> expectedResultData;
  724. // The total size should be (2 MiB + 2)
  725. constexpr size_t PartialReadSize = firstBlockEnd.size() + ArchiveBlockSizeForCompression + 1;
  726. static_assert(PartialReadSize == 2_mib + 2);
  727. expectedResultData.reserve(PartialReadSize);
  728. auto expectedDataBackInserter = AZStd::back_inserter(expectedResultData);
  729. AZStd::ranges::copy(AZStd::as_bytes(AZStd::span(firstBlockEnd)), expectedDataBackInserter);
  730. AZStd::ranges::copy(AZStd::as_bytes(AZStd::span(secondBlockBegin)), expectedDataBackInserter);
  731. AZStd::fill_n(expectedDataBackInserter, ArchiveBlockSizeForCompression - secondBlockBegin.size(), AZStd::byte{});
  732. // Only copy the first character from the final block
  733. AZStd::ranges::copy(AZStd::as_bytes(AZStd::span(finalBlockBegin)).first(1), expectedDataBackInserter);
  734. ASSERT_EQ(PartialReadSize, expectedResultData.size());
  735. AZStd::vector<AZStd::byte> archiveBuffer;
  736. AZ::IO::ByteContainerStream archiveStream(&archiveBuffer);
  737. {
  738. // Create an archive with a several files in it
  739. IArchiveWriter::ArchiveStreamPtr archiveWriterStreamPtr(&archiveStream, { false });
  740. auto createArchiveWriterResult = CreateArchiveWriter(AZStd::move(archiveWriterStreamPtr));
  741. ASSERT_TRUE(createArchiveWriterResult);
  742. AZStd::unique_ptr<IArchiveWriter> archiveWriter = AZStd::move(createArchiveWriterResult.value());
  743. ArchiveWriterFileSettings fileSettings;
  744. // Generate the data for the file to compressed
  745. AZStd::vector<AZStd::byte> fileBuffer;
  746. // The file should be 4-MiB + the the size of the string "Archive"
  747. constexpr size_t FileSize = ArchiveBlockSizeForCompression * 2 + finalBlockBegin.size();
  748. fileBuffer.reserve(FileSize);
  749. auto fileBufferBackInserter = AZStd::back_inserter(fileBuffer);
  750. // Generate the data for the first block
  751. AZStd::fill_n(fileBufferBackInserter, ArchiveBlockSizeForCompression - firstBlockEnd.size(), AZStd::byte{});
  752. AZStd::ranges::copy(AZStd::as_bytes(AZStd::span(firstBlockEnd)), fileBufferBackInserter);
  753. // Generate the data for the second block
  754. AZStd::ranges::copy(AZStd::as_bytes(AZStd::span(secondBlockBegin)), fileBufferBackInserter);
  755. AZStd::fill_n(fileBufferBackInserter, ArchiveBlockSizeForCompression - secondBlockBegin.size(), AZStd::byte{});
  756. // Generate the data for the final block
  757. AZStd::ranges::copy(AZStd::as_bytes(AZStd::span(finalBlockBegin)), fileBufferBackInserter);
  758. ASSERT_EQ(FileSize, fileBuffer.size());
  759. fileSettings.m_compressionAlgorithm = CompressionLZ4::GetLZ4CompressionAlgorithmId();
  760. fileSettings.m_relativeFilePath = "MultiblockCompressed.bin";
  761. EXPECT_TRUE(archiveWriter->AddFileToArchive(fileBuffer, fileSettings));
  762. IArchiveWriter::CommitResult commitResult = archiveWriter->Commit();
  763. ASSERT_TRUE(commitResult);
  764. }
  765. // Set the ArchiveStreamDeleter to not delete the stack ByteContainerStream
  766. IArchiveReader::ArchiveStreamPtr archiveReaderStreamPtr(&archiveStream, { false });
  767. auto createArchiveReaderResult = CreateArchiveReader(AZStd::move(archiveReaderStreamPtr));
  768. ASSERT_TRUE(createArchiveReaderResult);
  769. AZStd::unique_ptr<IArchiveReader> archiveReader = AZStd::move(createArchiveReaderResult.value());
  770. // No error should occur and the archive should have been successfully mounted
  771. EXPECT_TRUE(archiveReader->IsMounted());
  772. {
  773. // Extract subdirectory/Level.prefab
  774. // The default ArchiveWriterFileSettings used in this test lowercases the files paths that were added
  775. // So before looking up the Level.prefab in the "subdirectory" directory, lowercase the path
  776. AZ::IO::Path filePathLower = "MultiblockCompressed.bin";
  777. AZStd::to_lower(filePathLower.Native());
  778. // Use ArchiveListFileResult to lookup the file metadata
  779. const ArchiveListFileResult archiveListFileResult = archiveReader->ListFileInArchive(filePathLower);
  780. ASSERT_TRUE(archiveListFileResult);
  781. AZStd::vector<AZStd::byte> fileBuffer;
  782. fileBuffer.resize_no_construct(archiveListFileResult.m_uncompressedSize);
  783. // Populate settings structure needed to extract the file
  784. ArchiveReaderFileSettings fileSettings;
  785. // Use the file path token this time to extract the file
  786. fileSettings.m_filePathIdentifier = archiveListFileResult.m_filePathToken;
  787. // Set the start offset to start reading from the byte of the first block
  788. fileSettings.m_startOffset = ArchiveBlockSizeForCompression - 1;
  789. // Read 2-Mib + 2 to read the final byte of the first block, the entire second block
  790. // and the first byte from the final block
  791. fileSettings.m_bytesToRead = ArchiveBlockSizeForCompression + 2;
  792. const ArchiveExtractFileResult archiveExtractFileResult = archiveReader->ExtractFileFromArchive(
  793. fileBuffer, fileSettings);
  794. ASSERT_TRUE(archiveExtractFileResult);
  795. EXPECT_NE(InvalidArchiveFileToken, archiveExtractFileResult.m_filePathToken);
  796. EXPECT_EQ(filePathLower, archiveExtractFileResult.m_relativeFilePath);
  797. EXPECT_EQ(CompressionLZ4::GetLZ4CompressionAlgorithmId(), archiveExtractFileResult.m_compressionAlgorithm);
  798. // The file should have been compressed
  799. // Just validate that its size > 0
  800. EXPECT_GT(archiveExtractFileResult.m_compressedSize, 0);
  801. EXPECT_EQ(fileBuffer.size(), archiveExtractFileResult.m_uncompressedSize);
  802. // Since this is the only file within the achive it should start at offset 512
  803. AZ::u64 expectedFileOffset = ArchiveDefaultBlockAlignment;
  804. EXPECT_EQ(expectedFileOffset, archiveExtractFileResult.m_offset);
  805. // The file span should only view the 2-MiB + 2 sequence within the uncompressed file
  806. // that buffer
  807. AZStd::span<AZStd::byte> requestedFileData = archiveExtractFileResult.m_fileSpan;
  808. EXPECT_TRUE(AZStd::ranges::equal(requestedFileData, expectedResultData));
  809. }
  810. }
  811. }