LzmaBench.cpp 25 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024
  1. // LzmaBench.cpp
  2. #include "StdAfx.h"
  3. #include "LzmaBench.h"
  4. #ifndef _WIN32
  5. #define USE_POSIX_TIME
  6. #define USE_POSIX_TIME2
  7. #endif
  8. #ifdef USE_POSIX_TIME
  9. #include <time.h>
  10. #ifdef USE_POSIX_TIME2
  11. #include <sys/time.h>
  12. #endif
  13. #endif
  14. #ifdef _WIN32
  15. #define USE_ALLOCA
  16. #endif
  17. #ifdef USE_ALLOCA
  18. #ifdef _WIN32
  19. #include <malloc.h>
  20. #else
  21. #include <stdlib.h>
  22. #endif
  23. #endif
  24. extern "C"
  25. {
  26. #include "../../../../C/Alloc.h"
  27. #include "../../../../C/7zCrc.h"
  28. }
  29. #include "../../../Common/MyCom.h"
  30. #include "../../ICoder.h"
  31. #ifdef BENCH_MT
  32. #include "../../../Windows/Thread.h"
  33. #include "../../../Windows/Synchronization.h"
  34. #endif
  35. #ifdef EXTERNAL_LZMA
  36. #include "../../../Windows/PropVariant.h"
  37. #else
  38. #include "../LZMA/LZMADecoder.h"
  39. #include "../LZMA/LZMAEncoder.h"
  40. #endif
  41. static const UInt32 kUncompressMinBlockSize = 1 << 26;
  42. static const UInt32 kAdditionalSize = (1 << 16);
  43. static const UInt32 kCompressedAdditionalSize = (1 << 10);
  44. static const UInt32 kMaxLzmaPropSize = 5;
  45. class CBaseRandomGenerator
  46. {
  47. UInt32 A1;
  48. UInt32 A2;
  49. public:
  50. CBaseRandomGenerator() { Init(); }
  51. void Init() { A1 = 362436069; A2 = 521288629;}
  52. UInt32 GetRnd()
  53. {
  54. return
  55. ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) +
  56. ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) );
  57. }
  58. };
  59. class CBenchBuffer
  60. {
  61. public:
  62. size_t BufferSize;
  63. Byte *Buffer;
  64. CBenchBuffer(): Buffer(0) {}
  65. virtual ~CBenchBuffer() { Free(); }
  66. void Free()
  67. {
  68. ::MidFree(Buffer);
  69. Buffer = 0;
  70. }
  71. bool Alloc(size_t bufferSize)
  72. {
  73. if (Buffer != 0 && BufferSize == bufferSize)
  74. return true;
  75. Free();
  76. Buffer = (Byte *)::MidAlloc(bufferSize);
  77. BufferSize = bufferSize;
  78. return (Buffer != 0);
  79. }
  80. };
  81. class CBenchRandomGenerator: public CBenchBuffer
  82. {
  83. CBaseRandomGenerator *RG;
  84. public:
  85. void Set(CBaseRandomGenerator *rg) { RG = rg; }
  86. UInt32 GetVal(UInt32 &res, int numBits)
  87. {
  88. UInt32 val = res & (((UInt32)1 << numBits) - 1);
  89. res >>= numBits;
  90. return val;
  91. }
  92. UInt32 GetLen(UInt32 &res)
  93. {
  94. UInt32 len = GetVal(res, 2);
  95. return GetVal(res, 1 + len);
  96. }
  97. void Generate()
  98. {
  99. UInt32 pos = 0;
  100. UInt32 rep0 = 1;
  101. while (pos < BufferSize)
  102. {
  103. UInt32 res = RG->GetRnd();
  104. res >>= 1;
  105. if (GetVal(res, 1) == 0 || pos < 1024)
  106. Buffer[pos++] = (Byte)(res & 0xFF);
  107. else
  108. {
  109. UInt32 len;
  110. len = 1 + GetLen(res);
  111. if (GetVal(res, 3) != 0)
  112. {
  113. len += GetLen(res);
  114. do
  115. {
  116. UInt32 ppp = GetVal(res, 5) + 6;
  117. res = RG->GetRnd();
  118. if (ppp > 30)
  119. continue;
  120. rep0 = /* (1 << ppp) +*/ GetVal(res, ppp);
  121. res = RG->GetRnd();
  122. }
  123. while (rep0 >= pos);
  124. rep0++;
  125. }
  126. for (UInt32 i = 0; i < len && pos < BufferSize; i++, pos++)
  127. Buffer[pos] = Buffer[pos - rep0];
  128. }
  129. }
  130. }
  131. };
  132. class CBenchmarkInStream:
  133. public ISequentialInStream,
  134. public CMyUnknownImp
  135. {
  136. const Byte *Data;
  137. size_t Pos;
  138. size_t Size;
  139. public:
  140. MY_UNKNOWN_IMP
  141. void Init(const Byte *data, size_t size)
  142. {
  143. Data = data;
  144. Size = size;
  145. Pos = 0;
  146. }
  147. STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
  148. };
  149. STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
  150. {
  151. size_t remain = Size - Pos;
  152. UInt32 kMaxBlockSize = (1 << 20);
  153. if (size > kMaxBlockSize)
  154. size = kMaxBlockSize;
  155. if (size > remain)
  156. size = (UInt32)remain;
  157. for (UInt32 i = 0; i < size; i++)
  158. ((Byte *)data)[i] = Data[Pos + i];
  159. Pos += size;
  160. if(processedSize != NULL)
  161. *processedSize = size;
  162. return S_OK;
  163. }
  164. class CBenchmarkOutStream:
  165. public ISequentialOutStream,
  166. public CBenchBuffer,
  167. public CMyUnknownImp
  168. {
  169. // bool _overflow;
  170. public:
  171. UInt32 Pos;
  172. // CBenchmarkOutStream(): _overflow(false) {}
  173. void Init()
  174. {
  175. // _overflow = false;
  176. Pos = 0;
  177. }
  178. MY_UNKNOWN_IMP
  179. STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
  180. };
  181. STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)
  182. {
  183. size_t curSize = BufferSize - Pos;
  184. if (curSize > size)
  185. curSize = size;
  186. memcpy(Buffer + Pos, data, curSize);
  187. Pos += (UInt32)curSize;
  188. if(processedSize != NULL)
  189. *processedSize = (UInt32)curSize;
  190. if (curSize != size)
  191. {
  192. // _overflow = true;
  193. return E_FAIL;
  194. }
  195. return S_OK;
  196. }
  197. class CCrcOutStream:
  198. public ISequentialOutStream,
  199. public CMyUnknownImp
  200. {
  201. public:
  202. UInt32 Crc;
  203. MY_UNKNOWN_IMP
  204. void Init() { Crc = CRC_INIT_VAL; }
  205. STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
  206. };
  207. STDMETHODIMP CCrcOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)
  208. {
  209. Crc = CrcUpdate(Crc, data, size);
  210. if (processedSize != NULL)
  211. *processedSize = size;
  212. return S_OK;
  213. }
  214. static UInt64 GetTimeCount()
  215. {
  216. #ifdef USE_POSIX_TIME
  217. #ifdef USE_POSIX_TIME2
  218. timeval v;
  219. if (gettimeofday(&v, 0) == 0)
  220. return (UInt64)(v.tv_sec) * 1000000 + v.tv_usec;
  221. return (UInt64)time(NULL) * 1000000;
  222. #else
  223. return time(NULL);
  224. #endif
  225. #else
  226. /*
  227. LARGE_INTEGER value;
  228. if (::QueryPerformanceCounter(&value))
  229. return value.QuadPart;
  230. */
  231. return GetTickCount();
  232. #endif
  233. }
  234. static UInt64 GetFreq()
  235. {
  236. #ifdef USE_POSIX_TIME
  237. #ifdef USE_POSIX_TIME2
  238. return 1000000;
  239. #else
  240. return 1;
  241. #endif
  242. #else
  243. /*
  244. LARGE_INTEGER value;
  245. if (::QueryPerformanceFrequency(&value))
  246. return value.QuadPart;
  247. */
  248. return 1000;
  249. #endif
  250. }
  251. #ifndef USE_POSIX_TIME
  252. static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; }
  253. #endif
  254. static UInt64 GetUserTime()
  255. {
  256. #ifdef USE_POSIX_TIME
  257. return clock();
  258. #else
  259. FILETIME creationTime, exitTime, kernelTime, userTime;
  260. if (::GetProcessTimes(::GetCurrentProcess(), &creationTime, &exitTime, &kernelTime, &userTime) != 0)
  261. return GetTime64(userTime) + GetTime64(kernelTime);
  262. return (UInt64)GetTickCount() * 10000;
  263. #endif
  264. }
  265. static UInt64 GetUserFreq()
  266. {
  267. #ifdef USE_POSIX_TIME
  268. return CLOCKS_PER_SEC;
  269. #else
  270. return 10000000;
  271. #endif
  272. }
  273. class CBenchProgressStatus
  274. {
  275. #ifdef BENCH_MT
  276. NWindows::NSynchronization::CCriticalSection CS;
  277. #endif
  278. public:
  279. HRESULT Res;
  280. bool EncodeMode;
  281. void SetResult(HRESULT res)
  282. {
  283. #ifdef BENCH_MT
  284. NWindows::NSynchronization::CCriticalSectionLock lock(CS);
  285. #endif
  286. Res = res;
  287. }
  288. HRESULT GetResult()
  289. {
  290. #ifdef BENCH_MT
  291. NWindows::NSynchronization::CCriticalSectionLock lock(CS);
  292. #endif
  293. return Res;
  294. }
  295. };
  296. class CBenchProgressInfo:
  297. public ICompressProgressInfo,
  298. public CMyUnknownImp
  299. {
  300. public:
  301. CBenchProgressStatus *Status;
  302. CBenchInfo BenchInfo;
  303. HRESULT Res;
  304. IBenchCallback *callback;
  305. CBenchProgressInfo(): callback(0) {}
  306. MY_UNKNOWN_IMP
  307. STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize);
  308. };
  309. void SetStartTime(CBenchInfo &bi)
  310. {
  311. bi.GlobalFreq = GetFreq();
  312. bi.UserFreq = GetUserFreq();
  313. bi.GlobalTime = ::GetTimeCount();
  314. bi.UserTime = ::GetUserTime();
  315. }
  316. void SetFinishTime(const CBenchInfo &biStart, CBenchInfo &dest)
  317. {
  318. dest.GlobalFreq = GetFreq();
  319. dest.UserFreq = GetUserFreq();
  320. dest.GlobalTime = ::GetTimeCount() - biStart.GlobalTime;
  321. dest.UserTime = ::GetUserTime() - biStart.UserTime;
  322. }
  323. STDMETHODIMP CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize)
  324. {
  325. HRESULT res = Status->GetResult();
  326. if (res != S_OK)
  327. return res;
  328. if (!callback)
  329. return res;
  330. CBenchInfo info = BenchInfo;
  331. SetFinishTime(BenchInfo, info);
  332. if (Status->EncodeMode)
  333. {
  334. info.UnpackSize = *inSize;
  335. info.PackSize = *outSize;
  336. res = callback->SetEncodeResult(info, false);
  337. }
  338. else
  339. {
  340. info.PackSize = BenchInfo.PackSize + *inSize;
  341. info.UnpackSize = BenchInfo.UnpackSize + *outSize;
  342. res = callback->SetDecodeResult(info, false);
  343. }
  344. if (res != S_OK)
  345. Status->SetResult(res);
  346. return res;
  347. }
  348. static const int kSubBits = 8;
  349. static UInt32 GetLogSize(UInt32 size)
  350. {
  351. for (int i = kSubBits; i < 32; i++)
  352. for (UInt32 j = 0; j < (1 << kSubBits); j++)
  353. if (size <= (((UInt32)1) << i) + (j << (i - kSubBits)))
  354. return (i << kSubBits) + j;
  355. return (32 << kSubBits);
  356. }
  357. static void NormalizeVals(UInt64 &v1, UInt64 &v2)
  358. {
  359. while (v1 > 1000000)
  360. {
  361. v1 >>= 1;
  362. v2 >>= 1;
  363. }
  364. }
  365. UInt64 GetUsage(const CBenchInfo &info)
  366. {
  367. UInt64 userTime = info.UserTime;
  368. UInt64 userFreq = info.UserFreq;
  369. UInt64 globalTime = info.GlobalTime;
  370. UInt64 globalFreq = info.GlobalFreq;
  371. NormalizeVals(userTime, userFreq);
  372. NormalizeVals(globalFreq, globalTime);
  373. if (userFreq == 0)
  374. userFreq = 1;
  375. if (globalTime == 0)
  376. globalTime = 1;
  377. return userTime * globalFreq * 1000000 / userFreq / globalTime;
  378. }
  379. UInt64 GetRatingPerUsage(const CBenchInfo &info, UInt64 rating)
  380. {
  381. UInt64 userTime = info.UserTime;
  382. UInt64 userFreq = info.UserFreq;
  383. UInt64 globalTime = info.GlobalTime;
  384. UInt64 globalFreq = info.GlobalFreq;
  385. NormalizeVals(userFreq, userTime);
  386. NormalizeVals(globalTime, globalFreq);
  387. if (globalFreq == 0)
  388. globalFreq = 1;
  389. if (userTime == 0)
  390. userTime = 1;
  391. return userFreq * globalTime / globalFreq * rating / userTime;
  392. }
  393. static UInt64 MyMultDiv64(UInt64 value, UInt64 elapsedTime, UInt64 freq)
  394. {
  395. UInt64 elTime = elapsedTime;
  396. NormalizeVals(freq, elTime);
  397. if (elTime == 0)
  398. elTime = 1;
  399. return value * freq / elTime;
  400. }
  401. UInt64 GetCompressRating(UInt32 dictionarySize, UInt64 elapsedTime, UInt64 freq, UInt64 size)
  402. {
  403. UInt64 t = GetLogSize(dictionarySize) - (kBenchMinDicLogSize << kSubBits);
  404. // UInt64 numCommandsForOne = 1000 + ((t * t * 7) >> (2 * kSubBits)); // AMD K8
  405. UInt64 numCommandsForOne = 870 + ((t * t * 5) >> (2 * kSubBits)); // Intel Core2
  406. UInt64 numCommands = (UInt64)(size) * numCommandsForOne;
  407. return MyMultDiv64(numCommands, elapsedTime, freq);
  408. }
  409. UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt32 numIterations)
  410. {
  411. // UInt64 numCommands = (inSize * 216 + outSize * 14) * numIterations; // AMD K8
  412. UInt64 numCommands = (inSize * 220 + outSize * 8) * numIterations; // Intel Core2
  413. return MyMultDiv64(numCommands, elapsedTime, freq);
  414. }
  415. #ifdef EXTERNAL_LZMA
  416. typedef UInt32 (WINAPI * CreateObjectPointer)(const GUID *clsID,
  417. const GUID *interfaceID, void **outObject);
  418. #endif
  419. struct CEncoderInfo;
  420. struct CEncoderInfo
  421. {
  422. #ifdef BENCH_MT
  423. NWindows::CThread thread[2];
  424. #endif
  425. CMyComPtr<ICompressCoder> encoder;
  426. CBenchProgressInfo *progressInfoSpec[2];
  427. CMyComPtr<ICompressProgressInfo> progressInfo[2];
  428. UInt32 NumIterations;
  429. #ifdef USE_ALLOCA
  430. size_t AllocaSize;
  431. #endif
  432. struct CDecoderInfo
  433. {
  434. CEncoderInfo *Encoder;
  435. UInt32 DecoderIndex;
  436. #ifdef USE_ALLOCA
  437. size_t AllocaSize;
  438. #endif
  439. bool CallbackMode;
  440. };
  441. CDecoderInfo decodersInfo[2];
  442. CMyComPtr<ICompressCoder> decoders[2];
  443. HRESULT Results[2];
  444. CBenchmarkOutStream *outStreamSpec;
  445. CMyComPtr<ISequentialOutStream> outStream;
  446. IBenchCallback *callback;
  447. UInt32 crc;
  448. UInt32 kBufferSize;
  449. UInt32 compressedSize;
  450. CBenchRandomGenerator rg;
  451. CBenchmarkOutStream *propStreamSpec;
  452. CMyComPtr<ISequentialOutStream> propStream;
  453. HRESULT Init(UInt32 dictionarySize, UInt32 numThreads, CBaseRandomGenerator *rg);
  454. HRESULT Encode();
  455. HRESULT Decode(UInt32 decoderIndex);
  456. CEncoderInfo(): outStreamSpec(0), callback(0), propStreamSpec(0) {}
  457. #ifdef BENCH_MT
  458. static THREAD_FUNC_DECL EncodeThreadFunction(void *param)
  459. {
  460. CEncoderInfo *encoder = (CEncoderInfo *)param;
  461. #ifdef USE_ALLOCA
  462. alloca(encoder->AllocaSize);
  463. #endif
  464. HRESULT res = encoder->Encode();
  465. encoder->Results[0] = res;
  466. if (res != S_OK)
  467. encoder->progressInfoSpec[0]->Status->SetResult(res);
  468. return 0;
  469. }
  470. static THREAD_FUNC_DECL DecodeThreadFunction(void *param)
  471. {
  472. CDecoderInfo *decoder = (CDecoderInfo *)param;
  473. #ifdef USE_ALLOCA
  474. alloca(decoder->AllocaSize);
  475. #endif
  476. CEncoderInfo *encoder = decoder->Encoder;
  477. encoder->Results[decoder->DecoderIndex] = encoder->Decode(decoder->DecoderIndex);
  478. return 0;
  479. }
  480. HRESULT CreateEncoderThread()
  481. {
  482. return thread[0].Create(EncodeThreadFunction, this);
  483. }
  484. HRESULT CreateDecoderThread(int index, bool callbackMode
  485. #ifdef USE_ALLOCA
  486. , size_t allocaSize
  487. #endif
  488. )
  489. {
  490. CDecoderInfo &decoder = decodersInfo[index];
  491. decoder.DecoderIndex = index;
  492. decoder.Encoder = this;
  493. #ifdef USE_ALLOCA
  494. decoder.AllocaSize = allocaSize;
  495. #endif
  496. decoder.CallbackMode = callbackMode;
  497. return thread[index].Create(DecodeThreadFunction, &decoder);
  498. }
  499. #endif
  500. };
  501. HRESULT CEncoderInfo::Init(UInt32 dictionarySize, UInt32 numThreads, CBaseRandomGenerator *rgLoc)
  502. {
  503. rg.Set(rgLoc);
  504. kBufferSize = dictionarySize + kAdditionalSize;
  505. UInt32 kCompressedBufferSize = (kBufferSize / 2) + kCompressedAdditionalSize;
  506. if (!rg.Alloc(kBufferSize))
  507. return E_OUTOFMEMORY;
  508. rg.Generate();
  509. crc = CrcCalc(rg.Buffer, rg.BufferSize);
  510. outStreamSpec = new CBenchmarkOutStream;
  511. if (!outStreamSpec->Alloc(kCompressedBufferSize))
  512. return E_OUTOFMEMORY;
  513. outStream = outStreamSpec;
  514. propStreamSpec = 0;
  515. if (!propStream)
  516. {
  517. propStreamSpec = new CBenchmarkOutStream;
  518. propStream = propStreamSpec;
  519. }
  520. if (!propStreamSpec->Alloc(kMaxLzmaPropSize))
  521. return E_OUTOFMEMORY;
  522. propStreamSpec->Init();
  523. PROPID propIDs[] =
  524. {
  525. NCoderPropID::kDictionarySize,
  526. NCoderPropID::kMultiThread
  527. };
  528. const int kNumProps = sizeof(propIDs) / sizeof(propIDs[0]);
  529. PROPVARIANT properties[kNumProps];
  530. properties[0].vt = VT_UI4;
  531. properties[0].ulVal = (UInt32)dictionarySize;
  532. properties[1].vt = VT_BOOL;
  533. properties[1].boolVal = (numThreads > 1) ? VARIANT_TRUE : VARIANT_FALSE;
  534. {
  535. CMyComPtr<ICompressSetCoderProperties> setCoderProperties;
  536. RINOK(encoder.QueryInterface(IID_ICompressSetCoderProperties, &setCoderProperties));
  537. if (!setCoderProperties)
  538. return E_FAIL;
  539. RINOK(setCoderProperties->SetCoderProperties(propIDs, properties, kNumProps));
  540. CMyComPtr<ICompressWriteCoderProperties> writeCoderProperties;
  541. encoder.QueryInterface(IID_ICompressWriteCoderProperties, &writeCoderProperties);
  542. if (writeCoderProperties)
  543. {
  544. RINOK(writeCoderProperties->WriteCoderProperties(propStream));
  545. }
  546. }
  547. return S_OK;
  548. }
  549. HRESULT CEncoderInfo::Encode()
  550. {
  551. CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream;
  552. CMyComPtr<ISequentialInStream> inStream = inStreamSpec;
  553. inStreamSpec->Init(rg.Buffer, rg.BufferSize);
  554. outStreamSpec->Init();
  555. RINOK(encoder->Code(inStream, outStream, 0, 0, progressInfo[0]));
  556. compressedSize = outStreamSpec->Pos;
  557. encoder.Release();
  558. return S_OK;
  559. }
  560. HRESULT CEncoderInfo::Decode(UInt32 decoderIndex)
  561. {
  562. CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream;
  563. CMyComPtr<ISequentialInStream> inStream = inStreamSpec;
  564. CMyComPtr<ICompressCoder> &decoder = decoders[decoderIndex];
  565. CMyComPtr<ICompressSetDecoderProperties2> compressSetDecoderProperties;
  566. decoder.QueryInterface(IID_ICompressSetDecoderProperties2, &compressSetDecoderProperties);
  567. if (!compressSetDecoderProperties)
  568. return E_FAIL;
  569. CCrcOutStream *crcOutStreamSpec = new CCrcOutStream;
  570. CMyComPtr<ISequentialOutStream> crcOutStream = crcOutStreamSpec;
  571. CBenchProgressInfo *pi = progressInfoSpec[decoderIndex];
  572. pi->BenchInfo.UnpackSize = 0;
  573. pi->BenchInfo.PackSize = 0;
  574. for (UInt32 j = 0; j < NumIterations; j++)
  575. {
  576. inStreamSpec->Init(outStreamSpec->Buffer, compressedSize);
  577. crcOutStreamSpec->Init();
  578. RINOK(compressSetDecoderProperties->SetDecoderProperties2(propStreamSpec->Buffer, propStreamSpec->Pos));
  579. UInt64 outSize = kBufferSize;
  580. RINOK(decoder->Code(inStream, crcOutStream, 0, &outSize, progressInfo[decoderIndex]));
  581. if (CRC_GET_DIGEST(crcOutStreamSpec->Crc) != crc)
  582. return S_FALSE;
  583. pi->BenchInfo.UnpackSize += kBufferSize;
  584. pi->BenchInfo.PackSize += compressedSize;
  585. }
  586. decoder.Release();
  587. return S_OK;
  588. }
  589. static const UInt32 kNumThreadsMax = (1 << 16);
  590. struct CBenchEncoders
  591. {
  592. CEncoderInfo *encoders;
  593. CBenchEncoders(UInt32 num): encoders(0) { encoders = new CEncoderInfo[num]; }
  594. ~CBenchEncoders() { delete []encoders; }
  595. };
  596. HRESULT LzmaBench(
  597. #ifdef EXTERNAL_LZMA
  598. CCodecs *codecs,
  599. #endif
  600. UInt32 numThreads, UInt32 dictionarySize, IBenchCallback *callback)
  601. {
  602. UInt32 numEncoderThreads =
  603. #ifdef BENCH_MT
  604. (numThreads > 1 ? numThreads / 2 : 1);
  605. #else
  606. 1;
  607. #endif
  608. UInt32 numSubDecoderThreads =
  609. #ifdef BENCH_MT
  610. (numThreads > 1 ? 2 : 1);
  611. #else
  612. 1;
  613. #endif
  614. if (dictionarySize < (1 << kBenchMinDicLogSize) || numThreads < 1 || numEncoderThreads > kNumThreadsMax)
  615. {
  616. return E_INVALIDARG;
  617. }
  618. CBenchEncoders encodersSpec(numEncoderThreads);
  619. CEncoderInfo *encoders = encodersSpec.encoders;
  620. #ifdef EXTERNAL_LZMA
  621. UString name = L"LZMA";
  622. #endif
  623. UInt32 i;
  624. for (i = 0; i < numEncoderThreads; i++)
  625. {
  626. CEncoderInfo &encoder = encoders[i];
  627. encoder.callback = (i == 0) ? callback : 0;
  628. #ifdef EXTERNAL_LZMA
  629. RINOK(codecs->CreateCoder(name, true, encoder.encoder));
  630. #else
  631. encoder.encoder = new NCompress::NLZMA::CEncoder;
  632. #endif
  633. for (UInt32 j = 0; j < numSubDecoderThreads; j++)
  634. {
  635. #ifdef EXTERNAL_LZMA
  636. RINOK(codecs->CreateCoder(name, false, encoder.decoders[j]));
  637. #else
  638. encoder.decoders[j] = new NCompress::NLZMA::CDecoder;
  639. #endif
  640. }
  641. }
  642. CBaseRandomGenerator rg;
  643. rg.Init();
  644. for (i = 0; i < numEncoderThreads; i++)
  645. {
  646. RINOK(encoders[i].Init(dictionarySize, numThreads, &rg));
  647. }
  648. CBenchProgressStatus status;
  649. status.Res = S_OK;
  650. status.EncodeMode = true;
  651. for (i = 0; i < numEncoderThreads; i++)
  652. {
  653. CEncoderInfo &encoder = encoders[i];
  654. for (int j = 0; j < 2; j++)
  655. {
  656. encoder.progressInfo[j] = encoder.progressInfoSpec[j] = new CBenchProgressInfo;
  657. encoder.progressInfoSpec[j]->Status = &status;
  658. }
  659. if (i == 0)
  660. {
  661. encoder.progressInfoSpec[0]->callback = callback;
  662. encoder.progressInfoSpec[0]->BenchInfo.NumIterations = numEncoderThreads;
  663. SetStartTime(encoder.progressInfoSpec[0]->BenchInfo);
  664. }
  665. #ifdef BENCH_MT
  666. if (numEncoderThreads > 1)
  667. {
  668. #ifdef USE_ALLOCA
  669. encoder.AllocaSize = (i * 16 * 21) & 0x7FF;
  670. #endif
  671. RINOK(encoder.CreateEncoderThread())
  672. }
  673. else
  674. #endif
  675. {
  676. RINOK(encoder.Encode());
  677. }
  678. }
  679. #ifdef BENCH_MT
  680. if (numEncoderThreads > 1)
  681. for (i = 0; i < numEncoderThreads; i++)
  682. encoders[i].thread[0].Wait();
  683. #endif
  684. RINOK(status.Res);
  685. CBenchInfo info;
  686. SetFinishTime(encoders[0].progressInfoSpec[0]->BenchInfo, info);
  687. info.UnpackSize = 0;
  688. info.PackSize = 0;
  689. info.NumIterations = 1; // progressInfoSpec->NumIterations;
  690. for (i = 0; i < numEncoderThreads; i++)
  691. {
  692. CEncoderInfo &encoder = encoders[i];
  693. info.UnpackSize += encoder.kBufferSize;
  694. info.PackSize += encoder.compressedSize;
  695. }
  696. RINOK(callback->SetEncodeResult(info, true));
  697. status.Res = S_OK;
  698. status.EncodeMode = false;
  699. UInt32 numDecoderThreads = numEncoderThreads * numSubDecoderThreads;
  700. for (i = 0; i < numEncoderThreads; i++)
  701. {
  702. CEncoderInfo &encoder = encoders[i];
  703. encoder.NumIterations = 2 + kUncompressMinBlockSize / encoder.kBufferSize;
  704. if (i == 0)
  705. {
  706. encoder.progressInfoSpec[0]->callback = callback;
  707. encoder.progressInfoSpec[0]->BenchInfo.NumIterations = numDecoderThreads;
  708. SetStartTime(encoder.progressInfoSpec[0]->BenchInfo);
  709. }
  710. #ifdef BENCH_MT
  711. if (numDecoderThreads > 1)
  712. {
  713. for (UInt32 j = 0; j < numSubDecoderThreads; j++)
  714. {
  715. size_t allocaSize = ((i * numSubDecoderThreads + j) * 16 * 21) & 0x7FF;
  716. HRESULT res = encoder.CreateDecoderThread(j, (i == 0 && j == 0)
  717. #ifdef USE_ALLOCA
  718. , allocaSize
  719. #endif
  720. );
  721. RINOK(res);
  722. }
  723. }
  724. else
  725. #endif
  726. {
  727. RINOK(encoder.Decode(0));
  728. }
  729. }
  730. #ifdef BENCH_MT
  731. HRESULT res = S_OK;
  732. if (numDecoderThreads > 1)
  733. for (i = 0; i < numEncoderThreads; i++)
  734. for (UInt32 j = 0; j < numSubDecoderThreads; j++)
  735. {
  736. CEncoderInfo &encoder = encoders[i];
  737. encoder.thread[j].Wait();
  738. if (encoder.Results[j] != S_OK)
  739. res = encoder.Results[j];
  740. }
  741. RINOK(res);
  742. #endif
  743. RINOK(status.Res);
  744. SetFinishTime(encoders[0].progressInfoSpec[0]->BenchInfo, info);
  745. info.UnpackSize = 0;
  746. info.PackSize = 0;
  747. info.NumIterations = numSubDecoderThreads * encoders[0].NumIterations;
  748. for (i = 0; i < numEncoderThreads; i++)
  749. {
  750. CEncoderInfo &encoder = encoders[i];
  751. info.UnpackSize += encoder.kBufferSize;
  752. info.PackSize += encoder.compressedSize;
  753. }
  754. RINOK(callback->SetDecodeResult(info, false));
  755. RINOK(callback->SetDecodeResult(info, true));
  756. return S_OK;
  757. }
  758. inline UInt64 GetLZMAUsage(bool multiThread, UInt32 dictionary)
  759. {
  760. UInt32 hs = dictionary - 1;
  761. hs |= (hs >> 1);
  762. hs |= (hs >> 2);
  763. hs |= (hs >> 4);
  764. hs |= (hs >> 8);
  765. hs >>= 1;
  766. hs |= 0xFFFF;
  767. if (hs > (1 << 24))
  768. hs >>= 1;
  769. hs++;
  770. return ((hs + (1 << 16)) + (UInt64)dictionary * 2) * 4 + (UInt64)dictionary * 3 / 2 +
  771. (1 << 20) + (multiThread ? (6 << 20) : 0);
  772. }
  773. UInt64 GetBenchMemoryUsage(UInt32 numThreads, UInt32 dictionary)
  774. {
  775. const UInt32 kBufferSize = dictionary;
  776. const UInt32 kCompressedBufferSize = (kBufferSize / 2);
  777. UInt32 numSubThreads = (numThreads > 1) ? 2 : 1;
  778. UInt32 numBigThreads = numThreads / numSubThreads;
  779. return (kBufferSize + kCompressedBufferSize +
  780. GetLZMAUsage((numThreads > 1), dictionary) + (2 << 20)) * numBigThreads;
  781. }
  782. static bool CrcBig(const void *data, UInt32 size, UInt32 numCycles, UInt32 crcBase)
  783. {
  784. for (UInt32 i = 0; i < numCycles; i++)
  785. if (CrcCalc(data, size) != crcBase)
  786. return false;
  787. return true;
  788. }
  789. #ifdef BENCH_MT
  790. struct CCrcInfo
  791. {
  792. NWindows::CThread Thread;
  793. const Byte *Data;
  794. UInt32 Size;
  795. UInt32 NumCycles;
  796. UInt32 Crc;
  797. bool Res;
  798. void Wait()
  799. {
  800. Thread.Wait();
  801. Thread.Close();
  802. }
  803. };
  804. static THREAD_FUNC_DECL CrcThreadFunction(void *param)
  805. {
  806. CCrcInfo *p = (CCrcInfo *)param;
  807. p->Res = CrcBig(p->Data, p->Size, p->NumCycles, p->Crc);
  808. return 0;
  809. }
  810. struct CCrcThreads
  811. {
  812. UInt32 NumThreads;
  813. CCrcInfo *Items;
  814. CCrcThreads(): Items(0), NumThreads(0) {}
  815. void WaitAll()
  816. {
  817. for (UInt32 i = 0; i < NumThreads; i++)
  818. Items[i].Wait();
  819. NumThreads = 0;
  820. }
  821. ~CCrcThreads()
  822. {
  823. WaitAll();
  824. delete []Items;
  825. }
  826. };
  827. #endif
  828. static UInt32 CrcCalc1(const Byte *buf, UInt32 size)
  829. {
  830. UInt32 crc = CRC_INIT_VAL;;
  831. for (UInt32 i = 0; i < size; i++)
  832. crc = CRC_UPDATE_BYTE(crc, buf[i]);
  833. return CRC_GET_DIGEST(crc);
  834. }
  835. static void RandGen(Byte *buf, UInt32 size, CBaseRandomGenerator &RG)
  836. {
  837. for (UInt32 i = 0; i < size; i++)
  838. buf[i] = (Byte)RG.GetRnd();
  839. }
  840. static UInt32 RandGenCrc(Byte *buf, UInt32 size, CBaseRandomGenerator &RG)
  841. {
  842. RandGen(buf, size, RG);
  843. return CrcCalc1(buf, size);
  844. }
  845. bool CrcInternalTest()
  846. {
  847. CBenchBuffer buffer;
  848. const UInt32 kBufferSize0 = (1 << 8);
  849. const UInt32 kBufferSize1 = (1 << 10);
  850. const UInt32 kCheckSize = (1 << 5);
  851. if (!buffer.Alloc(kBufferSize0 + kBufferSize1))
  852. return false;
  853. Byte *buf = buffer.Buffer;
  854. UInt32 i;
  855. for (i = 0; i < kBufferSize0; i++)
  856. buf[i] = (Byte)i;
  857. UInt32 crc1 = CrcCalc1(buf, kBufferSize0);
  858. if (crc1 != 0x29058C73)
  859. return false;
  860. CBaseRandomGenerator RG;
  861. RandGen(buf + kBufferSize0, kBufferSize1, RG);
  862. for (i = 0; i < kBufferSize0 + kBufferSize1 - kCheckSize; i++)
  863. for (UInt32 j = 0; j < kCheckSize; j++)
  864. if (CrcCalc1(buf + i, j) != CrcCalc(buf + i, j))
  865. return false;
  866. return true;
  867. }
  868. HRESULT CrcBench(UInt32 numThreads, UInt32 bufferSize, UInt64 &speed)
  869. {
  870. if (numThreads == 0)
  871. numThreads = 1;
  872. CBenchBuffer buffer;
  873. size_t totalSize = (size_t)bufferSize * numThreads;
  874. if (totalSize / numThreads != bufferSize)
  875. return E_OUTOFMEMORY;
  876. if (!buffer.Alloc(totalSize))
  877. return E_OUTOFMEMORY;
  878. Byte *buf = buffer.Buffer;
  879. CBaseRandomGenerator RG;
  880. UInt32 numCycles = ((UInt32)1 << 30) / ((bufferSize >> 2) + 1) + 1;
  881. UInt64 timeVal;
  882. #ifdef BENCH_MT
  883. CCrcThreads threads;
  884. if (numThreads > 1)
  885. {
  886. threads.Items = new CCrcInfo[numThreads];
  887. UInt32 i;
  888. for (i = 0; i < numThreads; i++)
  889. {
  890. CCrcInfo &info = threads.Items[i];
  891. Byte *data = buf + (size_t)bufferSize * i;
  892. info.Data = data;
  893. info.NumCycles = numCycles;
  894. info.Size = bufferSize;
  895. info.Crc = RandGenCrc(data, bufferSize, RG);
  896. }
  897. timeVal = GetTimeCount();
  898. for (i = 0; i < numThreads; i++)
  899. {
  900. CCrcInfo &info = threads.Items[i];
  901. RINOK(info.Thread.Create(CrcThreadFunction, &info));
  902. threads.NumThreads++;
  903. }
  904. threads.WaitAll();
  905. for (i = 0; i < numThreads; i++)
  906. if (!threads.Items[i].Res)
  907. return S_FALSE;
  908. }
  909. else
  910. #endif
  911. {
  912. UInt32 crc = RandGenCrc(buf, bufferSize, RG);
  913. timeVal = GetTimeCount();
  914. if (!CrcBig(buf, bufferSize, numCycles, crc))
  915. return S_FALSE;
  916. }
  917. timeVal = GetTimeCount() - timeVal;
  918. if (timeVal == 0)
  919. timeVal = 1;
  920. UInt64 size = (UInt64)numCycles * totalSize;
  921. speed = MyMultDiv64(size, timeVal, GetFreq());
  922. return S_OK;
  923. }