alffplay.cpp 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181
  1. /*
  2. * An example showing how to play a stream sync'd to video, using ffmpeg.
  3. *
  4. * Requires C++14.
  5. */
  6. #include <condition_variable>
  7. #include <functional>
  8. #include <algorithm>
  9. #include <iostream>
  10. #include <utility>
  11. #include <iomanip>
  12. #include <cstdint>
  13. #include <cstring>
  14. #include <cstdlib>
  15. #include <atomic>
  16. #include <cerrno>
  17. #include <chrono>
  18. #include <cstdio>
  19. #include <future>
  20. #include <memory>
  21. #include <string>
  22. #include <thread>
  23. #include <vector>
  24. #include <array>
  25. #include <cmath>
  26. #include <deque>
  27. #include <mutex>
  28. #include <ratio>
  29. #ifdef __GNUC__
  30. _Pragma("GCC diagnostic push")
  31. _Pragma("GCC diagnostic ignored \"-Wconversion\"")
  32. _Pragma("GCC diagnostic ignored \"-Wold-style-cast\"")
  33. #endif
  34. extern "C" {
  35. #include "libavcodec/avcodec.h"
  36. #include "libavformat/avformat.h"
  37. #include "libavformat/avio.h"
  38. #include "libavformat/version.h"
  39. #include "libavutil/avutil.h"
  40. #include "libavutil/error.h"
  41. #include "libavutil/frame.h"
  42. #include "libavutil/mem.h"
  43. #include "libavutil/pixfmt.h"
  44. #include "libavutil/rational.h"
  45. #include "libavutil/samplefmt.h"
  46. #include "libavutil/time.h"
  47. #include "libavutil/version.h"
  48. #include "libavutil/channel_layout.h"
  49. #include "libswscale/swscale.h"
  50. #include "libswresample/swresample.h"
  51. constexpr auto AVNoPtsValue = AV_NOPTS_VALUE;
  52. constexpr auto AVErrorEOF = AVERROR_EOF;
  53. struct SwsContext;
  54. }
  55. #define SDL_MAIN_HANDLED
  56. #include "SDL.h"
  57. #ifdef __GNUC__
  58. _Pragma("GCC diagnostic pop")
  59. #endif
  60. #include "AL/alc.h"
  61. #include "AL/al.h"
  62. #include "AL/alext.h"
  63. #include "common/alhelpers.h"
  64. namespace {
  65. inline constexpr int64_t operator "" _i64(unsigned long long int n) noexcept { return static_cast<int64_t>(n); }
  66. #ifndef M_PI
  67. #define M_PI (3.14159265358979323846)
  68. #endif
  69. using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1_i64<<32)>>;
  70. using nanoseconds = std::chrono::nanoseconds;
  71. using microseconds = std::chrono::microseconds;
  72. using milliseconds = std::chrono::milliseconds;
  73. using seconds = std::chrono::seconds;
  74. using seconds_d64 = std::chrono::duration<double>;
  75. using std::chrono::duration_cast;
  76. const std::string AppName{"alffplay"};
  77. ALenum DirectOutMode{AL_FALSE};
  78. bool EnableWideStereo{false};
  79. bool EnableUhj{false};
  80. bool EnableSuperStereo{false};
  81. bool DisableVideo{false};
  82. LPALGETSOURCEI64VSOFT alGetSourcei64vSOFT;
  83. LPALCGETINTEGER64VSOFT alcGetInteger64vSOFT;
  84. LPALEVENTCONTROLSOFT alEventControlSOFT;
  85. LPALEVENTCALLBACKSOFT alEventCallbackSOFT;
  86. LPALBUFFERCALLBACKSOFT alBufferCallbackSOFT;
  87. const seconds AVNoSyncThreshold{10};
  88. #define VIDEO_PICTURE_QUEUE_SIZE 24
  89. const seconds_d64 AudioSyncThreshold{0.03};
  90. const milliseconds AudioSampleCorrectionMax{50};
  91. /* Averaging filter coefficient for audio sync. */
  92. #define AUDIO_DIFF_AVG_NB 20
  93. const double AudioAvgFilterCoeff{std::pow(0.01, 1.0/AUDIO_DIFF_AVG_NB)};
  94. /* Per-buffer size, in time */
  95. constexpr milliseconds AudioBufferTime{20};
  96. /* Buffer total size, in time (should be divisible by the buffer time) */
  97. constexpr milliseconds AudioBufferTotalTime{800};
  98. constexpr auto AudioBufferCount = AudioBufferTotalTime / AudioBufferTime;
  99. enum {
  100. FF_MOVIE_DONE_EVENT = SDL_USEREVENT
  101. };
  102. enum class SyncMaster {
  103. Audio,
  104. Video,
  105. External,
  106. Default = Audio
  107. };
  108. inline microseconds get_avtime()
  109. { return microseconds{av_gettime()}; }
  110. /* Define unique_ptrs to auto-cleanup associated ffmpeg objects. */
  111. struct AVIOContextDeleter {
  112. void operator()(AVIOContext *ptr) { avio_closep(&ptr); }
  113. };
  114. using AVIOContextPtr = std::unique_ptr<AVIOContext,AVIOContextDeleter>;
  115. struct AVFormatCtxDeleter {
  116. void operator()(AVFormatContext *ptr) { avformat_close_input(&ptr); }
  117. };
  118. using AVFormatCtxPtr = std::unique_ptr<AVFormatContext,AVFormatCtxDeleter>;
  119. struct AVCodecCtxDeleter {
  120. void operator()(AVCodecContext *ptr) { avcodec_free_context(&ptr); }
  121. };
  122. using AVCodecCtxPtr = std::unique_ptr<AVCodecContext,AVCodecCtxDeleter>;
  123. struct AVPacketDeleter {
  124. void operator()(AVPacket *pkt) { av_packet_free(&pkt); }
  125. };
  126. using AVPacketPtr = std::unique_ptr<AVPacket,AVPacketDeleter>;
  127. struct AVFrameDeleter {
  128. void operator()(AVFrame *ptr) { av_frame_free(&ptr); }
  129. };
  130. using AVFramePtr = std::unique_ptr<AVFrame,AVFrameDeleter>;
  131. struct SwrContextDeleter {
  132. void operator()(SwrContext *ptr) { swr_free(&ptr); }
  133. };
  134. using SwrContextPtr = std::unique_ptr<SwrContext,SwrContextDeleter>;
  135. struct SwsContextDeleter {
  136. void operator()(SwsContext *ptr) { sws_freeContext(ptr); }
  137. };
  138. using SwsContextPtr = std::unique_ptr<SwsContext,SwsContextDeleter>;
  139. struct ChannelLayout : public AVChannelLayout {
  140. ChannelLayout() : AVChannelLayout{} { }
  141. ~ChannelLayout() { av_channel_layout_uninit(this); }
  142. };
  143. template<size_t SizeLimit>
  144. class DataQueue {
  145. std::mutex mPacketMutex, mFrameMutex;
  146. std::condition_variable mPacketCond;
  147. std::condition_variable mInFrameCond, mOutFrameCond;
  148. std::deque<AVPacketPtr> mPackets;
  149. size_t mTotalSize{0};
  150. bool mFinished{false};
  151. AVPacketPtr getPacket()
  152. {
  153. std::unique_lock<std::mutex> plock{mPacketMutex};
  154. while(mPackets.empty() && !mFinished)
  155. mPacketCond.wait(plock);
  156. if(mPackets.empty())
  157. return nullptr;
  158. auto ret = std::move(mPackets.front());
  159. mPackets.pop_front();
  160. mTotalSize -= static_cast<unsigned int>(ret->size);
  161. return ret;
  162. }
  163. public:
  164. int sendPacket(AVCodecContext *codecctx)
  165. {
  166. AVPacketPtr packet{getPacket()};
  167. int ret{};
  168. {
  169. std::unique_lock<std::mutex> flock{mFrameMutex};
  170. while((ret=avcodec_send_packet(codecctx, packet.get())) == AVERROR(EAGAIN))
  171. mInFrameCond.wait_for(flock, milliseconds{50});
  172. }
  173. mOutFrameCond.notify_one();
  174. if(!packet)
  175. {
  176. if(!ret) return AVErrorEOF;
  177. std::cerr<< "Failed to send flush packet: "<<ret <<std::endl;
  178. return ret;
  179. }
  180. if(ret < 0)
  181. std::cerr<< "Failed to send packet: "<<ret <<std::endl;
  182. return ret;
  183. }
  184. int receiveFrame(AVCodecContext *codecctx, AVFrame *frame)
  185. {
  186. int ret{};
  187. {
  188. std::unique_lock<std::mutex> flock{mFrameMutex};
  189. while((ret=avcodec_receive_frame(codecctx, frame)) == AVERROR(EAGAIN))
  190. mOutFrameCond.wait_for(flock, milliseconds{50});
  191. }
  192. mInFrameCond.notify_one();
  193. return ret;
  194. }
  195. void setFinished()
  196. {
  197. {
  198. std::lock_guard<std::mutex> _{mPacketMutex};
  199. mFinished = true;
  200. }
  201. mPacketCond.notify_one();
  202. }
  203. void flush()
  204. {
  205. {
  206. std::lock_guard<std::mutex> _{mPacketMutex};
  207. mFinished = true;
  208. mPackets.clear();
  209. mTotalSize = 0;
  210. }
  211. mPacketCond.notify_one();
  212. }
  213. bool put(const AVPacket *pkt)
  214. {
  215. {
  216. std::unique_lock<std::mutex> lock{mPacketMutex};
  217. if(mTotalSize >= SizeLimit || mFinished)
  218. return false;
  219. mPackets.push_back(AVPacketPtr{av_packet_alloc()});
  220. if(av_packet_ref(mPackets.back().get(), pkt) != 0)
  221. {
  222. mPackets.pop_back();
  223. return true;
  224. }
  225. mTotalSize += static_cast<unsigned int>(mPackets.back()->size);
  226. }
  227. mPacketCond.notify_one();
  228. return true;
  229. }
  230. };
  231. struct MovieState;
  232. struct AudioState {
  233. MovieState &mMovie;
  234. AVStream *mStream{nullptr};
  235. AVCodecCtxPtr mCodecCtx;
  236. DataQueue<2*1024*1024> mQueue;
  237. /* Used for clock difference average computation */
  238. seconds_d64 mClockDiffAvg{0};
  239. /* Time of the next sample to be buffered */
  240. nanoseconds mCurrentPts{0};
  241. /* Device clock time that the stream started at. */
  242. nanoseconds mDeviceStartTime{nanoseconds::min()};
  243. /* Decompressed sample frame, and swresample context for conversion */
  244. AVFramePtr mDecodedFrame;
  245. SwrContextPtr mSwresCtx;
  246. /* Conversion format, for what gets fed to OpenAL */
  247. uint64_t mDstChanLayout{0};
  248. AVSampleFormat mDstSampleFmt{AV_SAMPLE_FMT_NONE};
  249. /* Storage of converted samples */
  250. uint8_t *mSamples{nullptr};
  251. int mSamplesLen{0}; /* In samples */
  252. int mSamplesPos{0};
  253. int mSamplesMax{0};
  254. std::unique_ptr<uint8_t[]> mBufferData;
  255. size_t mBufferDataSize{0};
  256. std::atomic<size_t> mReadPos{0};
  257. std::atomic<size_t> mWritePos{0};
  258. /* OpenAL format */
  259. ALenum mFormat{AL_NONE};
  260. ALuint mFrameSize{0};
  261. std::mutex mSrcMutex;
  262. std::condition_variable mSrcCond;
  263. std::atomic_flag mConnected;
  264. ALuint mSource{0};
  265. std::array<ALuint,AudioBufferCount> mBuffers{};
  266. ALuint mBufferIdx{0};
  267. AudioState(MovieState &movie) : mMovie(movie)
  268. { mConnected.test_and_set(std::memory_order_relaxed); }
  269. ~AudioState()
  270. {
  271. if(mSource)
  272. alDeleteSources(1, &mSource);
  273. if(mBuffers[0])
  274. alDeleteBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
  275. av_freep(&mSamples);
  276. }
  277. static void AL_APIENTRY eventCallbackC(ALenum eventType, ALuint object, ALuint param,
  278. ALsizei length, const ALchar *message, void *userParam)
  279. { static_cast<AudioState*>(userParam)->eventCallback(eventType, object, param, length, message); }
  280. void eventCallback(ALenum eventType, ALuint object, ALuint param, ALsizei length,
  281. const ALchar *message);
  282. static ALsizei AL_APIENTRY bufferCallbackC(void *userptr, void *data, ALsizei size)
  283. { return static_cast<AudioState*>(userptr)->bufferCallback(data, size); }
  284. ALsizei bufferCallback(void *data, ALsizei size);
  285. nanoseconds getClockNoLock();
  286. nanoseconds getClock()
  287. {
  288. std::lock_guard<std::mutex> lock{mSrcMutex};
  289. return getClockNoLock();
  290. }
  291. bool startPlayback();
  292. int getSync();
  293. int decodeFrame();
  294. bool readAudio(uint8_t *samples, unsigned int length, int &sample_skip);
  295. bool readAudio(int sample_skip);
  296. int handler();
  297. };
  298. struct VideoState {
  299. MovieState &mMovie;
  300. AVStream *mStream{nullptr};
  301. AVCodecCtxPtr mCodecCtx;
  302. DataQueue<14*1024*1024> mQueue;
  303. /* The pts of the currently displayed frame, and the time (av_gettime) it
  304. * was last updated - used to have running video pts
  305. */
  306. nanoseconds mDisplayPts{0};
  307. microseconds mDisplayPtsTime{microseconds::min()};
  308. std::mutex mDispPtsMutex;
  309. /* Swscale context for format conversion */
  310. SwsContextPtr mSwscaleCtx;
  311. struct Picture {
  312. AVFramePtr mFrame{};
  313. nanoseconds mPts{nanoseconds::min()};
  314. };
  315. std::array<Picture,VIDEO_PICTURE_QUEUE_SIZE> mPictQ;
  316. std::atomic<size_t> mPictQRead{0u}, mPictQWrite{1u};
  317. std::mutex mPictQMutex;
  318. std::condition_variable mPictQCond;
  319. SDL_Texture *mImage{nullptr};
  320. int mWidth{0}, mHeight{0}; /* Full texture size */
  321. bool mFirstUpdate{true};
  322. std::atomic<bool> mEOS{false};
  323. std::atomic<bool> mFinalUpdate{false};
  324. VideoState(MovieState &movie) : mMovie(movie) { }
  325. ~VideoState()
  326. {
  327. if(mImage)
  328. SDL_DestroyTexture(mImage);
  329. mImage = nullptr;
  330. }
  331. nanoseconds getClock();
  332. void display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame);
  333. void updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw);
  334. int handler();
  335. };
  336. struct MovieState {
  337. AVIOContextPtr mIOContext;
  338. AVFormatCtxPtr mFormatCtx;
  339. SyncMaster mAVSyncType{SyncMaster::Default};
  340. microseconds mClockBase{microseconds::min()};
  341. std::atomic<bool> mQuit{false};
  342. AudioState mAudio;
  343. VideoState mVideo;
  344. std::mutex mStartupMutex;
  345. std::condition_variable mStartupCond;
  346. bool mStartupDone{false};
  347. std::thread mParseThread;
  348. std::thread mAudioThread;
  349. std::thread mVideoThread;
  350. std::string mFilename;
  351. MovieState(std::string fname)
  352. : mAudio(*this), mVideo(*this), mFilename(std::move(fname))
  353. { }
  354. ~MovieState()
  355. {
  356. stop();
  357. if(mParseThread.joinable())
  358. mParseThread.join();
  359. }
  360. static int decode_interrupt_cb(void *ctx);
  361. bool prepare();
  362. void setTitle(SDL_Window *window);
  363. void stop();
  364. nanoseconds getClock();
  365. nanoseconds getMasterClock();
  366. nanoseconds getDuration();
  367. int streamComponentOpen(unsigned int stream_index);
  368. int parse_handler();
  369. };
  370. nanoseconds AudioState::getClockNoLock()
  371. {
  372. // The audio clock is the timestamp of the sample currently being heard.
  373. if(alcGetInteger64vSOFT)
  374. {
  375. // If device start time = min, we aren't playing yet.
  376. if(mDeviceStartTime == nanoseconds::min())
  377. return nanoseconds::zero();
  378. // Get the current device clock time and latency.
  379. auto device = alcGetContextsDevice(alcGetCurrentContext());
  380. ALCint64SOFT devtimes[2]{0,0};
  381. alcGetInteger64vSOFT(device, ALC_DEVICE_CLOCK_LATENCY_SOFT, 2, devtimes);
  382. auto latency = nanoseconds{devtimes[1]};
  383. auto device_time = nanoseconds{devtimes[0]};
  384. // The clock is simply the current device time relative to the recorded
  385. // start time. We can also subtract the latency to get more a accurate
  386. // position of where the audio device actually is in the output stream.
  387. return device_time - mDeviceStartTime - latency;
  388. }
  389. if(mBufferDataSize > 0)
  390. {
  391. if(mDeviceStartTime == nanoseconds::min())
  392. return nanoseconds::zero();
  393. /* With a callback buffer and no device clock, mDeviceStartTime is
  394. * actually the timestamp of the first sample frame played. The audio
  395. * clock, then, is that plus the current source offset.
  396. */
  397. ALint64SOFT offset[2];
  398. if(alGetSourcei64vSOFT)
  399. alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
  400. else
  401. {
  402. ALint ioffset;
  403. alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
  404. offset[0] = ALint64SOFT{ioffset} << 32;
  405. offset[1] = 0;
  406. }
  407. /* NOTE: The source state must be checked last, in case an underrun
  408. * occurs and the source stops between getting the state and retrieving
  409. * the offset+latency.
  410. */
  411. ALint status;
  412. alGetSourcei(mSource, AL_SOURCE_STATE, &status);
  413. nanoseconds pts{};
  414. if(status == AL_PLAYING || status == AL_PAUSED)
  415. pts = mDeviceStartTime - nanoseconds{offset[1]} +
  416. duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
  417. else
  418. {
  419. /* If the source is stopped, the pts of the next sample to be heard
  420. * is the pts of the next sample to be buffered, minus the amount
  421. * already in the buffer ready to play.
  422. */
  423. const size_t woffset{mWritePos.load(std::memory_order_acquire)};
  424. const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
  425. const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
  426. roffset};
  427. pts = mCurrentPts - nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
  428. }
  429. return pts;
  430. }
  431. /* The source-based clock is based on 4 components:
  432. * 1 - The timestamp of the next sample to buffer (mCurrentPts)
  433. * 2 - The length of the source's buffer queue
  434. * (AudioBufferTime*AL_BUFFERS_QUEUED)
  435. * 3 - The offset OpenAL is currently at in the source (the first value
  436. * from AL_SAMPLE_OFFSET_LATENCY_SOFT)
  437. * 4 - The latency between OpenAL and the DAC (the second value from
  438. * AL_SAMPLE_OFFSET_LATENCY_SOFT)
  439. *
  440. * Subtracting the length of the source queue from the next sample's
  441. * timestamp gives the timestamp of the sample at the start of the source
  442. * queue. Adding the source offset to that results in the timestamp for the
  443. * sample at OpenAL's current position, and subtracting the source latency
  444. * from that gives the timestamp of the sample currently at the DAC.
  445. */
  446. nanoseconds pts{mCurrentPts};
  447. if(mSource)
  448. {
  449. ALint64SOFT offset[2];
  450. if(alGetSourcei64vSOFT)
  451. alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
  452. else
  453. {
  454. ALint ioffset;
  455. alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
  456. offset[0] = ALint64SOFT{ioffset} << 32;
  457. offset[1] = 0;
  458. }
  459. ALint queued, status;
  460. alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
  461. alGetSourcei(mSource, AL_SOURCE_STATE, &status);
  462. /* If the source is AL_STOPPED, then there was an underrun and all
  463. * buffers are processed, so ignore the source queue. The audio thread
  464. * will put the source into an AL_INITIAL state and clear the queue
  465. * when it starts recovery.
  466. */
  467. if(status != AL_STOPPED)
  468. {
  469. pts -= AudioBufferTime*queued;
  470. pts += duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
  471. }
  472. /* Don't offset by the latency if the source isn't playing. */
  473. if(status == AL_PLAYING)
  474. pts -= nanoseconds{offset[1]};
  475. }
  476. return std::max(pts, nanoseconds::zero());
  477. }
  478. bool AudioState::startPlayback()
  479. {
  480. const size_t woffset{mWritePos.load(std::memory_order_acquire)};
  481. const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
  482. const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
  483. roffset};
  484. if(mBufferDataSize > 0)
  485. {
  486. if(readable == 0)
  487. return false;
  488. if(!alcGetInteger64vSOFT)
  489. mDeviceStartTime = mCurrentPts -
  490. nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
  491. }
  492. else
  493. {
  494. ALint queued{};
  495. alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
  496. if(queued == 0) return false;
  497. }
  498. alSourcePlay(mSource);
  499. if(alcGetInteger64vSOFT)
  500. {
  501. /* Subtract the total buffer queue time from the current pts to get the
  502. * pts of the start of the queue.
  503. */
  504. int64_t srctimes[2]{0,0};
  505. alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_CLOCK_SOFT, srctimes);
  506. auto device_time = nanoseconds{srctimes[1]};
  507. auto src_offset = duration_cast<nanoseconds>(fixed32{srctimes[0]}) /
  508. mCodecCtx->sample_rate;
  509. /* The mixer may have ticked and incremented the device time and sample
  510. * offset, so subtract the source offset from the device time to get
  511. * the device time the source started at. Also subtract startpts to get
  512. * the device time the stream would have started at to reach where it
  513. * is now.
  514. */
  515. if(mBufferDataSize > 0)
  516. {
  517. nanoseconds startpts{mCurrentPts -
  518. nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate};
  519. mDeviceStartTime = device_time - src_offset - startpts;
  520. }
  521. else
  522. {
  523. nanoseconds startpts{mCurrentPts - AudioBufferTotalTime};
  524. mDeviceStartTime = device_time - src_offset - startpts;
  525. }
  526. }
  527. return true;
  528. }
  529. int AudioState::getSync()
  530. {
  531. if(mMovie.mAVSyncType == SyncMaster::Audio)
  532. return 0;
  533. auto ref_clock = mMovie.getMasterClock();
  534. auto diff = ref_clock - getClockNoLock();
  535. if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
  536. {
  537. /* Difference is TOO big; reset accumulated average */
  538. mClockDiffAvg = seconds_d64::zero();
  539. return 0;
  540. }
  541. /* Accumulate the diffs */
  542. mClockDiffAvg = mClockDiffAvg*AudioAvgFilterCoeff + diff;
  543. auto avg_diff = mClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
  544. if(avg_diff < AudioSyncThreshold/2.0 && avg_diff > -AudioSyncThreshold)
  545. return 0;
  546. /* Constrain the per-update difference to avoid exceedingly large skips */
  547. diff = std::min<nanoseconds>(diff, AudioSampleCorrectionMax);
  548. return static_cast<int>(duration_cast<seconds>(diff*mCodecCtx->sample_rate).count());
  549. }
  550. int AudioState::decodeFrame()
  551. {
  552. do {
  553. while(int ret{mQueue.receiveFrame(mCodecCtx.get(), mDecodedFrame.get())})
  554. {
  555. if(ret == AVErrorEOF) return 0;
  556. std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
  557. }
  558. } while(mDecodedFrame->nb_samples <= 0);
  559. /* If provided, update w/ pts */
  560. if(mDecodedFrame->best_effort_timestamp != AVNoPtsValue)
  561. mCurrentPts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
  562. static_cast<double>(mDecodedFrame->best_effort_timestamp)});
  563. if(mDecodedFrame->nb_samples > mSamplesMax)
  564. {
  565. av_freep(&mSamples);
  566. av_samples_alloc(&mSamples, nullptr, mCodecCtx->ch_layout.nb_channels,
  567. mDecodedFrame->nb_samples, mDstSampleFmt, 0);
  568. mSamplesMax = mDecodedFrame->nb_samples;
  569. }
  570. /* Return the amount of sample frames converted */
  571. int data_size{swr_convert(mSwresCtx.get(), &mSamples, mDecodedFrame->nb_samples,
  572. const_cast<const uint8_t**>(mDecodedFrame->data), mDecodedFrame->nb_samples)};
  573. av_frame_unref(mDecodedFrame.get());
  574. return data_size;
  575. }
  576. /* Duplicates the sample at in to out, count times. The frame size is a
  577. * multiple of the template type size.
  578. */
  579. template<typename T>
  580. static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
  581. {
  582. auto *sample = reinterpret_cast<const T*>(in);
  583. auto *dst = reinterpret_cast<T*>(out);
  584. /* NOTE: frame_size is a multiple of sizeof(T). */
  585. size_t type_mult{frame_size / sizeof(T)};
  586. if(type_mult == 1)
  587. std::fill_n(dst, count, *sample);
  588. else for(size_t i{0};i < count;++i)
  589. {
  590. for(size_t j{0};j < type_mult;++j)
  591. dst[i*type_mult + j] = sample[j];
  592. }
  593. }
  594. static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
  595. {
  596. if((frame_size&7) == 0)
  597. sample_dup<uint64_t>(out, in, count, frame_size);
  598. else if((frame_size&3) == 0)
  599. sample_dup<uint32_t>(out, in, count, frame_size);
  600. else if((frame_size&1) == 0)
  601. sample_dup<uint16_t>(out, in, count, frame_size);
  602. else
  603. sample_dup<uint8_t>(out, in, count, frame_size);
  604. }
  605. bool AudioState::readAudio(uint8_t *samples, unsigned int length, int &sample_skip)
  606. {
  607. unsigned int audio_size{0};
  608. /* Read the next chunk of data, refill the buffer, and queue it
  609. * on the source */
  610. length /= mFrameSize;
  611. while(mSamplesLen > 0 && audio_size < length)
  612. {
  613. unsigned int rem{length - audio_size};
  614. if(mSamplesPos >= 0)
  615. {
  616. const auto len = static_cast<unsigned int>(mSamplesLen - mSamplesPos);
  617. if(rem > len) rem = len;
  618. std::copy_n(mSamples + static_cast<unsigned int>(mSamplesPos)*mFrameSize,
  619. rem*mFrameSize, samples);
  620. }
  621. else
  622. {
  623. rem = std::min(rem, static_cast<unsigned int>(-mSamplesPos));
  624. /* Add samples by copying the first sample */
  625. sample_dup(samples, mSamples, rem, mFrameSize);
  626. }
  627. mSamplesPos += rem;
  628. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  629. samples += rem*mFrameSize;
  630. audio_size += rem;
  631. while(mSamplesPos >= mSamplesLen)
  632. {
  633. mSamplesLen = decodeFrame();
  634. mSamplesPos = std::min(mSamplesLen, sample_skip);
  635. if(mSamplesLen <= 0) break;
  636. sample_skip -= mSamplesPos;
  637. // Adjust the device start time and current pts by the amount we're
  638. // skipping/duplicating, so that the clock remains correct for the
  639. // current stream position.
  640. auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
  641. mDeviceStartTime -= skip;
  642. mCurrentPts += skip;
  643. }
  644. }
  645. if(audio_size <= 0)
  646. return false;
  647. if(audio_size < length)
  648. {
  649. const unsigned int rem{length - audio_size};
  650. std::fill_n(samples, rem*mFrameSize,
  651. (mDstSampleFmt == AV_SAMPLE_FMT_U8) ? 0x80 : 0x00);
  652. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  653. }
  654. return true;
  655. }
  656. bool AudioState::readAudio(int sample_skip)
  657. {
  658. size_t woffset{mWritePos.load(std::memory_order_acquire)};
  659. const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
  660. while(mSamplesLen > 0)
  661. {
  662. const size_t nsamples{((roffset > woffset) ? roffset-woffset-1
  663. : (roffset == 0) ? (mBufferDataSize-woffset-1)
  664. : (mBufferDataSize-woffset)) / mFrameSize};
  665. if(!nsamples) break;
  666. if(mSamplesPos < 0)
  667. {
  668. const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(-mSamplesPos))};
  669. sample_dup(&mBufferData[woffset], mSamples, rem, mFrameSize);
  670. woffset += rem * mFrameSize;
  671. if(woffset == mBufferDataSize) woffset = 0;
  672. mWritePos.store(woffset, std::memory_order_release);
  673. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  674. mSamplesPos += static_cast<int>(rem);
  675. continue;
  676. }
  677. const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(mSamplesLen-mSamplesPos))};
  678. const size_t boffset{static_cast<ALuint>(mSamplesPos) * size_t{mFrameSize}};
  679. const size_t nbytes{rem * mFrameSize};
  680. memcpy(&mBufferData[woffset], mSamples + boffset, nbytes);
  681. woffset += nbytes;
  682. if(woffset == mBufferDataSize) woffset = 0;
  683. mWritePos.store(woffset, std::memory_order_release);
  684. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  685. mSamplesPos += static_cast<int>(rem);
  686. while(mSamplesPos >= mSamplesLen)
  687. {
  688. mSamplesLen = decodeFrame();
  689. mSamplesPos = std::min(mSamplesLen, sample_skip);
  690. if(mSamplesLen <= 0) return false;
  691. sample_skip -= mSamplesPos;
  692. auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
  693. mDeviceStartTime -= skip;
  694. mCurrentPts += skip;
  695. }
  696. }
  697. return true;
  698. }
  699. void AL_APIENTRY AudioState::eventCallback(ALenum eventType, ALuint object, ALuint param,
  700. ALsizei length, const ALchar *message)
  701. {
  702. if(eventType == AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT)
  703. {
  704. /* Temporarily lock the source mutex to ensure it's not between
  705. * checking the processed count and going to sleep.
  706. */
  707. std::unique_lock<std::mutex>{mSrcMutex}.unlock();
  708. mSrcCond.notify_one();
  709. return;
  710. }
  711. std::cout<< "\n---- AL Event on AudioState "<<this<<" ----\nEvent: ";
  712. switch(eventType)
  713. {
  714. case AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT: std::cout<< "Buffer completed"; break;
  715. case AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT: std::cout<< "Source state changed"; break;
  716. case AL_EVENT_TYPE_DISCONNECTED_SOFT: std::cout<< "Disconnected"; break;
  717. default:
  718. std::cout<< "0x"<<std::hex<<std::setw(4)<<std::setfill('0')<<eventType<<std::dec<<
  719. std::setw(0)<<std::setfill(' '); break;
  720. }
  721. std::cout<< "\n"
  722. "Object ID: "<<object<<"\n"
  723. "Parameter: "<<param<<"\n"
  724. "Message: "<<std::string{message, static_cast<ALuint>(length)}<<"\n----"<<
  725. std::endl;
  726. if(eventType == AL_EVENT_TYPE_DISCONNECTED_SOFT)
  727. {
  728. {
  729. std::lock_guard<std::mutex> lock{mSrcMutex};
  730. mConnected.clear(std::memory_order_release);
  731. }
  732. mSrcCond.notify_one();
  733. }
  734. }
  735. ALsizei AudioState::bufferCallback(void *data, ALsizei size)
  736. {
  737. ALsizei got{0};
  738. size_t roffset{mReadPos.load(std::memory_order_acquire)};
  739. while(got < size)
  740. {
  741. const size_t woffset{mWritePos.load(std::memory_order_relaxed)};
  742. if(woffset == roffset) break;
  743. size_t todo{((woffset < roffset) ? mBufferDataSize : woffset) - roffset};
  744. todo = std::min<size_t>(todo, static_cast<ALuint>(size-got));
  745. memcpy(data, &mBufferData[roffset], todo);
  746. data = static_cast<ALbyte*>(data) + todo;
  747. got += static_cast<ALsizei>(todo);
  748. roffset += todo;
  749. if(roffset == mBufferDataSize)
  750. roffset = 0;
  751. }
  752. mReadPos.store(roffset, std::memory_order_release);
  753. return got;
  754. }
  755. int AudioState::handler()
  756. {
  757. std::unique_lock<std::mutex> srclock{mSrcMutex, std::defer_lock};
  758. milliseconds sleep_time{AudioBufferTime / 3};
  759. struct EventControlManager {
  760. const std::array<ALenum,3> evt_types{{
  761. AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT, AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT,
  762. AL_EVENT_TYPE_DISCONNECTED_SOFT}};
  763. EventControlManager(milliseconds &sleep_time)
  764. {
  765. if(alEventControlSOFT)
  766. {
  767. alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
  768. AL_TRUE);
  769. alEventCallbackSOFT(&AudioState::eventCallbackC, this);
  770. sleep_time = AudioBufferTotalTime;
  771. }
  772. }
  773. ~EventControlManager()
  774. {
  775. if(alEventControlSOFT)
  776. {
  777. alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
  778. AL_FALSE);
  779. alEventCallbackSOFT(nullptr, nullptr);
  780. }
  781. }
  782. };
  783. EventControlManager event_controller{sleep_time};
  784. std::unique_ptr<uint8_t[]> samples;
  785. ALsizei buffer_len{0};
  786. /* Find a suitable format for OpenAL. */
  787. mDstChanLayout = 0;
  788. mFormat = AL_NONE;
  789. if((mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLT || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLTP
  790. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBL
  791. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBLP
  792. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32
  793. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32P
  794. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64
  795. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64P)
  796. && alIsExtensionPresent("AL_EXT_FLOAT32"))
  797. {
  798. mDstSampleFmt = AV_SAMPLE_FMT_FLT;
  799. mFrameSize = 4;
  800. if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
  801. {
  802. if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
  803. {
  804. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_7POINT1)
  805. {
  806. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  807. mFrameSize *= 8;
  808. mFormat = alGetEnumValue("AL_FORMAT_71CHN32");
  809. }
  810. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1
  811. || mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK)
  812. {
  813. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  814. mFrameSize *= 6;
  815. mFormat = alGetEnumValue("AL_FORMAT_51CHN32");
  816. }
  817. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_QUAD)
  818. {
  819. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  820. mFrameSize *= 4;
  821. mFormat = alGetEnumValue("AL_FORMAT_QUAD32");
  822. }
  823. }
  824. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_MONO)
  825. {
  826. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  827. mFrameSize *= 1;
  828. mFormat = AL_FORMAT_MONO_FLOAT32;
  829. }
  830. }
  831. else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
  832. && alIsExtensionPresent("AL_EXT_BFORMAT"))
  833. {
  834. /* Calculate what should be the ambisonic order from the number of
  835. * channels, and confirm that's the number of channels. Opus allows
  836. * an optional non-diegetic stereo stream with the B-Format stream,
  837. * which we can ignore, so check for that too.
  838. */
  839. auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
  840. int channels{(order+1) * (order+1)};
  841. if(channels == mCodecCtx->ch_layout.nb_channels
  842. || channels+2 == mCodecCtx->ch_layout.nb_channels)
  843. {
  844. /* OpenAL only supports first-order with AL_EXT_BFORMAT, which
  845. * is 4 channels for 3D buffers.
  846. */
  847. mFrameSize *= 4;
  848. mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_FLOAT32");
  849. }
  850. }
  851. if(!mFormat || mFormat == -1)
  852. {
  853. mDstChanLayout = AV_CH_LAYOUT_STEREO;
  854. mFrameSize *= 2;
  855. mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN_FLOAT32_SOFT : AL_FORMAT_STEREO_FLOAT32;
  856. }
  857. }
  858. if(mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8P)
  859. {
  860. mDstSampleFmt = AV_SAMPLE_FMT_U8;
  861. mFrameSize = 1;
  862. if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
  863. {
  864. if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
  865. {
  866. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_7POINT1)
  867. {
  868. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  869. mFrameSize *= 8;
  870. mFormat = alGetEnumValue("AL_FORMAT_71CHN8");
  871. }
  872. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1
  873. || mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK)
  874. {
  875. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  876. mFrameSize *= 6;
  877. mFormat = alGetEnumValue("AL_FORMAT_51CHN8");
  878. }
  879. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_QUAD)
  880. {
  881. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  882. mFrameSize *= 4;
  883. mFormat = alGetEnumValue("AL_FORMAT_QUAD8");
  884. }
  885. }
  886. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_MONO)
  887. {
  888. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  889. mFrameSize *= 1;
  890. mFormat = AL_FORMAT_MONO8;
  891. }
  892. }
  893. else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
  894. && alIsExtensionPresent("AL_EXT_BFORMAT"))
  895. {
  896. auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
  897. int channels{(order+1) * (order+1)};
  898. if(channels == mCodecCtx->ch_layout.nb_channels
  899. || channels+2 == mCodecCtx->ch_layout.nb_channels)
  900. {
  901. mFrameSize *= 4;
  902. mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_8");
  903. }
  904. }
  905. if(!mFormat || mFormat == -1)
  906. {
  907. mDstChanLayout = AV_CH_LAYOUT_STEREO;
  908. mFrameSize *= 2;
  909. mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN8_SOFT : AL_FORMAT_STEREO8;
  910. }
  911. }
  912. if(!mFormat || mFormat == -1)
  913. {
  914. mDstSampleFmt = AV_SAMPLE_FMT_S16;
  915. mFrameSize = 2;
  916. if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
  917. {
  918. if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
  919. {
  920. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_7POINT1)
  921. {
  922. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  923. mFrameSize *= 8;
  924. mFormat = alGetEnumValue("AL_FORMAT_71CHN16");
  925. }
  926. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1
  927. || mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_5POINT1_BACK)
  928. {
  929. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  930. mFrameSize *= 6;
  931. mFormat = alGetEnumValue("AL_FORMAT_51CHN16");
  932. }
  933. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_QUAD)
  934. {
  935. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  936. mFrameSize *= 4;
  937. mFormat = alGetEnumValue("AL_FORMAT_QUAD16");
  938. }
  939. }
  940. if(mCodecCtx->ch_layout.u.mask == AV_CH_LAYOUT_MONO)
  941. {
  942. mDstChanLayout = mCodecCtx->ch_layout.u.mask;
  943. mFrameSize *= 1;
  944. mFormat = AL_FORMAT_MONO16;
  945. }
  946. }
  947. else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
  948. && alIsExtensionPresent("AL_EXT_BFORMAT"))
  949. {
  950. auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
  951. int channels{(order+1) * (order+1)};
  952. if(channels == mCodecCtx->ch_layout.nb_channels
  953. || channels+2 == mCodecCtx->ch_layout.nb_channels)
  954. {
  955. mFrameSize *= 4;
  956. mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_16");
  957. }
  958. }
  959. if(!mFormat || mFormat == -1)
  960. {
  961. mDstChanLayout = AV_CH_LAYOUT_STEREO;
  962. mFrameSize *= 2;
  963. mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN16_SOFT : AL_FORMAT_STEREO16;
  964. }
  965. }
  966. mSamples = nullptr;
  967. mSamplesMax = 0;
  968. mSamplesPos = 0;
  969. mSamplesLen = 0;
  970. mDecodedFrame.reset(av_frame_alloc());
  971. if(!mDecodedFrame)
  972. {
  973. std::cerr<< "Failed to allocate audio frame" <<std::endl;
  974. return 0;
  975. }
  976. /* Note that ffmpeg assumes AmbiX (ACN layout, SN3D normalization). */
  977. const bool has_bfmt_ex{alIsExtensionPresent("AL_SOFT_bformat_ex") != AL_FALSE};
  978. const ALenum ambi_layout{AL_ACN_SOFT};
  979. const ALenum ambi_scale{AL_SN3D_SOFT};
  980. if(!mDstChanLayout)
  981. {
  982. /* OpenAL only supports first-order ambisonics with AL_EXT_BFORMAT, so
  983. * we have to drop any extra channels.
  984. */
  985. ChannelLayout layout{};
  986. av_channel_layout_from_string(&layout, "ambisonic 1");
  987. SwrContext *ps{};
  988. int err{swr_alloc_set_opts2(&ps, &layout, mDstSampleFmt, mCodecCtx->sample_rate,
  989. &mCodecCtx->ch_layout, mCodecCtx->sample_fmt, mCodecCtx->sample_rate, 0, nullptr)};
  990. mSwresCtx.reset(ps);
  991. if(err != 0)
  992. {
  993. char errstr[AV_ERROR_MAX_STRING_SIZE]{};
  994. std::cerr<< "Failed to allocate SwrContext: "
  995. <<av_make_error_string(errstr, AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
  996. return 0;
  997. }
  998. if(has_bfmt_ex)
  999. std::cout<< "Found AL_SOFT_bformat_ex" <<std::endl;
  1000. else
  1001. {
  1002. std::cout<< "Found AL_EXT_BFORMAT" <<std::endl;
  1003. /* Without AL_SOFT_bformat_ex, OpenAL only supports FuMa channel
  1004. * ordering and normalization, so a custom matrix is needed to
  1005. * scale and reorder the source from AmbiX.
  1006. */
  1007. std::vector<double> mtx(64*64, 0.0);
  1008. mtx[0 + 0*64] = std::sqrt(0.5);
  1009. mtx[3 + 1*64] = 1.0;
  1010. mtx[1 + 2*64] = 1.0;
  1011. mtx[2 + 3*64] = 1.0;
  1012. swr_set_matrix(mSwresCtx.get(), mtx.data(), 64);
  1013. }
  1014. }
  1015. else
  1016. {
  1017. ChannelLayout layout{};
  1018. av_channel_layout_from_mask(&layout, mDstChanLayout);
  1019. SwrContext *ps{};
  1020. int err{swr_alloc_set_opts2(&ps, &layout, mDstSampleFmt, mCodecCtx->sample_rate,
  1021. &mCodecCtx->ch_layout, mCodecCtx->sample_fmt, mCodecCtx->sample_rate, 0, nullptr)};
  1022. mSwresCtx.reset(ps);
  1023. if(err != 0)
  1024. {
  1025. char errstr[AV_ERROR_MAX_STRING_SIZE]{};
  1026. std::cerr<< "Failed to allocate SwrContext: "
  1027. <<av_make_error_string(errstr, AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
  1028. return 0;
  1029. }
  1030. }
  1031. if(int err{swr_init(mSwresCtx.get())})
  1032. {
  1033. char errstr[AV_ERROR_MAX_STRING_SIZE]{};
  1034. std::cerr<< "Failed to initialize audio converter: "
  1035. <<av_make_error_string(errstr, AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
  1036. return 0;
  1037. }
  1038. alGenBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
  1039. alGenSources(1, &mSource);
  1040. if(DirectOutMode)
  1041. alSourcei(mSource, AL_DIRECT_CHANNELS_SOFT, DirectOutMode);
  1042. if(EnableWideStereo)
  1043. {
  1044. const float angles[2]{static_cast<float>(M_PI / 3.0), static_cast<float>(-M_PI / 3.0)};
  1045. alSourcefv(mSource, AL_STEREO_ANGLES, angles);
  1046. }
  1047. if(has_bfmt_ex)
  1048. {
  1049. for(ALuint bufid : mBuffers)
  1050. {
  1051. alBufferi(bufid, AL_AMBISONIC_LAYOUT_SOFT, ambi_layout);
  1052. alBufferi(bufid, AL_AMBISONIC_SCALING_SOFT, ambi_scale);
  1053. }
  1054. }
  1055. #ifdef AL_SOFT_UHJ
  1056. if(EnableSuperStereo)
  1057. alSourcei(mSource, AL_STEREO_MODE_SOFT, AL_SUPER_STEREO_SOFT);
  1058. #endif
  1059. if(alGetError() != AL_NO_ERROR)
  1060. return 0;
  1061. bool callback_ok{false};
  1062. if(alBufferCallbackSOFT)
  1063. {
  1064. alBufferCallbackSOFT(mBuffers[0], mFormat, mCodecCtx->sample_rate, bufferCallbackC, this);
  1065. alSourcei(mSource, AL_BUFFER, static_cast<ALint>(mBuffers[0]));
  1066. if(alGetError() != AL_NO_ERROR)
  1067. {
  1068. fprintf(stderr, "Failed to set buffer callback\n");
  1069. alSourcei(mSource, AL_BUFFER, 0);
  1070. }
  1071. else
  1072. {
  1073. mBufferDataSize = static_cast<size_t>(duration_cast<seconds>(mCodecCtx->sample_rate *
  1074. AudioBufferTotalTime).count()) * mFrameSize;
  1075. mBufferData = std::make_unique<uint8_t[]>(mBufferDataSize);
  1076. std::fill_n(mBufferData.get(), mBufferDataSize, uint8_t{});
  1077. mReadPos.store(0, std::memory_order_relaxed);
  1078. mWritePos.store(mBufferDataSize/mFrameSize/2*mFrameSize, std::memory_order_relaxed);
  1079. ALCint refresh{};
  1080. alcGetIntegerv(alcGetContextsDevice(alcGetCurrentContext()), ALC_REFRESH, 1, &refresh);
  1081. sleep_time = milliseconds{seconds{1}} / refresh;
  1082. callback_ok = true;
  1083. }
  1084. }
  1085. if(!callback_ok)
  1086. buffer_len = static_cast<int>(duration_cast<seconds>(mCodecCtx->sample_rate *
  1087. AudioBufferTime).count() * mFrameSize);
  1088. if(buffer_len > 0)
  1089. samples = std::make_unique<uint8_t[]>(static_cast<ALuint>(buffer_len));
  1090. /* Prefill the codec buffer. */
  1091. auto packet_sender = [this]()
  1092. {
  1093. while(1)
  1094. {
  1095. const int ret{mQueue.sendPacket(mCodecCtx.get())};
  1096. if(ret == AVErrorEOF) break;
  1097. }
  1098. };
  1099. auto sender = std::async(std::launch::async, packet_sender);
  1100. srclock.lock();
  1101. if(alcGetInteger64vSOFT)
  1102. {
  1103. int64_t devtime{};
  1104. alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()), ALC_DEVICE_CLOCK_SOFT,
  1105. 1, &devtime);
  1106. mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
  1107. }
  1108. mSamplesLen = decodeFrame();
  1109. if(mSamplesLen > 0)
  1110. {
  1111. mSamplesPos = std::min(mSamplesLen, getSync());
  1112. auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
  1113. mDeviceStartTime -= skip;
  1114. mCurrentPts += skip;
  1115. }
  1116. while(1)
  1117. {
  1118. if(mMovie.mQuit.load(std::memory_order_relaxed))
  1119. {
  1120. /* If mQuit is set, drain frames until we can't get more audio,
  1121. * indicating we've reached the flush packet and the packet sender
  1122. * will also quit.
  1123. */
  1124. do {
  1125. mSamplesLen = decodeFrame();
  1126. mSamplesPos = mSamplesLen;
  1127. } while(mSamplesLen > 0);
  1128. goto finish;
  1129. }
  1130. ALenum state;
  1131. if(mBufferDataSize > 0)
  1132. {
  1133. alGetSourcei(mSource, AL_SOURCE_STATE, &state);
  1134. /* If mQuit is not set, don't quit even if there's no more audio,
  1135. * so what's buffered has a chance to play to the real end.
  1136. */
  1137. readAudio(getSync());
  1138. }
  1139. else
  1140. {
  1141. ALint processed, queued;
  1142. /* First remove any processed buffers. */
  1143. alGetSourcei(mSource, AL_BUFFERS_PROCESSED, &processed);
  1144. while(processed > 0)
  1145. {
  1146. ALuint bid;
  1147. alSourceUnqueueBuffers(mSource, 1, &bid);
  1148. --processed;
  1149. }
  1150. /* Refill the buffer queue. */
  1151. int sync_skip{getSync()};
  1152. alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
  1153. while(static_cast<ALuint>(queued) < mBuffers.size())
  1154. {
  1155. /* Read the next chunk of data, filling the buffer, and queue
  1156. * it on the source.
  1157. */
  1158. if(!readAudio(samples.get(), static_cast<ALuint>(buffer_len), sync_skip))
  1159. break;
  1160. const ALuint bufid{mBuffers[mBufferIdx]};
  1161. mBufferIdx = static_cast<ALuint>((mBufferIdx+1) % mBuffers.size());
  1162. alBufferData(bufid, mFormat, samples.get(), buffer_len, mCodecCtx->sample_rate);
  1163. alSourceQueueBuffers(mSource, 1, &bufid);
  1164. ++queued;
  1165. }
  1166. /* Check that the source is playing. */
  1167. alGetSourcei(mSource, AL_SOURCE_STATE, &state);
  1168. if(state == AL_STOPPED)
  1169. {
  1170. /* AL_STOPPED means there was an underrun. Clear the buffer
  1171. * queue since this likely means we're late, and rewind the
  1172. * source to get it back into an AL_INITIAL state.
  1173. */
  1174. alSourceRewind(mSource);
  1175. alSourcei(mSource, AL_BUFFER, 0);
  1176. if(alcGetInteger64vSOFT)
  1177. {
  1178. /* Also update the device start time with the current
  1179. * device clock, so the decoder knows we're running behind.
  1180. */
  1181. int64_t devtime{};
  1182. alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()),
  1183. ALC_DEVICE_CLOCK_SOFT, 1, &devtime);
  1184. mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
  1185. }
  1186. continue;
  1187. }
  1188. }
  1189. /* (re)start the source if needed, and wait for a buffer to finish */
  1190. if(state != AL_PLAYING && state != AL_PAUSED)
  1191. {
  1192. if(!startPlayback())
  1193. break;
  1194. }
  1195. if(ALenum err{alGetError()})
  1196. std::cerr<< "Got AL error: 0x"<<std::hex<<err<<std::dec
  1197. << " ("<<alGetString(err)<<")" <<std::endl;
  1198. mSrcCond.wait_for(srclock, sleep_time);
  1199. }
  1200. finish:
  1201. alSourceRewind(mSource);
  1202. alSourcei(mSource, AL_BUFFER, 0);
  1203. srclock.unlock();
  1204. return 0;
  1205. }
  1206. nanoseconds VideoState::getClock()
  1207. {
  1208. /* NOTE: This returns incorrect times while not playing. */
  1209. std::lock_guard<std::mutex> _{mDispPtsMutex};
  1210. if(mDisplayPtsTime == microseconds::min())
  1211. return nanoseconds::zero();
  1212. auto delta = get_avtime() - mDisplayPtsTime;
  1213. return mDisplayPts + delta;
  1214. }
  1215. /* Called by VideoState::updateVideo to display the next video frame. */
  1216. void VideoState::display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame)
  1217. {
  1218. if(!mImage)
  1219. return;
  1220. double aspect_ratio;
  1221. int win_w, win_h;
  1222. int w, h, x, y;
  1223. int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
  1224. int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
  1225. if(frame->sample_aspect_ratio.num == 0)
  1226. aspect_ratio = 0.0;
  1227. else
  1228. {
  1229. aspect_ratio = av_q2d(frame->sample_aspect_ratio) * frame_width /
  1230. frame_height;
  1231. }
  1232. if(aspect_ratio <= 0.0)
  1233. aspect_ratio = static_cast<double>(frame_width) / frame_height;
  1234. SDL_GetWindowSize(screen, &win_w, &win_h);
  1235. h = win_h;
  1236. w = (static_cast<int>(std::rint(h * aspect_ratio)) + 3) & ~3;
  1237. if(w > win_w)
  1238. {
  1239. w = win_w;
  1240. h = (static_cast<int>(std::rint(w / aspect_ratio)) + 3) & ~3;
  1241. }
  1242. x = (win_w - w) / 2;
  1243. y = (win_h - h) / 2;
  1244. SDL_Rect src_rect{ static_cast<int>(frame->crop_left), static_cast<int>(frame->crop_top),
  1245. frame_width, frame_height };
  1246. SDL_Rect dst_rect{ x, y, w, h };
  1247. SDL_RenderCopy(renderer, mImage, &src_rect, &dst_rect);
  1248. SDL_RenderPresent(renderer);
  1249. }
  1250. /* Called regularly on the main thread where the SDL_Renderer was created. It
  1251. * handles updating the textures of decoded frames and displaying the latest
  1252. * frame.
  1253. */
  1254. void VideoState::updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw)
  1255. {
  1256. size_t read_idx{mPictQRead.load(std::memory_order_relaxed)};
  1257. Picture *vp{&mPictQ[read_idx]};
  1258. auto clocktime = mMovie.getMasterClock();
  1259. bool updated{false};
  1260. while(1)
  1261. {
  1262. size_t next_idx{(read_idx+1)%mPictQ.size()};
  1263. if(next_idx == mPictQWrite.load(std::memory_order_acquire))
  1264. break;
  1265. Picture *nextvp{&mPictQ[next_idx]};
  1266. if(clocktime < nextvp->mPts && !mMovie.mQuit.load(std::memory_order_relaxed))
  1267. {
  1268. /* For the first update, ensure the first frame gets shown. */
  1269. if(!mFirstUpdate || updated)
  1270. break;
  1271. }
  1272. vp = nextvp;
  1273. updated = true;
  1274. read_idx = next_idx;
  1275. }
  1276. if(mMovie.mQuit.load(std::memory_order_relaxed))
  1277. {
  1278. if(mEOS)
  1279. mFinalUpdate = true;
  1280. mPictQRead.store(read_idx, std::memory_order_release);
  1281. std::unique_lock<std::mutex>{mPictQMutex}.unlock();
  1282. mPictQCond.notify_one();
  1283. return;
  1284. }
  1285. AVFrame *frame{vp->mFrame.get()};
  1286. if(updated)
  1287. {
  1288. mPictQRead.store(read_idx, std::memory_order_release);
  1289. std::unique_lock<std::mutex>{mPictQMutex}.unlock();
  1290. mPictQCond.notify_one();
  1291. /* allocate or resize the buffer! */
  1292. bool fmt_updated{false};
  1293. if(!mImage || mWidth != frame->width || mHeight != frame->height)
  1294. {
  1295. fmt_updated = true;
  1296. if(mImage)
  1297. SDL_DestroyTexture(mImage);
  1298. mImage = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING,
  1299. frame->width, frame->height);
  1300. if(!mImage)
  1301. std::cerr<< "Failed to create YV12 texture!" <<std::endl;
  1302. mWidth = frame->width;
  1303. mHeight = frame->height;
  1304. }
  1305. int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
  1306. int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
  1307. if(mFirstUpdate && frame_width > 0 && frame_height > 0)
  1308. {
  1309. /* For the first update, set the window size to the video size. */
  1310. mFirstUpdate = false;
  1311. if(frame->sample_aspect_ratio.den != 0)
  1312. {
  1313. double aspect_ratio = av_q2d(frame->sample_aspect_ratio);
  1314. if(aspect_ratio >= 1.0)
  1315. frame_width = static_cast<int>(frame_width*aspect_ratio + 0.5);
  1316. else if(aspect_ratio > 0.0)
  1317. frame_height = static_cast<int>(frame_height/aspect_ratio + 0.5);
  1318. }
  1319. SDL_SetWindowSize(screen, frame_width, frame_height);
  1320. }
  1321. if(mImage)
  1322. {
  1323. void *pixels{nullptr};
  1324. int pitch{0};
  1325. if(mCodecCtx->pix_fmt == AV_PIX_FMT_YUV420P)
  1326. SDL_UpdateYUVTexture(mImage, nullptr,
  1327. frame->data[0], frame->linesize[0],
  1328. frame->data[1], frame->linesize[1],
  1329. frame->data[2], frame->linesize[2]
  1330. );
  1331. else if(SDL_LockTexture(mImage, nullptr, &pixels, &pitch) != 0)
  1332. std::cerr<< "Failed to lock texture" <<std::endl;
  1333. else
  1334. {
  1335. // Convert the image into YUV format that SDL uses
  1336. int w{frame->width};
  1337. int h{frame->height};
  1338. if(!mSwscaleCtx || fmt_updated)
  1339. {
  1340. mSwscaleCtx.reset(sws_getContext(
  1341. w, h, mCodecCtx->pix_fmt,
  1342. w, h, AV_PIX_FMT_YUV420P, 0,
  1343. nullptr, nullptr, nullptr
  1344. ));
  1345. }
  1346. /* point pict at the queue */
  1347. uint8_t *pict_data[3];
  1348. pict_data[0] = static_cast<uint8_t*>(pixels);
  1349. pict_data[1] = pict_data[0] + w*h;
  1350. pict_data[2] = pict_data[1] + w*h/4;
  1351. int pict_linesize[3];
  1352. pict_linesize[0] = pitch;
  1353. pict_linesize[1] = pitch / 2;
  1354. pict_linesize[2] = pitch / 2;
  1355. sws_scale(mSwscaleCtx.get(), reinterpret_cast<uint8_t**>(frame->data), frame->linesize,
  1356. 0, h, pict_data, pict_linesize);
  1357. SDL_UnlockTexture(mImage);
  1358. }
  1359. redraw = true;
  1360. }
  1361. }
  1362. if(redraw)
  1363. {
  1364. /* Show the picture! */
  1365. display(screen, renderer, frame);
  1366. }
  1367. if(updated)
  1368. {
  1369. auto disp_time = get_avtime();
  1370. std::lock_guard<std::mutex> _{mDispPtsMutex};
  1371. mDisplayPts = vp->mPts;
  1372. mDisplayPtsTime = disp_time;
  1373. }
  1374. if(mEOS.load(std::memory_order_acquire))
  1375. {
  1376. if((read_idx+1)%mPictQ.size() == mPictQWrite.load(std::memory_order_acquire))
  1377. {
  1378. mFinalUpdate = true;
  1379. std::unique_lock<std::mutex>{mPictQMutex}.unlock();
  1380. mPictQCond.notify_one();
  1381. }
  1382. }
  1383. }
  1384. int VideoState::handler()
  1385. {
  1386. std::for_each(mPictQ.begin(), mPictQ.end(),
  1387. [](Picture &pict) -> void
  1388. { pict.mFrame = AVFramePtr{av_frame_alloc()}; });
  1389. /* Prefill the codec buffer. */
  1390. auto packet_sender = [this]()
  1391. {
  1392. while(1)
  1393. {
  1394. const int ret{mQueue.sendPacket(mCodecCtx.get())};
  1395. if(ret == AVErrorEOF) break;
  1396. }
  1397. };
  1398. auto sender = std::async(std::launch::async, packet_sender);
  1399. {
  1400. std::lock_guard<std::mutex> _{mDispPtsMutex};
  1401. mDisplayPtsTime = get_avtime();
  1402. }
  1403. auto current_pts = nanoseconds::zero();
  1404. while(1)
  1405. {
  1406. size_t write_idx{mPictQWrite.load(std::memory_order_relaxed)};
  1407. Picture *vp{&mPictQ[write_idx]};
  1408. /* Retrieve video frame. */
  1409. AVFrame *decoded_frame{vp->mFrame.get()};
  1410. while(int ret{mQueue.receiveFrame(mCodecCtx.get(), decoded_frame)})
  1411. {
  1412. if(ret == AVErrorEOF) goto finish;
  1413. std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
  1414. }
  1415. /* Get the PTS for this frame. */
  1416. if(decoded_frame->best_effort_timestamp != AVNoPtsValue)
  1417. current_pts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
  1418. static_cast<double>(decoded_frame->best_effort_timestamp)});
  1419. vp->mPts = current_pts;
  1420. /* Update the video clock to the next expected PTS. */
  1421. auto frame_delay = av_q2d(mCodecCtx->time_base);
  1422. frame_delay += decoded_frame->repeat_pict * (frame_delay * 0.5);
  1423. current_pts += duration_cast<nanoseconds>(seconds_d64{frame_delay});
  1424. /* Put the frame in the queue to be loaded into a texture and displayed
  1425. * by the rendering thread.
  1426. */
  1427. write_idx = (write_idx+1)%mPictQ.size();
  1428. mPictQWrite.store(write_idx, std::memory_order_release);
  1429. if(write_idx == mPictQRead.load(std::memory_order_acquire))
  1430. {
  1431. /* Wait until we have space for a new pic */
  1432. std::unique_lock<std::mutex> lock{mPictQMutex};
  1433. while(write_idx == mPictQRead.load(std::memory_order_acquire))
  1434. mPictQCond.wait(lock);
  1435. }
  1436. }
  1437. finish:
  1438. mEOS = true;
  1439. std::unique_lock<std::mutex> lock{mPictQMutex};
  1440. while(!mFinalUpdate) mPictQCond.wait(lock);
  1441. return 0;
  1442. }
  1443. int MovieState::decode_interrupt_cb(void *ctx)
  1444. {
  1445. return static_cast<MovieState*>(ctx)->mQuit.load(std::memory_order_relaxed);
  1446. }
  1447. bool MovieState::prepare()
  1448. {
  1449. AVIOContext *avioctx{nullptr};
  1450. AVIOInterruptCB intcb{decode_interrupt_cb, this};
  1451. if(avio_open2(&avioctx, mFilename.c_str(), AVIO_FLAG_READ, &intcb, nullptr))
  1452. {
  1453. std::cerr<< "Failed to open "<<mFilename <<std::endl;
  1454. return false;
  1455. }
  1456. mIOContext.reset(avioctx);
  1457. /* Open movie file. If avformat_open_input fails it will automatically free
  1458. * this context, so don't set it onto a smart pointer yet.
  1459. */
  1460. AVFormatContext *fmtctx{avformat_alloc_context()};
  1461. fmtctx->pb = mIOContext.get();
  1462. fmtctx->interrupt_callback = intcb;
  1463. if(avformat_open_input(&fmtctx, mFilename.c_str(), nullptr, nullptr) != 0)
  1464. {
  1465. std::cerr<< "Failed to open "<<mFilename <<std::endl;
  1466. return false;
  1467. }
  1468. mFormatCtx.reset(fmtctx);
  1469. /* Retrieve stream information */
  1470. if(avformat_find_stream_info(mFormatCtx.get(), nullptr) < 0)
  1471. {
  1472. std::cerr<< mFilename<<": failed to find stream info" <<std::endl;
  1473. return false;
  1474. }
  1475. /* Dump information about file onto standard error */
  1476. av_dump_format(mFormatCtx.get(), 0, mFilename.c_str(), 0);
  1477. mParseThread = std::thread{std::mem_fn(&MovieState::parse_handler), this};
  1478. std::unique_lock<std::mutex> slock{mStartupMutex};
  1479. while(!mStartupDone) mStartupCond.wait(slock);
  1480. return true;
  1481. }
  1482. void MovieState::setTitle(SDL_Window *window)
  1483. {
  1484. auto pos1 = mFilename.rfind('/');
  1485. auto pos2 = mFilename.rfind('\\');
  1486. auto fpos = ((pos1 == std::string::npos) ? pos2 :
  1487. (pos2 == std::string::npos) ? pos1 :
  1488. std::max(pos1, pos2)) + 1;
  1489. SDL_SetWindowTitle(window, (mFilename.substr(fpos)+" - "+AppName).c_str());
  1490. }
  1491. nanoseconds MovieState::getClock()
  1492. {
  1493. if(mClockBase == microseconds::min())
  1494. return nanoseconds::zero();
  1495. return get_avtime() - mClockBase;
  1496. }
  1497. nanoseconds MovieState::getMasterClock()
  1498. {
  1499. if(mAVSyncType == SyncMaster::Video && mVideo.mStream)
  1500. return mVideo.getClock();
  1501. if(mAVSyncType == SyncMaster::Audio && mAudio.mStream)
  1502. return mAudio.getClock();
  1503. return getClock();
  1504. }
  1505. nanoseconds MovieState::getDuration()
  1506. { return std::chrono::duration<int64_t,std::ratio<1,AV_TIME_BASE>>(mFormatCtx->duration); }
  1507. int MovieState::streamComponentOpen(unsigned int stream_index)
  1508. {
  1509. if(stream_index >= mFormatCtx->nb_streams)
  1510. return -1;
  1511. /* Get a pointer to the codec context for the stream, and open the
  1512. * associated codec.
  1513. */
  1514. AVCodecCtxPtr avctx{avcodec_alloc_context3(nullptr)};
  1515. if(!avctx) return -1;
  1516. if(avcodec_parameters_to_context(avctx.get(), mFormatCtx->streams[stream_index]->codecpar))
  1517. return -1;
  1518. const AVCodec *codec{avcodec_find_decoder(avctx->codec_id)};
  1519. if(!codec || avcodec_open2(avctx.get(), codec, nullptr) < 0)
  1520. {
  1521. std::cerr<< "Unsupported codec: "<<avcodec_get_name(avctx->codec_id)
  1522. << " (0x"<<std::hex<<avctx->codec_id<<std::dec<<")" <<std::endl;
  1523. return -1;
  1524. }
  1525. /* Initialize and start the media type handler */
  1526. switch(avctx->codec_type)
  1527. {
  1528. case AVMEDIA_TYPE_AUDIO:
  1529. mAudio.mStream = mFormatCtx->streams[stream_index];
  1530. mAudio.mCodecCtx = std::move(avctx);
  1531. break;
  1532. case AVMEDIA_TYPE_VIDEO:
  1533. mVideo.mStream = mFormatCtx->streams[stream_index];
  1534. mVideo.mCodecCtx = std::move(avctx);
  1535. break;
  1536. default:
  1537. return -1;
  1538. }
  1539. return static_cast<int>(stream_index);
  1540. }
  1541. int MovieState::parse_handler()
  1542. {
  1543. auto &audio_queue = mAudio.mQueue;
  1544. auto &video_queue = mVideo.mQueue;
  1545. int video_index{-1};
  1546. int audio_index{-1};
  1547. /* Find the first video and audio streams */
  1548. for(unsigned int i{0u};i < mFormatCtx->nb_streams;i++)
  1549. {
  1550. auto codecpar = mFormatCtx->streams[i]->codecpar;
  1551. if(codecpar->codec_type == AVMEDIA_TYPE_VIDEO && !DisableVideo && video_index < 0)
  1552. video_index = streamComponentOpen(i);
  1553. else if(codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_index < 0)
  1554. audio_index = streamComponentOpen(i);
  1555. }
  1556. {
  1557. std::unique_lock<std::mutex> slock{mStartupMutex};
  1558. mStartupDone = true;
  1559. }
  1560. mStartupCond.notify_all();
  1561. if(video_index < 0 && audio_index < 0)
  1562. {
  1563. std::cerr<< mFilename<<": could not open codecs" <<std::endl;
  1564. mQuit = true;
  1565. }
  1566. /* Set the base time 750ms ahead of the current av time. */
  1567. mClockBase = get_avtime() + milliseconds{750};
  1568. if(audio_index >= 0)
  1569. mAudioThread = std::thread{std::mem_fn(&AudioState::handler), &mAudio};
  1570. if(video_index >= 0)
  1571. mVideoThread = std::thread{std::mem_fn(&VideoState::handler), &mVideo};
  1572. /* Main packet reading/dispatching loop */
  1573. AVPacketPtr packet{av_packet_alloc()};
  1574. while(!mQuit.load(std::memory_order_relaxed))
  1575. {
  1576. if(av_read_frame(mFormatCtx.get(), packet.get()) < 0)
  1577. break;
  1578. /* Copy the packet into the queue it's meant for. */
  1579. if(packet->stream_index == video_index)
  1580. {
  1581. while(!mQuit.load(std::memory_order_acquire) && !video_queue.put(packet.get()))
  1582. std::this_thread::sleep_for(milliseconds{100});
  1583. }
  1584. else if(packet->stream_index == audio_index)
  1585. {
  1586. while(!mQuit.load(std::memory_order_acquire) && !audio_queue.put(packet.get()))
  1587. std::this_thread::sleep_for(milliseconds{100});
  1588. }
  1589. av_packet_unref(packet.get());
  1590. }
  1591. /* Finish the queues so the receivers know nothing more is coming. */
  1592. video_queue.setFinished();
  1593. audio_queue.setFinished();
  1594. /* all done - wait for it */
  1595. if(mVideoThread.joinable())
  1596. mVideoThread.join();
  1597. if(mAudioThread.joinable())
  1598. mAudioThread.join();
  1599. mVideo.mEOS = true;
  1600. std::unique_lock<std::mutex> lock{mVideo.mPictQMutex};
  1601. while(!mVideo.mFinalUpdate)
  1602. mVideo.mPictQCond.wait(lock);
  1603. lock.unlock();
  1604. SDL_Event evt{};
  1605. evt.user.type = FF_MOVIE_DONE_EVENT;
  1606. SDL_PushEvent(&evt);
  1607. return 0;
  1608. }
  1609. void MovieState::stop()
  1610. {
  1611. mQuit = true;
  1612. mAudio.mQueue.flush();
  1613. mVideo.mQueue.flush();
  1614. }
  1615. // Helper class+method to print the time with human-readable formatting.
  1616. struct PrettyTime {
  1617. seconds mTime;
  1618. };
  1619. std::ostream &operator<<(std::ostream &os, const PrettyTime &rhs)
  1620. {
  1621. using hours = std::chrono::hours;
  1622. using minutes = std::chrono::minutes;
  1623. seconds t{rhs.mTime};
  1624. if(t.count() < 0)
  1625. {
  1626. os << '-';
  1627. t *= -1;
  1628. }
  1629. // Only handle up to hour formatting
  1630. if(t >= hours{1})
  1631. os << duration_cast<hours>(t).count() << 'h' << std::setfill('0') << std::setw(2)
  1632. << (duration_cast<minutes>(t).count() % 60) << 'm';
  1633. else
  1634. os << duration_cast<minutes>(t).count() << 'm' << std::setfill('0');
  1635. os << std::setw(2) << (duration_cast<seconds>(t).count() % 60) << 's' << std::setw(0)
  1636. << std::setfill(' ');
  1637. return os;
  1638. }
  1639. } // namespace
  1640. int main(int argc, char *argv[])
  1641. {
  1642. SDL_SetMainReady();
  1643. std::unique_ptr<MovieState> movState;
  1644. if(argc < 2)
  1645. {
  1646. std::cerr<< "Usage: "<<argv[0]<<" [-device <device name>] [-direct] <files...>" <<std::endl;
  1647. return 1;
  1648. }
  1649. /* Register all formats and codecs */
  1650. #if !(LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 9, 100))
  1651. av_register_all();
  1652. #endif
  1653. /* Initialize networking protocols */
  1654. avformat_network_init();
  1655. if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS))
  1656. {
  1657. std::cerr<< "Could not initialize SDL - <<"<<SDL_GetError() <<std::endl;
  1658. return 1;
  1659. }
  1660. /* Make a window to put our video */
  1661. SDL_Window *screen{SDL_CreateWindow(AppName.c_str(), 0, 0, 640, 480, SDL_WINDOW_RESIZABLE)};
  1662. if(!screen)
  1663. {
  1664. std::cerr<< "SDL: could not set video mode - exiting" <<std::endl;
  1665. return 1;
  1666. }
  1667. /* Make a renderer to handle the texture image surface and rendering. */
  1668. Uint32 render_flags{SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC};
  1669. SDL_Renderer *renderer{SDL_CreateRenderer(screen, -1, render_flags)};
  1670. if(renderer)
  1671. {
  1672. SDL_RendererInfo rinf{};
  1673. bool ok{false};
  1674. /* Make sure the renderer supports IYUV textures. If not, fallback to a
  1675. * software renderer. */
  1676. if(SDL_GetRendererInfo(renderer, &rinf) == 0)
  1677. {
  1678. for(Uint32 i{0u};!ok && i < rinf.num_texture_formats;i++)
  1679. ok = (rinf.texture_formats[i] == SDL_PIXELFORMAT_IYUV);
  1680. }
  1681. if(!ok)
  1682. {
  1683. std::cerr<< "IYUV pixelformat textures not supported on renderer "<<rinf.name <<std::endl;
  1684. SDL_DestroyRenderer(renderer);
  1685. renderer = nullptr;
  1686. }
  1687. }
  1688. if(!renderer)
  1689. {
  1690. render_flags = SDL_RENDERER_SOFTWARE | SDL_RENDERER_PRESENTVSYNC;
  1691. renderer = SDL_CreateRenderer(screen, -1, render_flags);
  1692. }
  1693. if(!renderer)
  1694. {
  1695. std::cerr<< "SDL: could not create renderer - exiting" <<std::endl;
  1696. return 1;
  1697. }
  1698. SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
  1699. SDL_RenderFillRect(renderer, nullptr);
  1700. SDL_RenderPresent(renderer);
  1701. /* Open an audio device */
  1702. ++argv; --argc;
  1703. if(InitAL(&argv, &argc))
  1704. {
  1705. std::cerr<< "Failed to set up audio device" <<std::endl;
  1706. return 1;
  1707. }
  1708. {
  1709. auto device = alcGetContextsDevice(alcGetCurrentContext());
  1710. if(alcIsExtensionPresent(device, "ALC_SOFT_device_clock"))
  1711. {
  1712. std::cout<< "Found ALC_SOFT_device_clock" <<std::endl;
  1713. alcGetInteger64vSOFT = reinterpret_cast<LPALCGETINTEGER64VSOFT>(
  1714. alcGetProcAddress(device, "alcGetInteger64vSOFT")
  1715. );
  1716. }
  1717. }
  1718. if(alIsExtensionPresent("AL_SOFT_source_latency"))
  1719. {
  1720. std::cout<< "Found AL_SOFT_source_latency" <<std::endl;
  1721. alGetSourcei64vSOFT = reinterpret_cast<LPALGETSOURCEI64VSOFT>(
  1722. alGetProcAddress("alGetSourcei64vSOFT")
  1723. );
  1724. }
  1725. if(alIsExtensionPresent("AL_SOFT_events"))
  1726. {
  1727. std::cout<< "Found AL_SOFT_events" <<std::endl;
  1728. alEventControlSOFT = reinterpret_cast<LPALEVENTCONTROLSOFT>(
  1729. alGetProcAddress("alEventControlSOFT"));
  1730. alEventCallbackSOFT = reinterpret_cast<LPALEVENTCALLBACKSOFT>(
  1731. alGetProcAddress("alEventCallbackSOFT"));
  1732. }
  1733. if(alIsExtensionPresent("AL_SOFT_callback_buffer"))
  1734. {
  1735. std::cout<< "Found AL_SOFT_callback_buffer" <<std::endl;
  1736. alBufferCallbackSOFT = reinterpret_cast<LPALBUFFERCALLBACKSOFT>(
  1737. alGetProcAddress("alBufferCallbackSOFT"));
  1738. }
  1739. int fileidx{0};
  1740. for(;fileidx < argc;++fileidx)
  1741. {
  1742. if(strcmp(argv[fileidx], "-direct") == 0)
  1743. {
  1744. if(alIsExtensionPresent("AL_SOFT_direct_channels_remix"))
  1745. {
  1746. std::cout<< "Found AL_SOFT_direct_channels_remix" <<std::endl;
  1747. DirectOutMode = AL_REMIX_UNMATCHED_SOFT;
  1748. }
  1749. else if(alIsExtensionPresent("AL_SOFT_direct_channels"))
  1750. {
  1751. std::cout<< "Found AL_SOFT_direct_channels" <<std::endl;
  1752. DirectOutMode = AL_DROP_UNMATCHED_SOFT;
  1753. }
  1754. else
  1755. std::cerr<< "AL_SOFT_direct_channels not supported for direct output" <<std::endl;
  1756. }
  1757. else if(strcmp(argv[fileidx], "-wide") == 0)
  1758. {
  1759. if(!alIsExtensionPresent("AL_EXT_STEREO_ANGLES"))
  1760. std::cerr<< "AL_EXT_STEREO_ANGLES not supported for wide stereo" <<std::endl;
  1761. else
  1762. {
  1763. std::cout<< "Found AL_EXT_STEREO_ANGLES" <<std::endl;
  1764. EnableWideStereo = true;
  1765. }
  1766. }
  1767. else if(strcmp(argv[fileidx], "-uhj") == 0)
  1768. {
  1769. if(!alIsExtensionPresent("AL_SOFT_UHJ"))
  1770. std::cerr<< "AL_SOFT_UHJ not supported for UHJ decoding" <<std::endl;
  1771. else
  1772. {
  1773. std::cout<< "Found AL_SOFT_UHJ" <<std::endl;
  1774. EnableUhj = true;
  1775. }
  1776. }
  1777. else if(strcmp(argv[fileidx], "-superstereo") == 0)
  1778. {
  1779. if(!alIsExtensionPresent("AL_SOFT_UHJ"))
  1780. std::cerr<< "AL_SOFT_UHJ not supported for Super Stereo decoding" <<std::endl;
  1781. else
  1782. {
  1783. std::cout<< "Found AL_SOFT_UHJ (Super Stereo)" <<std::endl;
  1784. EnableSuperStereo = true;
  1785. }
  1786. }
  1787. else if(strcmp(argv[fileidx], "-novideo") == 0)
  1788. DisableVideo = true;
  1789. else
  1790. break;
  1791. }
  1792. while(fileidx < argc && !movState)
  1793. {
  1794. movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
  1795. if(!movState->prepare()) movState = nullptr;
  1796. }
  1797. if(!movState)
  1798. {
  1799. std::cerr<< "Could not start a video" <<std::endl;
  1800. return 1;
  1801. }
  1802. movState->setTitle(screen);
  1803. /* Default to going to the next movie at the end of one. */
  1804. enum class EomAction {
  1805. Next, Quit
  1806. } eom_action{EomAction::Next};
  1807. seconds last_time{seconds::min()};
  1808. while(1)
  1809. {
  1810. /* SDL_WaitEventTimeout is broken, just force a 10ms sleep. */
  1811. std::this_thread::sleep_for(milliseconds{10});
  1812. auto cur_time = std::chrono::duration_cast<seconds>(movState->getMasterClock());
  1813. if(cur_time != last_time)
  1814. {
  1815. auto end_time = std::chrono::duration_cast<seconds>(movState->getDuration());
  1816. std::cout<< " \r "<<PrettyTime{cur_time}<<" / "<<PrettyTime{end_time} <<std::flush;
  1817. last_time = cur_time;
  1818. }
  1819. bool force_redraw{false};
  1820. SDL_Event event{};
  1821. while(SDL_PollEvent(&event) != 0)
  1822. {
  1823. switch(event.type)
  1824. {
  1825. case SDL_KEYDOWN:
  1826. switch(event.key.keysym.sym)
  1827. {
  1828. case SDLK_ESCAPE:
  1829. movState->stop();
  1830. eom_action = EomAction::Quit;
  1831. break;
  1832. case SDLK_n:
  1833. movState->stop();
  1834. eom_action = EomAction::Next;
  1835. break;
  1836. default:
  1837. break;
  1838. }
  1839. break;
  1840. case SDL_WINDOWEVENT:
  1841. switch(event.window.event)
  1842. {
  1843. case SDL_WINDOWEVENT_RESIZED:
  1844. SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
  1845. SDL_RenderFillRect(renderer, nullptr);
  1846. force_redraw = true;
  1847. break;
  1848. case SDL_WINDOWEVENT_EXPOSED:
  1849. force_redraw = true;
  1850. break;
  1851. default:
  1852. break;
  1853. }
  1854. break;
  1855. case SDL_QUIT:
  1856. movState->stop();
  1857. eom_action = EomAction::Quit;
  1858. break;
  1859. case FF_MOVIE_DONE_EVENT:
  1860. std::cout<<'\n';
  1861. last_time = seconds::min();
  1862. if(eom_action != EomAction::Quit)
  1863. {
  1864. movState = nullptr;
  1865. while(fileidx < argc && !movState)
  1866. {
  1867. movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
  1868. if(!movState->prepare()) movState = nullptr;
  1869. }
  1870. if(movState)
  1871. {
  1872. movState->setTitle(screen);
  1873. break;
  1874. }
  1875. }
  1876. /* Nothing more to play. Shut everything down and quit. */
  1877. movState = nullptr;
  1878. CloseAL();
  1879. SDL_DestroyRenderer(renderer);
  1880. renderer = nullptr;
  1881. SDL_DestroyWindow(screen);
  1882. screen = nullptr;
  1883. SDL_Quit();
  1884. exit(0);
  1885. default:
  1886. break;
  1887. }
  1888. }
  1889. movState->mVideo.updateVideo(screen, renderer, force_redraw);
  1890. }
  1891. std::cerr<< "SDL_WaitEvent error - "<<SDL_GetError() <<std::endl;
  1892. return 1;
  1893. }