alffplay.cpp 72 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190
  1. /*
  2. * An example showing how to play a stream sync'd to video, using ffmpeg.
  3. *
  4. * Requires C++14.
  5. */
  6. #include <algorithm>
  7. #include <array>
  8. #include <atomic>
  9. #include <cassert>
  10. #include <cerrno>
  11. #include <chrono>
  12. #include <cmath>
  13. #include <condition_variable>
  14. #include <cstdint>
  15. #include <cstdio>
  16. #include <cstdlib>
  17. #include <cstring>
  18. #include <deque>
  19. #include <functional>
  20. #include <future>
  21. #include <iomanip>
  22. #include <iostream>
  23. #include <memory>
  24. #include <mutex>
  25. #include <ratio>
  26. #include <string>
  27. #include <string_view>
  28. #include <thread>
  29. #include <utility>
  30. #include <vector>
  31. #ifdef __GNUC__
  32. _Pragma("GCC diagnostic push")
  33. _Pragma("GCC diagnostic ignored \"-Wconversion\"")
  34. _Pragma("GCC diagnostic ignored \"-Wold-style-cast\"")
  35. #endif
  36. extern "C" {
  37. #include "libavcodec/avcodec.h"
  38. #include "libavformat/avformat.h"
  39. #include "libavformat/avio.h"
  40. #include "libavformat/version.h"
  41. #include "libavutil/avutil.h"
  42. #include "libavutil/error.h"
  43. #include "libavutil/frame.h"
  44. #include "libavutil/mem.h"
  45. #include "libavutil/pixfmt.h"
  46. #include "libavutil/rational.h"
  47. #include "libavutil/samplefmt.h"
  48. #include "libavutil/time.h"
  49. #include "libavutil/version.h"
  50. #include "libavutil/channel_layout.h"
  51. #include "libswscale/swscale.h"
  52. #include "libswresample/swresample.h"
  53. constexpr auto AVNoPtsValue = AV_NOPTS_VALUE;
  54. constexpr auto AVErrorEOF = AVERROR_EOF;
  55. struct SwsContext;
  56. }
  57. #define SDL_MAIN_HANDLED
  58. #include "SDL.h"
  59. #ifdef __GNUC__
  60. _Pragma("GCC diagnostic pop")
  61. #endif
  62. #include "AL/alc.h"
  63. #include "AL/al.h"
  64. #include "AL/alext.h"
  65. #include "alnumbers.h"
  66. #include "alnumeric.h"
  67. #include "alspan.h"
  68. #include "common/alhelpers.h"
  69. namespace {
  70. using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1_i64<<32)>>;
  71. using nanoseconds = std::chrono::nanoseconds;
  72. using microseconds = std::chrono::microseconds;
  73. using milliseconds = std::chrono::milliseconds;
  74. using seconds = std::chrono::seconds;
  75. using seconds_d64 = std::chrono::duration<double>;
  76. using std::chrono::duration_cast;
  77. const std::string AppName{"alffplay"};
  78. ALenum DirectOutMode{AL_FALSE};
  79. bool EnableWideStereo{false};
  80. bool EnableUhj{false};
  81. bool EnableSuperStereo{false};
  82. bool DisableVideo{false};
  83. LPALGETSOURCEI64VSOFT alGetSourcei64vSOFT;
  84. LPALCGETINTEGER64VSOFT alcGetInteger64vSOFT;
  85. LPALEVENTCONTROLSOFT alEventControlSOFT;
  86. LPALEVENTCALLBACKSOFT alEventCallbackSOFT;
  87. LPALBUFFERCALLBACKSOFT alBufferCallbackSOFT;
  88. const seconds AVNoSyncThreshold{10};
  89. #define VIDEO_PICTURE_QUEUE_SIZE 24
  90. const seconds_d64 AudioSyncThreshold{0.03};
  91. const milliseconds AudioSampleCorrectionMax{50};
  92. /* Averaging filter coefficient for audio sync. */
  93. #define AUDIO_DIFF_AVG_NB 20
  94. const double AudioAvgFilterCoeff{std::pow(0.01, 1.0/AUDIO_DIFF_AVG_NB)};
  95. /* Per-buffer size, in time */
  96. constexpr milliseconds AudioBufferTime{20};
  97. /* Buffer total size, in time (should be divisible by the buffer time) */
  98. constexpr milliseconds AudioBufferTotalTime{800};
  99. constexpr auto AudioBufferCount = AudioBufferTotalTime / AudioBufferTime;
  100. enum {
  101. FF_MOVIE_DONE_EVENT = SDL_USEREVENT
  102. };
  103. enum class SyncMaster {
  104. Audio,
  105. Video,
  106. External,
  107. Default = Audio
  108. };
  109. inline microseconds get_avtime()
  110. { return microseconds{av_gettime()}; }
  111. /* Define unique_ptrs to auto-cleanup associated ffmpeg objects. */
  112. struct AVIOContextDeleter {
  113. void operator()(AVIOContext *ptr) { avio_closep(&ptr); }
  114. };
  115. using AVIOContextPtr = std::unique_ptr<AVIOContext,AVIOContextDeleter>;
  116. struct AVFormatCtxDeleter {
  117. void operator()(AVFormatContext *ptr) { avformat_close_input(&ptr); }
  118. };
  119. using AVFormatCtxPtr = std::unique_ptr<AVFormatContext,AVFormatCtxDeleter>;
  120. struct AVCodecCtxDeleter {
  121. void operator()(AVCodecContext *ptr) { avcodec_free_context(&ptr); }
  122. };
  123. using AVCodecCtxPtr = std::unique_ptr<AVCodecContext,AVCodecCtxDeleter>;
  124. struct AVPacketDeleter {
  125. void operator()(AVPacket *pkt) { av_packet_free(&pkt); }
  126. };
  127. using AVPacketPtr = std::unique_ptr<AVPacket,AVPacketDeleter>;
  128. struct AVFrameDeleter {
  129. void operator()(AVFrame *ptr) { av_frame_free(&ptr); }
  130. };
  131. using AVFramePtr = std::unique_ptr<AVFrame,AVFrameDeleter>;
  132. struct SwrContextDeleter {
  133. void operator()(SwrContext *ptr) { swr_free(&ptr); }
  134. };
  135. using SwrContextPtr = std::unique_ptr<SwrContext,SwrContextDeleter>;
  136. struct SwsContextDeleter {
  137. void operator()(SwsContext *ptr) { sws_freeContext(ptr); }
  138. };
  139. using SwsContextPtr = std::unique_ptr<SwsContext,SwsContextDeleter>;
  140. struct ChannelLayout : public AVChannelLayout {
  141. ChannelLayout() : AVChannelLayout{} { }
  142. ChannelLayout(const ChannelLayout &rhs) : AVChannelLayout{}
  143. { av_channel_layout_copy(this, &rhs); }
  144. ~ChannelLayout() { av_channel_layout_uninit(this); }
  145. auto operator=(const ChannelLayout &rhs) -> ChannelLayout&
  146. { av_channel_layout_copy(this, &rhs); return *this; }
  147. };
  148. class DataQueue {
  149. const size_t mSizeLimit;
  150. std::mutex mPacketMutex, mFrameMutex;
  151. std::condition_variable mPacketCond;
  152. std::condition_variable mInFrameCond, mOutFrameCond;
  153. std::deque<AVPacketPtr> mPackets;
  154. size_t mTotalSize{0};
  155. bool mFinished{false};
  156. AVPacketPtr getPacket()
  157. {
  158. std::unique_lock<std::mutex> plock{mPacketMutex};
  159. while(mPackets.empty() && !mFinished)
  160. mPacketCond.wait(plock);
  161. if(mPackets.empty())
  162. return nullptr;
  163. auto ret = std::move(mPackets.front());
  164. mPackets.pop_front();
  165. mTotalSize -= static_cast<unsigned int>(ret->size);
  166. return ret;
  167. }
  168. public:
  169. DataQueue(size_t size_limit) : mSizeLimit{size_limit} { }
  170. int sendPacket(AVCodecContext *codecctx)
  171. {
  172. AVPacketPtr packet{getPacket()};
  173. int ret{};
  174. {
  175. std::unique_lock<std::mutex> flock{mFrameMutex};
  176. while((ret=avcodec_send_packet(codecctx, packet.get())) == AVERROR(EAGAIN))
  177. mInFrameCond.wait_for(flock, milliseconds{50});
  178. }
  179. mOutFrameCond.notify_one();
  180. if(!packet)
  181. {
  182. if(!ret) return AVErrorEOF;
  183. std::cerr<< "Failed to send flush packet: "<<ret <<std::endl;
  184. return ret;
  185. }
  186. if(ret < 0)
  187. std::cerr<< "Failed to send packet: "<<ret <<std::endl;
  188. return ret;
  189. }
  190. int receiveFrame(AVCodecContext *codecctx, AVFrame *frame)
  191. {
  192. int ret{};
  193. {
  194. std::unique_lock<std::mutex> flock{mFrameMutex};
  195. while((ret=avcodec_receive_frame(codecctx, frame)) == AVERROR(EAGAIN))
  196. mOutFrameCond.wait_for(flock, milliseconds{50});
  197. }
  198. mInFrameCond.notify_one();
  199. return ret;
  200. }
  201. void setFinished()
  202. {
  203. {
  204. std::lock_guard<std::mutex> packetlock{mPacketMutex};
  205. mFinished = true;
  206. }
  207. mPacketCond.notify_one();
  208. }
  209. void flush()
  210. {
  211. {
  212. std::lock_guard<std::mutex> packetlock{mPacketMutex};
  213. mFinished = true;
  214. mPackets.clear();
  215. mTotalSize = 0;
  216. }
  217. mPacketCond.notify_one();
  218. }
  219. bool put(const AVPacket *pkt)
  220. {
  221. {
  222. std::lock_guard<std::mutex> packet_lock{mPacketMutex};
  223. if(mTotalSize >= mSizeLimit || mFinished)
  224. return false;
  225. mPackets.push_back(AVPacketPtr{av_packet_alloc()});
  226. if(av_packet_ref(mPackets.back().get(), pkt) != 0)
  227. {
  228. mPackets.pop_back();
  229. return true;
  230. }
  231. mTotalSize += static_cast<unsigned int>(mPackets.back()->size);
  232. }
  233. mPacketCond.notify_one();
  234. return true;
  235. }
  236. };
  237. struct MovieState;
  238. struct AudioState {
  239. MovieState &mMovie;
  240. AVStream *mStream{nullptr};
  241. AVCodecCtxPtr mCodecCtx;
  242. DataQueue mQueue{2_uz*1024_uz*1024_uz};
  243. /* Used for clock difference average computation */
  244. seconds_d64 mClockDiffAvg{0};
  245. /* Time of the next sample to be buffered */
  246. nanoseconds mCurrentPts{0};
  247. /* Device clock time that the stream started at. */
  248. nanoseconds mDeviceStartTime{nanoseconds::min()};
  249. /* Decompressed sample frame, and swresample context for conversion */
  250. AVFramePtr mDecodedFrame;
  251. SwrContextPtr mSwresCtx;
  252. /* Conversion format, for what gets fed to OpenAL */
  253. uint64_t mDstChanLayout{0};
  254. AVSampleFormat mDstSampleFmt{AV_SAMPLE_FMT_NONE};
  255. /* Storage of converted samples */
  256. std::array<uint8_t*,1> mSamples{};
  257. al::span<uint8_t> mSamplesSpan{};
  258. int mSamplesLen{0}; /* In samples */
  259. int mSamplesPos{0};
  260. int mSamplesMax{0};
  261. std::vector<uint8_t> mBufferData;
  262. std::atomic<size_t> mReadPos{0};
  263. std::atomic<size_t> mWritePos{0};
  264. /* OpenAL format */
  265. ALenum mFormat{AL_NONE};
  266. ALuint mFrameSize{0};
  267. std::mutex mSrcMutex;
  268. std::condition_variable mSrcCond;
  269. std::atomic_flag mConnected{};
  270. ALuint mSource{0};
  271. std::array<ALuint,AudioBufferCount> mBuffers{};
  272. ALuint mBufferIdx{0};
  273. AudioState(MovieState &movie) : mMovie(movie)
  274. { mConnected.test_and_set(std::memory_order_relaxed); }
  275. ~AudioState()
  276. {
  277. if(mSource)
  278. alDeleteSources(1, &mSource);
  279. if(mBuffers[0])
  280. alDeleteBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
  281. av_freep(mSamples.data());
  282. }
  283. static void AL_APIENTRY eventCallbackC(ALenum eventType, ALuint object, ALuint param,
  284. ALsizei length, const ALchar *message, void *userParam) noexcept
  285. { static_cast<AudioState*>(userParam)->eventCallback(eventType, object, param, length, message); }
  286. void eventCallback(ALenum eventType, ALuint object, ALuint param, ALsizei length,
  287. const ALchar *message) noexcept;
  288. static ALsizei AL_APIENTRY bufferCallbackC(void *userptr, void *data, ALsizei size) noexcept
  289. { return static_cast<AudioState*>(userptr)->bufferCallback(data, size); }
  290. ALsizei bufferCallback(void *data, ALsizei size) noexcept;
  291. nanoseconds getClockNoLock();
  292. nanoseconds getClock()
  293. {
  294. std::lock_guard<std::mutex> lock{mSrcMutex};
  295. return getClockNoLock();
  296. }
  297. bool startPlayback();
  298. int getSync();
  299. int decodeFrame();
  300. bool readAudio(al::span<uint8_t> samples, unsigned int length, int &sample_skip);
  301. bool readAudio(int sample_skip);
  302. int handler();
  303. };
  304. struct VideoState {
  305. MovieState &mMovie;
  306. AVStream *mStream{nullptr};
  307. AVCodecCtxPtr mCodecCtx;
  308. DataQueue mQueue{14_uz*1024_uz*1024_uz};
  309. /* The pts of the currently displayed frame, and the time (av_gettime) it
  310. * was last updated - used to have running video pts
  311. */
  312. nanoseconds mDisplayPts{0};
  313. microseconds mDisplayPtsTime{microseconds::min()};
  314. std::mutex mDispPtsMutex;
  315. /* Swscale context for format conversion */
  316. SwsContextPtr mSwscaleCtx;
  317. struct Picture {
  318. AVFramePtr mFrame{};
  319. nanoseconds mPts{nanoseconds::min()};
  320. };
  321. std::array<Picture,VIDEO_PICTURE_QUEUE_SIZE> mPictQ;
  322. std::atomic<size_t> mPictQRead{0u}, mPictQWrite{1u};
  323. std::mutex mPictQMutex;
  324. std::condition_variable mPictQCond;
  325. SDL_Texture *mImage{nullptr};
  326. int mWidth{0}, mHeight{0}; /* Full texture size */
  327. bool mFirstUpdate{true};
  328. std::atomic<bool> mEOS{false};
  329. std::atomic<bool> mFinalUpdate{false};
  330. VideoState(MovieState &movie) : mMovie(movie) { }
  331. ~VideoState()
  332. {
  333. if(mImage)
  334. SDL_DestroyTexture(mImage);
  335. mImage = nullptr;
  336. }
  337. nanoseconds getClock();
  338. void display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame) const;
  339. void updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw);
  340. int handler();
  341. };
  342. struct MovieState {
  343. AVIOContextPtr mIOContext;
  344. AVFormatCtxPtr mFormatCtx;
  345. SyncMaster mAVSyncType{SyncMaster::Default};
  346. microseconds mClockBase{microseconds::min()};
  347. std::atomic<bool> mQuit{false};
  348. AudioState mAudio;
  349. VideoState mVideo;
  350. std::mutex mStartupMutex;
  351. std::condition_variable mStartupCond;
  352. bool mStartupDone{false};
  353. std::thread mParseThread;
  354. std::thread mAudioThread;
  355. std::thread mVideoThread;
  356. std::string mFilename;
  357. MovieState(std::string_view fname) : mAudio{*this}, mVideo{*this}, mFilename{fname}
  358. { }
  359. ~MovieState()
  360. {
  361. stop();
  362. if(mParseThread.joinable())
  363. mParseThread.join();
  364. }
  365. static int decode_interrupt_cb(void *ctx);
  366. bool prepare();
  367. void setTitle(SDL_Window *window) const;
  368. void stop();
  369. [[nodiscard]] nanoseconds getClock() const;
  370. [[nodiscard]] nanoseconds getMasterClock();
  371. [[nodiscard]] nanoseconds getDuration() const;
  372. bool streamComponentOpen(AVStream *stream);
  373. int parse_handler();
  374. };
  375. nanoseconds AudioState::getClockNoLock()
  376. {
  377. // The audio clock is the timestamp of the sample currently being heard.
  378. if(alcGetInteger64vSOFT)
  379. {
  380. // If device start time = min, we aren't playing yet.
  381. if(mDeviceStartTime == nanoseconds::min())
  382. return nanoseconds::zero();
  383. // Get the current device clock time and latency.
  384. auto device = alcGetContextsDevice(alcGetCurrentContext());
  385. std::array<ALCint64SOFT,2> devtimes{};
  386. alcGetInteger64vSOFT(device, ALC_DEVICE_CLOCK_LATENCY_SOFT, 2, devtimes.data());
  387. auto latency = nanoseconds{devtimes[1]};
  388. auto device_time = nanoseconds{devtimes[0]};
  389. // The clock is simply the current device time relative to the recorded
  390. // start time. We can also subtract the latency to get more a accurate
  391. // position of where the audio device actually is in the output stream.
  392. return device_time - mDeviceStartTime - latency;
  393. }
  394. if(!mBufferData.empty())
  395. {
  396. if(mDeviceStartTime == nanoseconds::min())
  397. return nanoseconds::zero();
  398. /* With a callback buffer and no device clock, mDeviceStartTime is
  399. * actually the timestamp of the first sample frame played. The audio
  400. * clock, then, is that plus the current source offset.
  401. */
  402. std::array<ALint64SOFT,2> offset{};
  403. if(alGetSourcei64vSOFT)
  404. alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset.data());
  405. else
  406. {
  407. ALint ioffset;
  408. alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
  409. offset[0] = ALint64SOFT{ioffset} << 32;
  410. }
  411. /* NOTE: The source state must be checked last, in case an underrun
  412. * occurs and the source stops between getting the state and retrieving
  413. * the offset+latency.
  414. */
  415. ALint status;
  416. alGetSourcei(mSource, AL_SOURCE_STATE, &status);
  417. nanoseconds pts{};
  418. if(status == AL_PLAYING || status == AL_PAUSED)
  419. pts = mDeviceStartTime - nanoseconds{offset[1]} +
  420. duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
  421. else
  422. {
  423. /* If the source is stopped, the pts of the next sample to be heard
  424. * is the pts of the next sample to be buffered, minus the amount
  425. * already in the buffer ready to play.
  426. */
  427. const size_t woffset{mWritePos.load(std::memory_order_acquire)};
  428. const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
  429. const size_t readable{((woffset>=roffset) ? woffset : (mBufferData.size()+woffset)) -
  430. roffset};
  431. pts = mCurrentPts - nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
  432. }
  433. return pts;
  434. }
  435. /* The source-based clock is based on 4 components:
  436. * 1 - The timestamp of the next sample to buffer (mCurrentPts)
  437. * 2 - The length of the source's buffer queue
  438. * (AudioBufferTime*AL_BUFFERS_QUEUED)
  439. * 3 - The offset OpenAL is currently at in the source (the first value
  440. * from AL_SAMPLE_OFFSET_LATENCY_SOFT)
  441. * 4 - The latency between OpenAL and the DAC (the second value from
  442. * AL_SAMPLE_OFFSET_LATENCY_SOFT)
  443. *
  444. * Subtracting the length of the source queue from the next sample's
  445. * timestamp gives the timestamp of the sample at the start of the source
  446. * queue. Adding the source offset to that results in the timestamp for the
  447. * sample at OpenAL's current position, and subtracting the source latency
  448. * from that gives the timestamp of the sample currently at the DAC.
  449. */
  450. nanoseconds pts{mCurrentPts};
  451. if(mSource)
  452. {
  453. std::array<ALint64SOFT,2> offset{};
  454. if(alGetSourcei64vSOFT)
  455. alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset.data());
  456. else
  457. {
  458. ALint ioffset;
  459. alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
  460. offset[0] = ALint64SOFT{ioffset} << 32;
  461. }
  462. ALint queued, status;
  463. alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
  464. alGetSourcei(mSource, AL_SOURCE_STATE, &status);
  465. /* If the source is AL_STOPPED, then there was an underrun and all
  466. * buffers are processed, so ignore the source queue. The audio thread
  467. * will put the source into an AL_INITIAL state and clear the queue
  468. * when it starts recovery.
  469. */
  470. if(status != AL_STOPPED)
  471. {
  472. pts -= AudioBufferTime*queued;
  473. pts += duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
  474. }
  475. /* Don't offset by the latency if the source isn't playing. */
  476. if(status == AL_PLAYING)
  477. pts -= nanoseconds{offset[1]};
  478. }
  479. return std::max(pts, nanoseconds::zero());
  480. }
  481. bool AudioState::startPlayback()
  482. {
  483. const size_t woffset{mWritePos.load(std::memory_order_acquire)};
  484. const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
  485. const size_t readable{((woffset >= roffset) ? woffset : (mBufferData.size()+woffset)) -
  486. roffset};
  487. if(!mBufferData.empty())
  488. {
  489. if(readable == 0)
  490. return false;
  491. if(!alcGetInteger64vSOFT)
  492. mDeviceStartTime = mCurrentPts -
  493. nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
  494. }
  495. else
  496. {
  497. ALint queued{};
  498. alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
  499. if(queued == 0) return false;
  500. }
  501. alSourcePlay(mSource);
  502. if(alcGetInteger64vSOFT)
  503. {
  504. /* Subtract the total buffer queue time from the current pts to get the
  505. * pts of the start of the queue.
  506. */
  507. std::array<int64_t,2> srctimes{};
  508. alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_CLOCK_SOFT, srctimes.data());
  509. auto device_time = nanoseconds{srctimes[1]};
  510. auto src_offset = duration_cast<nanoseconds>(fixed32{srctimes[0]}) /
  511. mCodecCtx->sample_rate;
  512. /* The mixer may have ticked and incremented the device time and sample
  513. * offset, so subtract the source offset from the device time to get
  514. * the device time the source started at. Also subtract startpts to get
  515. * the device time the stream would have started at to reach where it
  516. * is now.
  517. */
  518. if(!mBufferData.empty())
  519. {
  520. nanoseconds startpts{mCurrentPts -
  521. nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate};
  522. mDeviceStartTime = device_time - src_offset - startpts;
  523. }
  524. else
  525. {
  526. nanoseconds startpts{mCurrentPts - AudioBufferTotalTime};
  527. mDeviceStartTime = device_time - src_offset - startpts;
  528. }
  529. }
  530. return true;
  531. }
  532. int AudioState::getSync()
  533. {
  534. if(mMovie.mAVSyncType == SyncMaster::Audio)
  535. return 0;
  536. auto ref_clock = mMovie.getMasterClock();
  537. auto diff = ref_clock - getClockNoLock();
  538. if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
  539. {
  540. /* Difference is TOO big; reset accumulated average */
  541. mClockDiffAvg = seconds_d64::zero();
  542. return 0;
  543. }
  544. /* Accumulate the diffs */
  545. mClockDiffAvg = mClockDiffAvg*AudioAvgFilterCoeff + diff;
  546. auto avg_diff = mClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
  547. if(avg_diff < AudioSyncThreshold/2.0 && avg_diff > -AudioSyncThreshold)
  548. return 0;
  549. /* Constrain the per-update difference to avoid exceedingly large skips */
  550. diff = std::min<nanoseconds>(diff, AudioSampleCorrectionMax);
  551. return static_cast<int>(duration_cast<seconds>(diff*mCodecCtx->sample_rate).count());
  552. }
  553. int AudioState::decodeFrame()
  554. {
  555. do {
  556. while(int ret{mQueue.receiveFrame(mCodecCtx.get(), mDecodedFrame.get())})
  557. {
  558. if(ret == AVErrorEOF) return 0;
  559. std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
  560. }
  561. } while(mDecodedFrame->nb_samples <= 0);
  562. /* If provided, update w/ pts */
  563. if(mDecodedFrame->best_effort_timestamp != AVNoPtsValue)
  564. mCurrentPts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
  565. static_cast<double>(mDecodedFrame->best_effort_timestamp)});
  566. if(mDecodedFrame->nb_samples > mSamplesMax)
  567. {
  568. av_freep(mSamples.data());
  569. av_samples_alloc(mSamples.data(), nullptr, mCodecCtx->ch_layout.nb_channels,
  570. mDecodedFrame->nb_samples, mDstSampleFmt, 0);
  571. mSamplesMax = mDecodedFrame->nb_samples;
  572. mSamplesSpan = {mSamples[0], static_cast<size_t>(mSamplesMax)*mFrameSize};
  573. }
  574. /* Copy to a local to mark const. Don't know why this can't be implicit. */
  575. using data_t = decltype(decltype(mDecodedFrame)::element_type::data);
  576. std::array<const uint8_t*,std::extent_v<data_t>> cdata{};
  577. std::copy(std::begin(mDecodedFrame->data), std::end(mDecodedFrame->data), cdata.begin());
  578. /* Return the amount of sample frames converted */
  579. const int data_size{swr_convert(mSwresCtx.get(), mSamples.data(), mDecodedFrame->nb_samples,
  580. cdata.data(), mDecodedFrame->nb_samples)};
  581. av_frame_unref(mDecodedFrame.get());
  582. return data_size;
  583. }
  584. /* Duplicates the sample at in to out, count times. The frame size is a
  585. * multiple of the template type size.
  586. */
  587. template<typename T>
  588. void sample_dup(al::span<uint8_t> out, al::span<const uint8_t> in, size_t count, size_t frame_size)
  589. {
  590. auto sample = al::span{reinterpret_cast<const T*>(in.data()), in.size()/sizeof(T)};
  591. auto dst = al::span{reinterpret_cast<T*>(out.data()), out.size()/sizeof(T)};
  592. /* NOTE: frame_size is a multiple of sizeof(T). */
  593. const size_t type_mult{frame_size / sizeof(T)};
  594. if(type_mult == 1)
  595. std::fill_n(dst.begin(), count, sample.front());
  596. else for(size_t i{0};i < count;++i)
  597. {
  598. for(size_t j{0};j < type_mult;++j)
  599. dst[i*type_mult + j] = sample[j];
  600. }
  601. }
  602. void sample_dup(al::span<uint8_t> out, al::span<const uint8_t> in, size_t count, size_t frame_size)
  603. {
  604. if((frame_size&7) == 0)
  605. sample_dup<uint64_t>(out, in, count, frame_size);
  606. else if((frame_size&3) == 0)
  607. sample_dup<uint32_t>(out, in, count, frame_size);
  608. else if((frame_size&1) == 0)
  609. sample_dup<uint16_t>(out, in, count, frame_size);
  610. else
  611. sample_dup<uint8_t>(out, in, count, frame_size);
  612. }
  613. bool AudioState::readAudio(al::span<uint8_t> samples, unsigned int length, int &sample_skip)
  614. {
  615. unsigned int audio_size{0};
  616. /* Read the next chunk of data, refill the buffer, and queue it
  617. * on the source */
  618. length /= mFrameSize;
  619. while(mSamplesLen > 0 && audio_size < length)
  620. {
  621. unsigned int rem{length - audio_size};
  622. if(mSamplesPos >= 0)
  623. {
  624. const auto len = static_cast<unsigned int>(mSamplesLen - mSamplesPos);
  625. if(rem > len) rem = len;
  626. const size_t boffset{static_cast<ALuint>(mSamplesPos) * size_t{mFrameSize}};
  627. std::copy_n(mSamplesSpan.cbegin()+ptrdiff_t(boffset), rem*size_t{mFrameSize},
  628. samples.begin());
  629. }
  630. else
  631. {
  632. rem = std::min(rem, static_cast<unsigned int>(-mSamplesPos));
  633. /* Add samples by copying the first sample */
  634. sample_dup(samples, mSamplesSpan, rem, mFrameSize);
  635. }
  636. mSamplesPos += static_cast<int>(rem);
  637. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  638. samples = samples.subspan(rem*size_t{mFrameSize});
  639. audio_size += rem;
  640. while(mSamplesPos >= mSamplesLen)
  641. {
  642. mSamplesLen = decodeFrame();
  643. mSamplesPos = std::min(mSamplesLen, sample_skip);
  644. if(mSamplesLen <= 0) break;
  645. sample_skip -= mSamplesPos;
  646. // Adjust the device start time and current pts by the amount we're
  647. // skipping/duplicating, so that the clock remains correct for the
  648. // current stream position.
  649. auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
  650. mDeviceStartTime -= skip;
  651. mCurrentPts += skip;
  652. }
  653. }
  654. if(audio_size <= 0)
  655. return false;
  656. if(audio_size < length)
  657. {
  658. const unsigned int rem{length - audio_size};
  659. std::fill_n(samples.begin(), rem*mFrameSize,
  660. (mDstSampleFmt == AV_SAMPLE_FMT_U8) ? 0x80 : 0x00);
  661. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  662. }
  663. return true;
  664. }
  665. bool AudioState::readAudio(int sample_skip)
  666. {
  667. size_t woffset{mWritePos.load(std::memory_order_acquire)};
  668. const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
  669. while(mSamplesLen > 0)
  670. {
  671. const size_t nsamples{((roffset > woffset) ? roffset-woffset-1
  672. : (roffset == 0) ? (mBufferData.size()-woffset-1)
  673. : (mBufferData.size()-woffset)) / mFrameSize};
  674. if(!nsamples) break;
  675. if(mSamplesPos < 0)
  676. {
  677. const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(-mSamplesPos))};
  678. sample_dup(al::span{mBufferData}.subspan(woffset), mSamplesSpan, rem, mFrameSize);
  679. woffset += rem * mFrameSize;
  680. if(woffset == mBufferData.size()) woffset = 0;
  681. mWritePos.store(woffset, std::memory_order_release);
  682. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  683. mSamplesPos += static_cast<int>(rem);
  684. continue;
  685. }
  686. const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(mSamplesLen-mSamplesPos))};
  687. const size_t boffset{static_cast<ALuint>(mSamplesPos) * size_t{mFrameSize}};
  688. const size_t nbytes{rem * mFrameSize};
  689. std::copy_n(mSamplesSpan.cbegin()+ptrdiff_t(boffset), nbytes,
  690. mBufferData.begin()+ptrdiff_t(woffset));
  691. woffset += nbytes;
  692. if(woffset == mBufferData.size()) woffset = 0;
  693. mWritePos.store(woffset, std::memory_order_release);
  694. mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
  695. mSamplesPos += static_cast<int>(rem);
  696. while(mSamplesPos >= mSamplesLen)
  697. {
  698. mSamplesLen = decodeFrame();
  699. mSamplesPos = std::min(mSamplesLen, sample_skip);
  700. if(mSamplesLen <= 0) return false;
  701. sample_skip -= mSamplesPos;
  702. auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
  703. mDeviceStartTime -= skip;
  704. mCurrentPts += skip;
  705. }
  706. }
  707. return true;
  708. }
  709. void AL_APIENTRY AudioState::eventCallback(ALenum eventType, ALuint object, ALuint param,
  710. ALsizei length, const ALchar *message) noexcept
  711. {
  712. if(eventType == AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT)
  713. {
  714. /* Temporarily lock the source mutex to ensure it's not between
  715. * checking the processed count and going to sleep.
  716. */
  717. std::unique_lock<std::mutex>{mSrcMutex}.unlock();
  718. mSrcCond.notify_one();
  719. return;
  720. }
  721. std::cout<< "\n---- AL Event on AudioState "<<this<<" ----\nEvent: ";
  722. switch(eventType)
  723. {
  724. case AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT: std::cout<< "Buffer completed"; break;
  725. case AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT: std::cout<< "Source state changed"; break;
  726. case AL_EVENT_TYPE_DISCONNECTED_SOFT: std::cout<< "Disconnected"; break;
  727. default:
  728. std::cout<< "0x"<<std::hex<<std::setw(4)<<std::setfill('0')<<eventType<<std::dec<<
  729. std::setw(0)<<std::setfill(' '); break;
  730. }
  731. std::cout<< "\n"
  732. "Object ID: "<<object<<"\n"
  733. "Parameter: "<<param<<"\n"
  734. "Message: "<<std::string{message, static_cast<ALuint>(length)}<<"\n----"<<
  735. std::endl;
  736. if(eventType == AL_EVENT_TYPE_DISCONNECTED_SOFT)
  737. {
  738. {
  739. std::lock_guard<std::mutex> lock{mSrcMutex};
  740. mConnected.clear(std::memory_order_release);
  741. }
  742. mSrcCond.notify_one();
  743. }
  744. }
  745. ALsizei AudioState::bufferCallback(void *data, ALsizei size) noexcept
  746. {
  747. auto dst = al::span{static_cast<ALbyte*>(data), static_cast<ALuint>(size)};
  748. ALsizei got{0};
  749. size_t roffset{mReadPos.load(std::memory_order_acquire)};
  750. while(!dst.empty())
  751. {
  752. const size_t woffset{mWritePos.load(std::memory_order_relaxed)};
  753. if(woffset == roffset) break;
  754. size_t todo{((woffset < roffset) ? mBufferData.size() : woffset) - roffset};
  755. todo = std::min(todo, dst.size());
  756. std::copy_n(mBufferData.cbegin()+ptrdiff_t(roffset), todo, dst.begin());
  757. dst = dst.subspan(todo);
  758. got += static_cast<ALsizei>(todo);
  759. roffset += todo;
  760. if(roffset == mBufferData.size())
  761. roffset = 0;
  762. }
  763. mReadPos.store(roffset, std::memory_order_release);
  764. return got;
  765. }
  766. int AudioState::handler()
  767. {
  768. std::unique_lock<std::mutex> srclock{mSrcMutex, std::defer_lock};
  769. milliseconds sleep_time{AudioBufferTime / 3};
  770. struct EventControlManager {
  771. const std::array<ALenum,3> evt_types{{
  772. AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT, AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT,
  773. AL_EVENT_TYPE_DISCONNECTED_SOFT}};
  774. EventControlManager(milliseconds &sleep_time)
  775. {
  776. if(alEventControlSOFT)
  777. {
  778. alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
  779. AL_TRUE);
  780. alEventCallbackSOFT(&AudioState::eventCallbackC, this);
  781. sleep_time = AudioBufferTotalTime;
  782. }
  783. }
  784. ~EventControlManager()
  785. {
  786. if(alEventControlSOFT)
  787. {
  788. alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
  789. AL_FALSE);
  790. alEventCallbackSOFT(nullptr, nullptr);
  791. }
  792. }
  793. };
  794. EventControlManager event_controller{sleep_time};
  795. std::vector<uint8_t> samples;
  796. ALsizei buffer_len{0};
  797. /* Find a suitable format for OpenAL. */
  798. const auto layoutmask = mCodecCtx->ch_layout.u.mask; /* NOLINT(*-union-access) */
  799. mDstChanLayout = 0;
  800. mFormat = AL_NONE;
  801. if((mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLT || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLTP
  802. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBL
  803. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBLP
  804. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32
  805. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32P
  806. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64
  807. || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64P)
  808. && alIsExtensionPresent("AL_EXT_FLOAT32"))
  809. {
  810. mDstSampleFmt = AV_SAMPLE_FMT_FLT;
  811. mFrameSize = 4;
  812. if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
  813. {
  814. if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
  815. {
  816. if(layoutmask == AV_CH_LAYOUT_7POINT1)
  817. {
  818. mDstChanLayout = layoutmask;
  819. mFrameSize *= 8;
  820. mFormat = alGetEnumValue("AL_FORMAT_71CHN32");
  821. }
  822. if(layoutmask == AV_CH_LAYOUT_5POINT1 || layoutmask == AV_CH_LAYOUT_5POINT1_BACK)
  823. {
  824. mDstChanLayout = layoutmask;
  825. mFrameSize *= 6;
  826. mFormat = alGetEnumValue("AL_FORMAT_51CHN32");
  827. }
  828. if(layoutmask == AV_CH_LAYOUT_QUAD)
  829. {
  830. mDstChanLayout = layoutmask;
  831. mFrameSize *= 4;
  832. mFormat = alGetEnumValue("AL_FORMAT_QUAD32");
  833. }
  834. }
  835. if(layoutmask == AV_CH_LAYOUT_MONO)
  836. {
  837. mDstChanLayout = layoutmask;
  838. mFrameSize *= 1;
  839. mFormat = AL_FORMAT_MONO_FLOAT32;
  840. }
  841. }
  842. else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
  843. && alIsExtensionPresent("AL_EXT_BFORMAT"))
  844. {
  845. /* Calculate what should be the ambisonic order from the number of
  846. * channels, and confirm that's the number of channels. Opus allows
  847. * an optional non-diegetic stereo stream with the B-Format stream,
  848. * which we can ignore, so check for that too.
  849. */
  850. auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
  851. int channels{(order+1) * (order+1)};
  852. if(channels == mCodecCtx->ch_layout.nb_channels
  853. || channels+2 == mCodecCtx->ch_layout.nb_channels)
  854. {
  855. /* OpenAL only supports first-order with AL_EXT_BFORMAT, which
  856. * is 4 channels for 3D buffers.
  857. */
  858. mFrameSize *= 4;
  859. mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_FLOAT32");
  860. }
  861. }
  862. if(!mFormat || mFormat == -1)
  863. {
  864. mDstChanLayout = AV_CH_LAYOUT_STEREO;
  865. mFrameSize *= 2;
  866. mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN_FLOAT32_SOFT : AL_FORMAT_STEREO_FLOAT32;
  867. }
  868. }
  869. if(mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8P)
  870. {
  871. mDstSampleFmt = AV_SAMPLE_FMT_U8;
  872. mFrameSize = 1;
  873. if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
  874. {
  875. if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
  876. {
  877. if(layoutmask == AV_CH_LAYOUT_7POINT1)
  878. {
  879. mDstChanLayout = layoutmask;
  880. mFrameSize *= 8;
  881. mFormat = alGetEnumValue("AL_FORMAT_71CHN8");
  882. }
  883. if(layoutmask == AV_CH_LAYOUT_5POINT1 || layoutmask == AV_CH_LAYOUT_5POINT1_BACK)
  884. {
  885. mDstChanLayout = layoutmask;
  886. mFrameSize *= 6;
  887. mFormat = alGetEnumValue("AL_FORMAT_51CHN8");
  888. }
  889. if(layoutmask == AV_CH_LAYOUT_QUAD)
  890. {
  891. mDstChanLayout = layoutmask;
  892. mFrameSize *= 4;
  893. mFormat = alGetEnumValue("AL_FORMAT_QUAD8");
  894. }
  895. }
  896. if(layoutmask == AV_CH_LAYOUT_MONO)
  897. {
  898. mDstChanLayout = layoutmask;
  899. mFrameSize *= 1;
  900. mFormat = AL_FORMAT_MONO8;
  901. }
  902. }
  903. else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
  904. && alIsExtensionPresent("AL_EXT_BFORMAT"))
  905. {
  906. auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
  907. int channels{(order+1) * (order+1)};
  908. if(channels == mCodecCtx->ch_layout.nb_channels
  909. || channels+2 == mCodecCtx->ch_layout.nb_channels)
  910. {
  911. mFrameSize *= 4;
  912. mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_8");
  913. }
  914. }
  915. if(!mFormat || mFormat == -1)
  916. {
  917. mDstChanLayout = AV_CH_LAYOUT_STEREO;
  918. mFrameSize *= 2;
  919. mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN8_SOFT : AL_FORMAT_STEREO8;
  920. }
  921. }
  922. if(!mFormat || mFormat == -1)
  923. {
  924. mDstSampleFmt = AV_SAMPLE_FMT_S16;
  925. mFrameSize = 2;
  926. if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
  927. {
  928. if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
  929. {
  930. if(layoutmask == AV_CH_LAYOUT_7POINT1)
  931. {
  932. mDstChanLayout = layoutmask;
  933. mFrameSize *= 8;
  934. mFormat = alGetEnumValue("AL_FORMAT_71CHN16");
  935. }
  936. if(layoutmask == AV_CH_LAYOUT_5POINT1 || layoutmask == AV_CH_LAYOUT_5POINT1_BACK)
  937. {
  938. mDstChanLayout = layoutmask;
  939. mFrameSize *= 6;
  940. mFormat = alGetEnumValue("AL_FORMAT_51CHN16");
  941. }
  942. if(layoutmask == AV_CH_LAYOUT_QUAD)
  943. {
  944. mDstChanLayout = layoutmask;
  945. mFrameSize *= 4;
  946. mFormat = alGetEnumValue("AL_FORMAT_QUAD16");
  947. }
  948. }
  949. if(layoutmask == AV_CH_LAYOUT_MONO)
  950. {
  951. mDstChanLayout = layoutmask;
  952. mFrameSize *= 1;
  953. mFormat = AL_FORMAT_MONO16;
  954. }
  955. }
  956. else if(mCodecCtx->ch_layout.order == AV_CHANNEL_ORDER_AMBISONIC
  957. && alIsExtensionPresent("AL_EXT_BFORMAT"))
  958. {
  959. auto order = static_cast<int>(std::sqrt(mCodecCtx->ch_layout.nb_channels)) - 1;
  960. int channels{(order+1) * (order+1)};
  961. if(channels == mCodecCtx->ch_layout.nb_channels
  962. || channels+2 == mCodecCtx->ch_layout.nb_channels)
  963. {
  964. mFrameSize *= 4;
  965. mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_16");
  966. }
  967. }
  968. if(!mFormat || mFormat == -1)
  969. {
  970. mDstChanLayout = AV_CH_LAYOUT_STEREO;
  971. mFrameSize *= 2;
  972. mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN16_SOFT : AL_FORMAT_STEREO16;
  973. }
  974. }
  975. mSamples.fill(nullptr);
  976. mSamplesSpan = {};
  977. mSamplesMax = 0;
  978. mSamplesPos = 0;
  979. mSamplesLen = 0;
  980. mDecodedFrame.reset(av_frame_alloc());
  981. if(!mDecodedFrame)
  982. {
  983. std::cerr<< "Failed to allocate audio frame" <<std::endl;
  984. return 0;
  985. }
  986. /* Note that ffmpeg assumes AmbiX (ACN layout, SN3D normalization). */
  987. const bool has_bfmt_ex{alIsExtensionPresent("AL_SOFT_bformat_ex") != AL_FALSE};
  988. const ALenum ambi_layout{AL_ACN_SOFT};
  989. const ALenum ambi_scale{AL_SN3D_SOFT};
  990. if(!mDstChanLayout)
  991. {
  992. /* OpenAL only supports first-order ambisonics with AL_EXT_BFORMAT, so
  993. * we have to drop any extra channels.
  994. */
  995. ChannelLayout layout{};
  996. av_channel_layout_from_string(&layout, "ambisonic 1");
  997. SwrContext *ps{};
  998. int err{swr_alloc_set_opts2(&ps, &layout, mDstSampleFmt, mCodecCtx->sample_rate,
  999. &mCodecCtx->ch_layout, mCodecCtx->sample_fmt, mCodecCtx->sample_rate, 0, nullptr)};
  1000. mSwresCtx.reset(ps);
  1001. if(err != 0)
  1002. {
  1003. std::array<char,AV_ERROR_MAX_STRING_SIZE> errstr{};
  1004. std::cerr<< "Failed to allocate SwrContext: "
  1005. <<av_make_error_string(errstr.data(), AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
  1006. return 0;
  1007. }
  1008. if(has_bfmt_ex)
  1009. std::cout<< "Found AL_SOFT_bformat_ex" <<std::endl;
  1010. else
  1011. {
  1012. std::cout<< "Found AL_EXT_BFORMAT" <<std::endl;
  1013. /* Without AL_SOFT_bformat_ex, OpenAL only supports FuMa channel
  1014. * ordering and normalization, so a custom matrix is needed to
  1015. * scale and reorder the source from AmbiX.
  1016. */
  1017. std::vector<double> mtx(64_uz*64_uz, 0.0);
  1018. mtx[0 + 0*64] = std::sqrt(0.5);
  1019. mtx[3 + 1*64] = 1.0;
  1020. mtx[1 + 2*64] = 1.0;
  1021. mtx[2 + 3*64] = 1.0;
  1022. swr_set_matrix(mSwresCtx.get(), mtx.data(), 64);
  1023. }
  1024. }
  1025. else
  1026. {
  1027. ChannelLayout layout{};
  1028. av_channel_layout_from_mask(&layout, mDstChanLayout);
  1029. SwrContext *ps{};
  1030. int err{swr_alloc_set_opts2(&ps, &layout, mDstSampleFmt, mCodecCtx->sample_rate,
  1031. &mCodecCtx->ch_layout, mCodecCtx->sample_fmt, mCodecCtx->sample_rate, 0, nullptr)};
  1032. mSwresCtx.reset(ps);
  1033. if(err != 0)
  1034. {
  1035. std::array<char,AV_ERROR_MAX_STRING_SIZE> errstr{};
  1036. std::cerr<< "Failed to allocate SwrContext: "
  1037. <<av_make_error_string(errstr.data(), AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
  1038. return 0;
  1039. }
  1040. }
  1041. if(int err{swr_init(mSwresCtx.get())})
  1042. {
  1043. std::array<char,AV_ERROR_MAX_STRING_SIZE> errstr{};
  1044. std::cerr<< "Failed to initialize audio converter: "
  1045. <<av_make_error_string(errstr.data(), AV_ERROR_MAX_STRING_SIZE, err) <<std::endl;
  1046. return 0;
  1047. }
  1048. alGenBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
  1049. alGenSources(1, &mSource);
  1050. if(DirectOutMode)
  1051. alSourcei(mSource, AL_DIRECT_CHANNELS_SOFT, DirectOutMode);
  1052. if(EnableWideStereo)
  1053. {
  1054. static constexpr std::array angles{static_cast<float>(al::numbers::pi / 3.0),
  1055. static_cast<float>(-al::numbers::pi / 3.0)};
  1056. alSourcefv(mSource, AL_STEREO_ANGLES, angles.data());
  1057. }
  1058. if(has_bfmt_ex)
  1059. {
  1060. for(ALuint bufid : mBuffers)
  1061. {
  1062. alBufferi(bufid, AL_AMBISONIC_LAYOUT_SOFT, ambi_layout);
  1063. alBufferi(bufid, AL_AMBISONIC_SCALING_SOFT, ambi_scale);
  1064. }
  1065. }
  1066. #ifdef AL_SOFT_UHJ
  1067. if(EnableSuperStereo)
  1068. alSourcei(mSource, AL_STEREO_MODE_SOFT, AL_SUPER_STEREO_SOFT);
  1069. #endif
  1070. if(alGetError() != AL_NO_ERROR)
  1071. return 0;
  1072. bool callback_ok{false};
  1073. if(alBufferCallbackSOFT)
  1074. {
  1075. alBufferCallbackSOFT(mBuffers[0], mFormat, mCodecCtx->sample_rate, bufferCallbackC, this);
  1076. alSourcei(mSource, AL_BUFFER, static_cast<ALint>(mBuffers[0]));
  1077. if(alGetError() != AL_NO_ERROR)
  1078. {
  1079. fprintf(stderr, "Failed to set buffer callback\n");
  1080. alSourcei(mSource, AL_BUFFER, 0);
  1081. }
  1082. else
  1083. {
  1084. mBufferData.resize(static_cast<size_t>(duration_cast<seconds>(mCodecCtx->sample_rate *
  1085. AudioBufferTotalTime).count()) * mFrameSize);
  1086. std::fill(mBufferData.begin(), mBufferData.end(), uint8_t{});
  1087. mReadPos.store(0, std::memory_order_relaxed);
  1088. mWritePos.store(mBufferData.size()/mFrameSize/2*mFrameSize, std::memory_order_relaxed);
  1089. ALCint refresh{};
  1090. alcGetIntegerv(alcGetContextsDevice(alcGetCurrentContext()), ALC_REFRESH, 1, &refresh);
  1091. sleep_time = milliseconds{seconds{1}} / refresh;
  1092. callback_ok = true;
  1093. }
  1094. }
  1095. if(!callback_ok)
  1096. buffer_len = static_cast<int>(duration_cast<seconds>(mCodecCtx->sample_rate *
  1097. AudioBufferTime).count() * mFrameSize);
  1098. if(buffer_len > 0)
  1099. samples.resize(static_cast<ALuint>(buffer_len));
  1100. /* Prefill the codec buffer. */
  1101. auto packet_sender = [this]()
  1102. {
  1103. while(true)
  1104. {
  1105. const int ret{mQueue.sendPacket(mCodecCtx.get())};
  1106. if(ret == AVErrorEOF) break;
  1107. }
  1108. };
  1109. auto sender = std::async(std::launch::async, packet_sender);
  1110. srclock.lock();
  1111. if(alcGetInteger64vSOFT)
  1112. {
  1113. int64_t devtime{};
  1114. alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()), ALC_DEVICE_CLOCK_SOFT,
  1115. 1, &devtime);
  1116. mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
  1117. }
  1118. mSamplesLen = decodeFrame();
  1119. if(mSamplesLen > 0)
  1120. {
  1121. mSamplesPos = std::min(mSamplesLen, getSync());
  1122. auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
  1123. mDeviceStartTime -= skip;
  1124. mCurrentPts += skip;
  1125. }
  1126. while(true)
  1127. {
  1128. if(mMovie.mQuit.load(std::memory_order_relaxed))
  1129. {
  1130. /* If mQuit is set, drain frames until we can't get more audio,
  1131. * indicating we've reached the flush packet and the packet sender
  1132. * will also quit.
  1133. */
  1134. do {
  1135. mSamplesLen = decodeFrame();
  1136. mSamplesPos = mSamplesLen;
  1137. } while(mSamplesLen > 0);
  1138. break;
  1139. }
  1140. ALenum state;
  1141. if(!mBufferData.empty())
  1142. {
  1143. alGetSourcei(mSource, AL_SOURCE_STATE, &state);
  1144. /* If mQuit is not set, don't quit even if there's no more audio,
  1145. * so what's buffered has a chance to play to the real end.
  1146. */
  1147. readAudio(getSync());
  1148. }
  1149. else
  1150. {
  1151. ALint processed, queued;
  1152. /* First remove any processed buffers. */
  1153. alGetSourcei(mSource, AL_BUFFERS_PROCESSED, &processed);
  1154. while(processed > 0)
  1155. {
  1156. ALuint bid;
  1157. alSourceUnqueueBuffers(mSource, 1, &bid);
  1158. --processed;
  1159. }
  1160. /* Refill the buffer queue. */
  1161. int sync_skip{getSync()};
  1162. alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
  1163. while(static_cast<ALuint>(queued) < mBuffers.size())
  1164. {
  1165. /* Read the next chunk of data, filling the buffer, and queue
  1166. * it on the source.
  1167. */
  1168. if(!readAudio(samples, static_cast<ALuint>(buffer_len), sync_skip))
  1169. break;
  1170. const ALuint bufid{mBuffers[mBufferIdx]};
  1171. mBufferIdx = static_cast<ALuint>((mBufferIdx+1) % mBuffers.size());
  1172. alBufferData(bufid, mFormat, samples.data(), buffer_len, mCodecCtx->sample_rate);
  1173. alSourceQueueBuffers(mSource, 1, &bufid);
  1174. ++queued;
  1175. }
  1176. /* Check that the source is playing. */
  1177. alGetSourcei(mSource, AL_SOURCE_STATE, &state);
  1178. if(state == AL_STOPPED)
  1179. {
  1180. /* AL_STOPPED means there was an underrun. Clear the buffer
  1181. * queue since this likely means we're late, and rewind the
  1182. * source to get it back into an AL_INITIAL state.
  1183. */
  1184. alSourceRewind(mSource);
  1185. alSourcei(mSource, AL_BUFFER, 0);
  1186. if(alcGetInteger64vSOFT)
  1187. {
  1188. /* Also update the device start time with the current
  1189. * device clock, so the decoder knows we're running behind.
  1190. */
  1191. int64_t devtime{};
  1192. alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()),
  1193. ALC_DEVICE_CLOCK_SOFT, 1, &devtime);
  1194. mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
  1195. }
  1196. continue;
  1197. }
  1198. }
  1199. /* (re)start the source if needed, and wait for a buffer to finish */
  1200. if(state != AL_PLAYING && state != AL_PAUSED)
  1201. {
  1202. if(!startPlayback())
  1203. break;
  1204. }
  1205. if(ALenum err{alGetError()})
  1206. std::cerr<< "Got AL error: 0x"<<std::hex<<err<<std::dec
  1207. << " ("<<alGetString(err)<<")" <<std::endl;
  1208. mSrcCond.wait_for(srclock, sleep_time);
  1209. }
  1210. alSourceRewind(mSource);
  1211. alSourcei(mSource, AL_BUFFER, 0);
  1212. srclock.unlock();
  1213. return 0;
  1214. }
  1215. nanoseconds VideoState::getClock()
  1216. {
  1217. /* NOTE: This returns incorrect times while not playing. */
  1218. std::lock_guard<std::mutex> displock{mDispPtsMutex};
  1219. if(mDisplayPtsTime == microseconds::min())
  1220. return nanoseconds::zero();
  1221. auto delta = get_avtime() - mDisplayPtsTime;
  1222. return mDisplayPts + delta;
  1223. }
  1224. /* Called by VideoState::updateVideo to display the next video frame. */
  1225. void VideoState::display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame) const
  1226. {
  1227. if(!mImage)
  1228. return;
  1229. double aspect_ratio;
  1230. int win_w, win_h;
  1231. int w, h, x, y;
  1232. int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
  1233. int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
  1234. if(frame->sample_aspect_ratio.num == 0)
  1235. aspect_ratio = 0.0;
  1236. else
  1237. {
  1238. aspect_ratio = av_q2d(frame->sample_aspect_ratio) * frame_width /
  1239. frame_height;
  1240. }
  1241. if(aspect_ratio <= 0.0)
  1242. aspect_ratio = static_cast<double>(frame_width) / frame_height;
  1243. SDL_GetWindowSize(screen, &win_w, &win_h);
  1244. h = win_h;
  1245. w = (static_cast<int>(std::rint(h * aspect_ratio)) + 3) & ~3;
  1246. if(w > win_w)
  1247. {
  1248. w = win_w;
  1249. h = (static_cast<int>(std::rint(w / aspect_ratio)) + 3) & ~3;
  1250. }
  1251. x = (win_w - w) / 2;
  1252. y = (win_h - h) / 2;
  1253. SDL_Rect src_rect{ static_cast<int>(frame->crop_left), static_cast<int>(frame->crop_top),
  1254. frame_width, frame_height };
  1255. SDL_Rect dst_rect{ x, y, w, h };
  1256. SDL_RenderCopy(renderer, mImage, &src_rect, &dst_rect);
  1257. SDL_RenderPresent(renderer);
  1258. }
  1259. /* Called regularly on the main thread where the SDL_Renderer was created. It
  1260. * handles updating the textures of decoded frames and displaying the latest
  1261. * frame.
  1262. */
  1263. void VideoState::updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw)
  1264. {
  1265. size_t read_idx{mPictQRead.load(std::memory_order_relaxed)};
  1266. Picture *vp{&mPictQ[read_idx]};
  1267. auto clocktime = mMovie.getMasterClock();
  1268. bool updated{false};
  1269. while(true)
  1270. {
  1271. size_t next_idx{(read_idx+1)%mPictQ.size()};
  1272. if(next_idx == mPictQWrite.load(std::memory_order_acquire))
  1273. break;
  1274. Picture *nextvp{&mPictQ[next_idx]};
  1275. if(clocktime < nextvp->mPts && !mMovie.mQuit.load(std::memory_order_relaxed))
  1276. {
  1277. /* For the first update, ensure the first frame gets shown. */
  1278. if(!mFirstUpdate || updated)
  1279. break;
  1280. }
  1281. vp = nextvp;
  1282. updated = true;
  1283. read_idx = next_idx;
  1284. }
  1285. if(mMovie.mQuit.load(std::memory_order_relaxed))
  1286. {
  1287. if(mEOS)
  1288. mFinalUpdate = true;
  1289. mPictQRead.store(read_idx, std::memory_order_release);
  1290. std::unique_lock<std::mutex>{mPictQMutex}.unlock();
  1291. mPictQCond.notify_one();
  1292. return;
  1293. }
  1294. AVFrame *frame{vp->mFrame.get()};
  1295. if(updated)
  1296. {
  1297. mPictQRead.store(read_idx, std::memory_order_release);
  1298. std::unique_lock<std::mutex>{mPictQMutex}.unlock();
  1299. mPictQCond.notify_one();
  1300. /* allocate or resize the buffer! */
  1301. bool fmt_updated{false};
  1302. if(!mImage || mWidth != frame->width || mHeight != frame->height)
  1303. {
  1304. fmt_updated = true;
  1305. if(mImage)
  1306. SDL_DestroyTexture(mImage);
  1307. mImage = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING,
  1308. frame->width, frame->height);
  1309. if(!mImage)
  1310. std::cerr<< "Failed to create YV12 texture!" <<std::endl;
  1311. mWidth = frame->width;
  1312. mHeight = frame->height;
  1313. }
  1314. int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
  1315. int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
  1316. if(mFirstUpdate && frame_width > 0 && frame_height > 0)
  1317. {
  1318. /* For the first update, set the window size to the video size. */
  1319. mFirstUpdate = false;
  1320. if(frame->sample_aspect_ratio.den != 0)
  1321. {
  1322. double aspect_ratio = av_q2d(frame->sample_aspect_ratio);
  1323. if(aspect_ratio >= 1.0)
  1324. frame_width = static_cast<int>(std::lround(frame_width * aspect_ratio));
  1325. else if(aspect_ratio > 0.0)
  1326. frame_height = static_cast<int>(std::lround(frame_height / aspect_ratio));
  1327. }
  1328. SDL_SetWindowSize(screen, frame_width, frame_height);
  1329. }
  1330. if(mImage)
  1331. {
  1332. void *pixels{nullptr};
  1333. int pitch{0};
  1334. if(mCodecCtx->pix_fmt == AV_PIX_FMT_YUV420P)
  1335. SDL_UpdateYUVTexture(mImage, nullptr,
  1336. frame->data[0], frame->linesize[0],
  1337. frame->data[1], frame->linesize[1],
  1338. frame->data[2], frame->linesize[2]
  1339. );
  1340. else if(SDL_LockTexture(mImage, nullptr, &pixels, &pitch) != 0)
  1341. std::cerr<< "Failed to lock texture" <<std::endl;
  1342. else
  1343. {
  1344. // Convert the image into YUV format that SDL uses
  1345. int w{frame->width};
  1346. int h{frame->height};
  1347. if(!mSwscaleCtx || fmt_updated)
  1348. {
  1349. mSwscaleCtx.reset(sws_getContext(
  1350. w, h, mCodecCtx->pix_fmt,
  1351. w, h, AV_PIX_FMT_YUV420P, 0,
  1352. nullptr, nullptr, nullptr
  1353. ));
  1354. }
  1355. /* point pict at the queue */
  1356. const auto framesize = static_cast<size_t>(w)*static_cast<size_t>(h);
  1357. const auto pixelspan = al::span{static_cast<uint8_t*>(pixels), framesize*3/2};
  1358. const std::array pict_data{
  1359. al::to_address(pixelspan.begin()),
  1360. al::to_address(pixelspan.begin() + ptrdiff_t{w}*h),
  1361. al::to_address(pixelspan.begin() + ptrdiff_t{w}*h + ptrdiff_t{w}*h/4)
  1362. };
  1363. const std::array pict_linesize{pitch, pitch/2, pitch/2};
  1364. sws_scale(mSwscaleCtx.get(), std::data(frame->data), std::data(frame->linesize),
  1365. 0, h, pict_data.data(), pict_linesize.data());
  1366. SDL_UnlockTexture(mImage);
  1367. }
  1368. redraw = true;
  1369. }
  1370. }
  1371. if(redraw)
  1372. {
  1373. /* Show the picture! */
  1374. display(screen, renderer, frame);
  1375. }
  1376. if(updated)
  1377. {
  1378. auto disp_time = get_avtime();
  1379. std::lock_guard<std::mutex> displock{mDispPtsMutex};
  1380. mDisplayPts = vp->mPts;
  1381. mDisplayPtsTime = disp_time;
  1382. }
  1383. if(mEOS.load(std::memory_order_acquire))
  1384. {
  1385. if((read_idx+1)%mPictQ.size() == mPictQWrite.load(std::memory_order_acquire))
  1386. {
  1387. mFinalUpdate = true;
  1388. std::unique_lock<std::mutex>{mPictQMutex}.unlock();
  1389. mPictQCond.notify_one();
  1390. }
  1391. }
  1392. }
  1393. int VideoState::handler()
  1394. {
  1395. std::for_each(mPictQ.begin(), mPictQ.end(),
  1396. [](Picture &pict) -> void
  1397. { pict.mFrame = AVFramePtr{av_frame_alloc()}; });
  1398. /* Prefill the codec buffer. */
  1399. auto packet_sender = [this]()
  1400. {
  1401. while(true)
  1402. {
  1403. const int ret{mQueue.sendPacket(mCodecCtx.get())};
  1404. if(ret == AVErrorEOF) break;
  1405. }
  1406. };
  1407. auto sender = std::async(std::launch::async, packet_sender);
  1408. {
  1409. std::lock_guard<std::mutex> displock{mDispPtsMutex};
  1410. mDisplayPtsTime = get_avtime();
  1411. }
  1412. auto current_pts = nanoseconds::zero();
  1413. while(true)
  1414. {
  1415. size_t write_idx{mPictQWrite.load(std::memory_order_relaxed)};
  1416. Picture *vp{&mPictQ[write_idx]};
  1417. /* Retrieve video frame. */
  1418. AVFrame *decoded_frame{vp->mFrame.get()};
  1419. while(int ret{mQueue.receiveFrame(mCodecCtx.get(), decoded_frame)})
  1420. {
  1421. if(ret == AVErrorEOF) goto finish;
  1422. std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
  1423. }
  1424. /* Get the PTS for this frame. */
  1425. if(decoded_frame->best_effort_timestamp != AVNoPtsValue)
  1426. current_pts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
  1427. static_cast<double>(decoded_frame->best_effort_timestamp)});
  1428. vp->mPts = current_pts;
  1429. /* Update the video clock to the next expected PTS. */
  1430. auto frame_delay = av_q2d(mCodecCtx->time_base);
  1431. frame_delay += decoded_frame->repeat_pict * (frame_delay * 0.5);
  1432. current_pts += duration_cast<nanoseconds>(seconds_d64{frame_delay});
  1433. /* Put the frame in the queue to be loaded into a texture and displayed
  1434. * by the rendering thread.
  1435. */
  1436. write_idx = (write_idx+1)%mPictQ.size();
  1437. mPictQWrite.store(write_idx, std::memory_order_release);
  1438. if(write_idx == mPictQRead.load(std::memory_order_acquire))
  1439. {
  1440. /* Wait until we have space for a new pic */
  1441. std::unique_lock<std::mutex> lock{mPictQMutex};
  1442. while(write_idx == mPictQRead.load(std::memory_order_acquire))
  1443. mPictQCond.wait(lock);
  1444. }
  1445. }
  1446. finish:
  1447. mEOS = true;
  1448. std::unique_lock<std::mutex> lock{mPictQMutex};
  1449. while(!mFinalUpdate) mPictQCond.wait(lock);
  1450. return 0;
  1451. }
  1452. int MovieState::decode_interrupt_cb(void *ctx)
  1453. {
  1454. return static_cast<MovieState*>(ctx)->mQuit.load(std::memory_order_relaxed);
  1455. }
  1456. bool MovieState::prepare()
  1457. {
  1458. AVIOContext *avioctx{nullptr};
  1459. AVIOInterruptCB intcb{decode_interrupt_cb, this};
  1460. if(avio_open2(&avioctx, mFilename.c_str(), AVIO_FLAG_READ, &intcb, nullptr))
  1461. {
  1462. std::cerr<< "Failed to open "<<mFilename <<std::endl;
  1463. return false;
  1464. }
  1465. mIOContext.reset(avioctx);
  1466. /* Open movie file. If avformat_open_input fails it will automatically free
  1467. * this context, so don't set it onto a smart pointer yet.
  1468. */
  1469. AVFormatContext *fmtctx{avformat_alloc_context()};
  1470. fmtctx->pb = mIOContext.get();
  1471. fmtctx->interrupt_callback = intcb;
  1472. if(avformat_open_input(&fmtctx, mFilename.c_str(), nullptr, nullptr) != 0)
  1473. {
  1474. std::cerr<< "Failed to open "<<mFilename <<std::endl;
  1475. return false;
  1476. }
  1477. mFormatCtx.reset(fmtctx);
  1478. /* Retrieve stream information */
  1479. if(avformat_find_stream_info(mFormatCtx.get(), nullptr) < 0)
  1480. {
  1481. std::cerr<< mFilename<<": failed to find stream info" <<std::endl;
  1482. return false;
  1483. }
  1484. /* Dump information about file onto standard error */
  1485. av_dump_format(mFormatCtx.get(), 0, mFilename.c_str(), 0);
  1486. mParseThread = std::thread{std::mem_fn(&MovieState::parse_handler), this};
  1487. std::unique_lock<std::mutex> slock{mStartupMutex};
  1488. while(!mStartupDone) mStartupCond.wait(slock);
  1489. return true;
  1490. }
  1491. void MovieState::setTitle(SDL_Window *window) const
  1492. {
  1493. auto pos1 = mFilename.rfind('/');
  1494. auto pos2 = mFilename.rfind('\\');
  1495. auto fpos = ((pos1 == std::string::npos) ? pos2 :
  1496. (pos2 == std::string::npos) ? pos1 :
  1497. std::max(pos1, pos2)) + 1;
  1498. SDL_SetWindowTitle(window, (mFilename.substr(fpos)+" - "+AppName).c_str());
  1499. }
  1500. nanoseconds MovieState::getClock() const
  1501. {
  1502. if(mClockBase == microseconds::min())
  1503. return nanoseconds::zero();
  1504. return get_avtime() - mClockBase;
  1505. }
  1506. nanoseconds MovieState::getMasterClock()
  1507. {
  1508. if(mAVSyncType == SyncMaster::Video && mVideo.mStream)
  1509. return mVideo.getClock();
  1510. if(mAVSyncType == SyncMaster::Audio && mAudio.mStream)
  1511. return mAudio.getClock();
  1512. return getClock();
  1513. }
  1514. nanoseconds MovieState::getDuration() const
  1515. { return std::chrono::duration<int64_t,std::ratio<1,AV_TIME_BASE>>(mFormatCtx->duration); }
  1516. bool MovieState::streamComponentOpen(AVStream *stream)
  1517. {
  1518. /* Get a pointer to the codec context for the stream, and open the
  1519. * associated codec.
  1520. */
  1521. AVCodecCtxPtr avctx{avcodec_alloc_context3(nullptr)};
  1522. if(!avctx) return false;
  1523. if(avcodec_parameters_to_context(avctx.get(), stream->codecpar))
  1524. return false;
  1525. const AVCodec *codec{avcodec_find_decoder(avctx->codec_id)};
  1526. if(!codec || avcodec_open2(avctx.get(), codec, nullptr) < 0)
  1527. {
  1528. std::cerr<< "Unsupported codec: "<<avcodec_get_name(avctx->codec_id)
  1529. << " (0x"<<std::hex<<avctx->codec_id<<std::dec<<")" <<std::endl;
  1530. return false;
  1531. }
  1532. /* Initialize and start the media type handler */
  1533. switch(avctx->codec_type)
  1534. {
  1535. case AVMEDIA_TYPE_AUDIO:
  1536. mAudio.mStream = stream;
  1537. mAudio.mCodecCtx = std::move(avctx);
  1538. return true;
  1539. case AVMEDIA_TYPE_VIDEO:
  1540. mVideo.mStream = stream;
  1541. mVideo.mCodecCtx = std::move(avctx);
  1542. return true;
  1543. default:
  1544. break;
  1545. }
  1546. return false;
  1547. }
  1548. int MovieState::parse_handler()
  1549. {
  1550. auto &audio_queue = mAudio.mQueue;
  1551. auto &video_queue = mVideo.mQueue;
  1552. int video_index{-1};
  1553. int audio_index{-1};
  1554. /* Find the first video and audio streams */
  1555. const auto ctxstreams = al::span{mFormatCtx->streams, mFormatCtx->nb_streams};
  1556. for(size_t i{0};i < ctxstreams.size();++i)
  1557. {
  1558. auto codecpar = ctxstreams[i]->codecpar;
  1559. if(codecpar->codec_type == AVMEDIA_TYPE_VIDEO && !DisableVideo && video_index < 0
  1560. && streamComponentOpen(ctxstreams[i]))
  1561. video_index = static_cast<int>(i);
  1562. else if(codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_index < 0
  1563. && streamComponentOpen(ctxstreams[i]))
  1564. audio_index = static_cast<int>(i);
  1565. }
  1566. {
  1567. std::unique_lock<std::mutex> slock{mStartupMutex};
  1568. mStartupDone = true;
  1569. }
  1570. mStartupCond.notify_all();
  1571. if(video_index < 0 && audio_index < 0)
  1572. {
  1573. std::cerr<< mFilename<<": could not open codecs" <<std::endl;
  1574. mQuit = true;
  1575. }
  1576. /* Set the base time 750ms ahead of the current av time. */
  1577. mClockBase = get_avtime() + milliseconds{750};
  1578. if(audio_index >= 0)
  1579. mAudioThread = std::thread{std::mem_fn(&AudioState::handler), &mAudio};
  1580. if(video_index >= 0)
  1581. mVideoThread = std::thread{std::mem_fn(&VideoState::handler), &mVideo};
  1582. /* Main packet reading/dispatching loop */
  1583. AVPacketPtr packet{av_packet_alloc()};
  1584. while(!mQuit.load(std::memory_order_relaxed))
  1585. {
  1586. if(av_read_frame(mFormatCtx.get(), packet.get()) < 0)
  1587. break;
  1588. /* Copy the packet into the queue it's meant for. */
  1589. if(packet->stream_index == video_index)
  1590. {
  1591. while(!mQuit.load(std::memory_order_acquire) && !video_queue.put(packet.get()))
  1592. std::this_thread::sleep_for(milliseconds{100});
  1593. }
  1594. else if(packet->stream_index == audio_index)
  1595. {
  1596. while(!mQuit.load(std::memory_order_acquire) && !audio_queue.put(packet.get()))
  1597. std::this_thread::sleep_for(milliseconds{100});
  1598. }
  1599. av_packet_unref(packet.get());
  1600. }
  1601. /* Finish the queues so the receivers know nothing more is coming. */
  1602. video_queue.setFinished();
  1603. audio_queue.setFinished();
  1604. /* all done - wait for it */
  1605. if(mVideoThread.joinable())
  1606. mVideoThread.join();
  1607. if(mAudioThread.joinable())
  1608. mAudioThread.join();
  1609. mVideo.mEOS = true;
  1610. std::unique_lock<std::mutex> lock{mVideo.mPictQMutex};
  1611. while(!mVideo.mFinalUpdate)
  1612. mVideo.mPictQCond.wait(lock);
  1613. lock.unlock();
  1614. SDL_Event evt{};
  1615. evt.user.type = FF_MOVIE_DONE_EVENT;
  1616. SDL_PushEvent(&evt);
  1617. return 0;
  1618. }
  1619. void MovieState::stop()
  1620. {
  1621. mQuit = true;
  1622. mAudio.mQueue.flush();
  1623. mVideo.mQueue.flush();
  1624. }
  1625. // Helper class+method to print the time with human-readable formatting.
  1626. struct PrettyTime {
  1627. seconds mTime;
  1628. };
  1629. std::ostream &operator<<(std::ostream &os, const PrettyTime &rhs)
  1630. {
  1631. using hours = std::chrono::hours;
  1632. using minutes = std::chrono::minutes;
  1633. seconds t{rhs.mTime};
  1634. if(t.count() < 0)
  1635. {
  1636. os << '-';
  1637. t *= -1;
  1638. }
  1639. // Only handle up to hour formatting
  1640. if(t >= hours{1})
  1641. os << duration_cast<hours>(t).count() << 'h' << std::setfill('0') << std::setw(2)
  1642. << (duration_cast<minutes>(t).count() % 60) << 'm';
  1643. else
  1644. os << duration_cast<minutes>(t).count() << 'm' << std::setfill('0');
  1645. os << std::setw(2) << (duration_cast<seconds>(t).count() % 60) << 's' << std::setw(0)
  1646. << std::setfill(' ');
  1647. return os;
  1648. }
  1649. int main(al::span<std::string_view> args)
  1650. {
  1651. SDL_SetMainReady();
  1652. std::unique_ptr<MovieState> movState;
  1653. if(args.size() < 2)
  1654. {
  1655. std::cerr<< "Usage: "<<args[0]<<" [-device <device name>] [-direct] <files...>" <<std::endl;
  1656. return 1;
  1657. }
  1658. /* Register all formats and codecs */
  1659. #if !(LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 9, 100))
  1660. av_register_all();
  1661. #endif
  1662. /* Initialize networking protocols */
  1663. avformat_network_init();
  1664. if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS))
  1665. {
  1666. std::cerr<< "Could not initialize SDL - <<"<<SDL_GetError() <<std::endl;
  1667. return 1;
  1668. }
  1669. /* Make a window to put our video */
  1670. SDL_Window *screen{SDL_CreateWindow(AppName.c_str(), 0, 0, 640, 480, SDL_WINDOW_RESIZABLE)};
  1671. if(!screen)
  1672. {
  1673. std::cerr<< "SDL: could not set video mode - exiting" <<std::endl;
  1674. return 1;
  1675. }
  1676. /* Make a renderer to handle the texture image surface and rendering. */
  1677. Uint32 render_flags{SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC};
  1678. SDL_Renderer *renderer{SDL_CreateRenderer(screen, -1, render_flags)};
  1679. if(renderer)
  1680. {
  1681. SDL_RendererInfo rinf{};
  1682. bool ok{false};
  1683. /* Make sure the renderer supports IYUV textures. If not, fallback to a
  1684. * software renderer. */
  1685. if(SDL_GetRendererInfo(renderer, &rinf) == 0)
  1686. {
  1687. for(Uint32 i{0u};!ok && i < rinf.num_texture_formats;i++)
  1688. ok = (rinf.texture_formats[i] == SDL_PIXELFORMAT_IYUV);
  1689. }
  1690. if(!ok)
  1691. {
  1692. std::cerr<< "IYUV pixelformat textures not supported on renderer "<<rinf.name <<std::endl;
  1693. SDL_DestroyRenderer(renderer);
  1694. renderer = nullptr;
  1695. }
  1696. }
  1697. if(!renderer)
  1698. {
  1699. render_flags = SDL_RENDERER_SOFTWARE | SDL_RENDERER_PRESENTVSYNC;
  1700. renderer = SDL_CreateRenderer(screen, -1, render_flags);
  1701. }
  1702. if(!renderer)
  1703. {
  1704. std::cerr<< "SDL: could not create renderer - exiting" <<std::endl;
  1705. return 1;
  1706. }
  1707. SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
  1708. SDL_RenderFillRect(renderer, nullptr);
  1709. SDL_RenderPresent(renderer);
  1710. /* Open an audio device */
  1711. args = args.subspan(1);
  1712. if(InitAL(args) != 0)
  1713. return 1;
  1714. {
  1715. ALCdevice *device{alcGetContextsDevice(alcGetCurrentContext())};
  1716. if(alcIsExtensionPresent(device,"ALC_SOFT_device_clock"))
  1717. {
  1718. std::cout<< "Found ALC_SOFT_device_clock" <<std::endl;
  1719. alcGetInteger64vSOFT = reinterpret_cast<LPALCGETINTEGER64VSOFT>(
  1720. alcGetProcAddress(device, "alcGetInteger64vSOFT"));
  1721. }
  1722. }
  1723. if(alIsExtensionPresent("AL_SOFT_source_latency"))
  1724. {
  1725. std::cout<< "Found AL_SOFT_source_latency" <<std::endl;
  1726. alGetSourcei64vSOFT = reinterpret_cast<LPALGETSOURCEI64VSOFT>(
  1727. alGetProcAddress("alGetSourcei64vSOFT"));
  1728. }
  1729. if(alIsExtensionPresent("AL_SOFT_events"))
  1730. {
  1731. std::cout<< "Found AL_SOFT_events" <<std::endl;
  1732. alEventControlSOFT = reinterpret_cast<LPALEVENTCONTROLSOFT>(
  1733. alGetProcAddress("alEventControlSOFT"));
  1734. alEventCallbackSOFT = reinterpret_cast<LPALEVENTCALLBACKSOFT>(
  1735. alGetProcAddress("alEventCallbackSOFT"));
  1736. }
  1737. if(alIsExtensionPresent("AL_SOFT_callback_buffer"))
  1738. {
  1739. std::cout<< "Found AL_SOFT_callback_buffer" <<std::endl;
  1740. alBufferCallbackSOFT = reinterpret_cast<LPALBUFFERCALLBACKSOFT>(
  1741. alGetProcAddress("alBufferCallbackSOFT"));
  1742. }
  1743. size_t fileidx{0};
  1744. for(;fileidx < args.size();++fileidx)
  1745. {
  1746. if(args[fileidx] == "-direct")
  1747. {
  1748. if(alIsExtensionPresent("AL_SOFT_direct_channels_remix"))
  1749. {
  1750. std::cout<< "Found AL_SOFT_direct_channels_remix" <<std::endl;
  1751. DirectOutMode = AL_REMIX_UNMATCHED_SOFT;
  1752. }
  1753. else if(alIsExtensionPresent("AL_SOFT_direct_channels"))
  1754. {
  1755. std::cout<< "Found AL_SOFT_direct_channels" <<std::endl;
  1756. DirectOutMode = AL_DROP_UNMATCHED_SOFT;
  1757. }
  1758. else
  1759. std::cerr<< "AL_SOFT_direct_channels not supported for direct output" <<std::endl;
  1760. }
  1761. else if(args[fileidx] == "-wide")
  1762. {
  1763. if(!alIsExtensionPresent("AL_EXT_STEREO_ANGLES"))
  1764. std::cerr<< "AL_EXT_STEREO_ANGLES not supported for wide stereo" <<std::endl;
  1765. else
  1766. {
  1767. std::cout<< "Found AL_EXT_STEREO_ANGLES" <<std::endl;
  1768. EnableWideStereo = true;
  1769. }
  1770. }
  1771. else if(args[fileidx] == "-uhj")
  1772. {
  1773. if(!alIsExtensionPresent("AL_SOFT_UHJ"))
  1774. std::cerr<< "AL_SOFT_UHJ not supported for UHJ decoding" <<std::endl;
  1775. else
  1776. {
  1777. std::cout<< "Found AL_SOFT_UHJ" <<std::endl;
  1778. EnableUhj = true;
  1779. }
  1780. }
  1781. else if(args[fileidx] == "-superstereo")
  1782. {
  1783. if(!alIsExtensionPresent("AL_SOFT_UHJ"))
  1784. std::cerr<< "AL_SOFT_UHJ not supported for Super Stereo decoding" <<std::endl;
  1785. else
  1786. {
  1787. std::cout<< "Found AL_SOFT_UHJ (Super Stereo)" <<std::endl;
  1788. EnableSuperStereo = true;
  1789. }
  1790. }
  1791. else if(args[fileidx] == "-novideo")
  1792. DisableVideo = true;
  1793. else
  1794. break;
  1795. }
  1796. while(fileidx < args.size() && !movState)
  1797. {
  1798. movState = std::make_unique<MovieState>(args[fileidx++]);
  1799. if(!movState->prepare()) movState = nullptr;
  1800. }
  1801. if(!movState)
  1802. {
  1803. std::cerr<< "Could not start a video" <<std::endl;
  1804. return 1;
  1805. }
  1806. movState->setTitle(screen);
  1807. /* Default to going to the next movie at the end of one. */
  1808. enum class EomAction {
  1809. Next, Quit
  1810. } eom_action{EomAction::Next};
  1811. seconds last_time{seconds::min()};
  1812. while(true)
  1813. {
  1814. /* SDL_WaitEventTimeout is broken, just force a 10ms sleep. */
  1815. std::this_thread::sleep_for(milliseconds{10});
  1816. auto cur_time = std::chrono::duration_cast<seconds>(movState->getMasterClock());
  1817. if(cur_time != last_time)
  1818. {
  1819. auto end_time = std::chrono::duration_cast<seconds>(movState->getDuration());
  1820. std::cout<< " \r "<<PrettyTime{cur_time}<<" / "<<PrettyTime{end_time} <<std::flush;
  1821. last_time = cur_time;
  1822. }
  1823. bool force_redraw{false};
  1824. SDL_Event event{};
  1825. while(SDL_PollEvent(&event) != 0)
  1826. {
  1827. switch(event.type)
  1828. {
  1829. case SDL_KEYDOWN:
  1830. switch(event.key.keysym.sym)
  1831. {
  1832. case SDLK_ESCAPE:
  1833. movState->stop();
  1834. eom_action = EomAction::Quit;
  1835. break;
  1836. case SDLK_n:
  1837. movState->stop();
  1838. eom_action = EomAction::Next;
  1839. break;
  1840. default:
  1841. break;
  1842. }
  1843. break;
  1844. case SDL_WINDOWEVENT:
  1845. switch(event.window.event)
  1846. {
  1847. case SDL_WINDOWEVENT_RESIZED:
  1848. SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
  1849. SDL_RenderFillRect(renderer, nullptr);
  1850. force_redraw = true;
  1851. break;
  1852. case SDL_WINDOWEVENT_EXPOSED:
  1853. force_redraw = true;
  1854. break;
  1855. default:
  1856. break;
  1857. }
  1858. break;
  1859. case SDL_QUIT:
  1860. movState->stop();
  1861. eom_action = EomAction::Quit;
  1862. break;
  1863. case FF_MOVIE_DONE_EVENT:
  1864. std::cout<<'\n';
  1865. last_time = seconds::min();
  1866. if(eom_action != EomAction::Quit)
  1867. {
  1868. movState = nullptr;
  1869. while(fileidx < args.size() && !movState)
  1870. {
  1871. movState = std::make_unique<MovieState>(args[fileidx++]);
  1872. if(!movState->prepare()) movState = nullptr;
  1873. }
  1874. if(movState)
  1875. {
  1876. movState->setTitle(screen);
  1877. break;
  1878. }
  1879. }
  1880. /* Nothing more to play. Shut everything down and quit. */
  1881. movState = nullptr;
  1882. CloseAL();
  1883. SDL_DestroyRenderer(renderer);
  1884. renderer = nullptr;
  1885. SDL_DestroyWindow(screen);
  1886. screen = nullptr;
  1887. SDL_Quit();
  1888. exit(0);
  1889. default:
  1890. break;
  1891. }
  1892. }
  1893. movState->mVideo.updateVideo(screen, renderer, force_redraw);
  1894. }
  1895. std::cerr<< "SDL_WaitEvent error - "<<SDL_GetError() <<std::endl;
  1896. return 1;
  1897. }
  1898. } // namespace
  1899. int main(int argc, char *argv[])
  1900. {
  1901. assert(argc >= 0);
  1902. auto args = std::vector<std::string_view>(static_cast<unsigned int>(argc));
  1903. std::copy_n(argv, args.size(), args.begin());
  1904. return main(al::span{args});
  1905. }