diff --git a/README.md b/README.md index 74740413..ef2795ad 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,23 @@ mpegts.js works by transmuxing MPEG2-TS stream into ISO BMFF (Fragmented MP4) se [Media Source Extensions]: https://w3c.github.io/media-source/ ## News -- v1.7.3 +- **v1.8.0** + + Support working on **iOS Safari** with iOS 17.1+ through Apple's [ManagedMediaSource API](https://github.com/w3c/media-source/issues/320) + + Great performance improvements by supporting [MSE in Workers](https://github.com/w3c/media-source/issues/175) on Chrome, Safari 18 (includes iOS) + + Introduced support for [AV1 over MPEG-TS](https://aomediacodec.github.io/av1-mpeg2-ts/) + + Introduced support for AV1 over HTTP-FLV defined in [Enhanced RTMP](https://github.com/veovera/enhanced-rtmp) + + Support chasing live latency more smoothly by changing playback rate + + Introduced ATSC EAC-3 audio codec in MPEG-TS + + Support Opus and FLAC audio codec over HTTP-FLV (Enhanced RTMP) + +- **v1.7.3** Introduced [Enhanced RTMP] with HEVC support for FLV. @@ -22,7 +38,7 @@ mpegts.js works by transmuxing MPEG2-TS stream into ISO BMFF (Fragmented MP4) se Introduced LOAS AAC support for MPEG-TS. -- v1.7.0 +- **v1.7.0** Introduced H.265/HEVC over MPEG-TS/FLV support. @@ -87,8 +103,6 @@ mpegts.js could be tested with [Simple Realtime Server](https://github.com/ossrs ## TODO - MPEG2-TS static file playback (seeking is not supported now) -- MP3/AC3 audio codec support -- AV1/OPUS codec over MPEG2-TS stream support (?) ## Limitations - mpeg2video is not supported diff --git a/README_ja.md b/README_ja.md index 1b9154c4..720b5219 100644 --- a/README_ja.md +++ b/README_ja.md @@ -12,7 +12,34 @@ mpegts.js は、JavaScript で MPEG2-TS ストリームを解析しながら、 [Media Source Extensions]: https://w3c.github.io/media-source/ ## News -H.265/HEVC 再生支援（over FLV/MPEG-TS）は v1.7.0 から導入されています。 +- **v1.8.0** + + **iOS Safari**（iOS 17.1+）での動作をサポートし、Apple の [ManagedMediaSource API](https://github.com/w3c/media-source/issues/320) を使用 + + [MSE in Workers](https://github.com/w3c/media-source/issues/175) 利用により、パフォーマンスが大幅に向上：Chrome と Safari 18（iOS 含む）対応 + + [AV1 over MPEG-TS](https://aomediacodec.github.io/av1-mpeg2-ts/) をサポート + + [Enhanced RTMP](https://github.com/veovera/enhanced-rtmp) の定義による AV1 over HTTP-FLV をサポート + + 再生速度を動的に変更することで、ライブ遅延の追従をよりスムーズにできる + + MPEG-TS に ATSC EAC-3 音声サポートを追加 + + HTTP-FLV（Enhanced RTMP）で Opus および FLAC 音声サポートを追加 + +- **v1.7.3** + + [Enhanced RTMP](https://github.com/veovera/enhanced-rtmp) における FLV での HEVC 転送の仕様をサポート + + MPEG-TS に Opus および ATSC AC-3 音声コーデックのサポートを追加 + + MPEG-TS に LOAS AAC 再生のサポートを追加 + +- **v1.7.0** + + H.265/HEVC 再生のサポートを追加（FLV および MPEG-TS の両方で対応） + ## Demo [http://xqq.github.io/mpegts.js/demo/](http://xqq.github.io/mpegts.js/demo/) @@ -71,8 +98,6 @@ npm run build # packaged & minimized js will be emitted in dist fo ## TODO - 静的 MPEG2-TS ファイルの再生 （現時点ではシークできません） -- MP3/AC3 audio codec の支援 -- AV1/OPUS codec over MPEG2-TS stream support (?) ## Limitations - mpeg2video はサポートしていません。映像は H.264 であることが求められます diff --git a/README_zh.md b/README_zh.md index fdd3bd7f..856a3701 100644 --- a/README_zh.md +++ b/README_zh.md @@ -12,7 +12,23 @@ mpegts.js 通过在 JavaScript 中渐进化解析 MPEG2-TS 流并实时转封装 [Media Source Extensions]: https://w3c.github.io/media-source/ ## News -- v1.7.3 +- **v1.8.0** + + 支持在 **iOS Safari**（iOS 17.1+）上运行，使用 Apple [ManagedMediaSource API](https://github.com/w3c/media-source/issues/320) + + 通过使用 [MSE in Workers](https://github.com/w3c/media-source/issues/175) 来显著提升性能（适用于 Chrome 和 Safari 18，包括 iOS） + + 支持 [AV1 over MPEG-TS](https://aomediacodec.github.io/av1-mpeg2-ts/) + + 支持 AV1 over HTTP-FLV，定义于 [Enhanced RTMP](https://github.com/veovera/enhanced-rtmp) + + 支持更平滑的直播延迟追赶（通过动态改变播放速率） + + MPEG-TS 新增 ATSC EAC-3 音频支持 + + HTTP-FLV（Enhanced RTMP）新增 Opus 和 FLAC 音频支持 + +- **v1.7.3** 支持 [Enhanced RTMP] 中关于 FLV 传输 HEVC 的规范 @@ -20,7 +36,7 @@ mpegts.js 通过在 JavaScript 中渐进化解析 MPEG2-TS 流并实时转封装 MPEG-TS 新增了 LOAS AAC 播放支持 -- v1.7.0 +- **v1.7.0** H.265/HEVC 播放支持（FLV 或 MPEG-TS 均已支持） @@ -86,8 +102,6 @@ npm run build # packaged & minimized js will be emitted in dist fo ## TODO - MPEG2-TS 静态文件回放 （目前还不支持 seek） -- MP3/AC3 音频编码支持 -- AV1/OPUS codec over MPEG2-TS stream support (?) ## Limitations - 不支持 mpeg2video diff --git a/d.ts/mpegts.d.ts b/d.ts/mpegts.d.ts index 431be109..7d009bba 100644 --- a/d.ts/mpegts.d.ts +++ b/d.ts/mpegts.d.ts @@ -420,6 +420,7 @@ declare namespace Mpegts { METADATA_ARRIVED: string; SCRIPTDATA_ARRIVED: string; TIMED_ID3_METADATA_ARRIVED: string; + PGS_SUBTITLE_ARRIVED: string; SYNCHRONOUS_KLV_METADATA_ARRIVED: string; ASYNCHRONOUS_KLV_METADATA_ARRIVED: string; SMPTE2038_METADATA_ARRIVED: string; diff --git a/d.ts/src/core/transmuxing-events.d.ts b/d.ts/src/core/transmuxing-events.d.ts index 82b2f678..b9561486 100644 --- a/d.ts/src/core/transmuxing-events.d.ts +++ b/d.ts/src/core/transmuxing-events.d.ts @@ -9,6 +9,7 @@ declare enum TransmuxingEvents { METADATA_ARRIVED = "metadata_arrived", SCRIPTDATA_ARRIVED = "scriptdata_arrived", TIMED_ID3_METADATA_ARRIVED = "timed_id3_metadata_arrived", + PGS_SUBTITLE_ARRIVED = "pgs_subtitle_arrived", SYNCHRONOUS_KLV_METADATA_ARRIVED = "synchronous_klv_metadata_arrived", ASYNCHRONOUS_KLV_METADATA_ARRIVED = "asynchronous_klv_metadata_arrived", SMPTE2038_METADATA_ARRIVED = "smpte2038_metadata_arrived", diff --git a/d.ts/src/demux/base-demuxer.d.ts b/d.ts/src/demux/base-demuxer.d.ts index 1b97eed8..d46476f4 100644 --- a/d.ts/src/demux/base-demuxer.d.ts +++ b/d.ts/src/demux/base-demuxer.d.ts @@ -3,12 +3,14 @@ import { PESPrivateData, PESPrivateDataDescriptor } from './pes-private-data'; import { SMPTE2038Data } from './smpte2038'; import { SCTE35Data } from './scte35'; import { KLVData } from './klv'; +import { PGSData } from './pgs-data'; type OnErrorCallback = (type: string, info: string) => void; type OnMediaInfoCallback = (mediaInfo: MediaInfo) => void; type OnMetaDataArrivedCallback = (metadata: any) => void; type OnTrackMetadataCallback = (type: string, metadata: any) => void; type OnDataAvailableCallback = (audioTrack: any, videoTrack: any) => void; type OnTimedID3MetadataCallback = (timed_id3_data: PESPrivateData) => void; +type onPGSSubitleDataCallback = (pgs_data: PGSData) => void; type OnSynchronousKLVMetadataCallback = (synchronous_klv_data: KLVData) => void; type OnAsynchronousKLVMetadataCallback = (asynchronous_klv_data: PESPrivateData) => void; type OnSMPTE2038MetadataCallback = (smpte2038_data: SMPTE2038Data) => void; @@ -22,6 +24,7 @@ export default abstract class BaseDemuxer { onTrackMetadata: OnTrackMetadataCallback; onDataAvailable: OnDataAvailableCallback; onTimedID3Metadata: OnTimedID3MetadataCallback; + onPGSSubtitleData: onPGSSubitleDataCallback; onSynchronousKLVMetadata: OnSynchronousKLVMetadataCallback; onAsynchronousKLVMetadata: OnAsynchronousKLVMetadataCallback; onSMPTE2038Metadata: OnSMPTE2038MetadataCallback; diff --git a/d.ts/src/demux/pat-pmt-pes.d.ts b/d.ts/src/demux/pat-pmt-pes.d.ts index 5e9ef412..64264464 100644 --- a/d.ts/src/demux/pat-pmt-pes.d.ts +++ b/d.ts/src/demux/pat-pmt-pes.d.ts @@ -16,6 +16,7 @@ export declare enum StreamType { kEAC3 = 135, kMetadata = 21, kSCTE35 = 134, + kPGS = 144, kH264 = 27, kH265 = 36 } @@ -44,6 +45,12 @@ export declare class PMT { timed_id3_pids: { [pid: number]: boolean; }; + pgs_pids: { + [pid: number]: boolean; + }; + pgs_langs: { + [pid: number]: string; + }; synchronous_klv_pids: { [pid: number]: boolean; }; diff --git a/d.ts/src/demux/pgs-data.d.ts b/d.ts/src/demux/pgs-data.d.ts new file mode 100644 index 00000000..b27f7512 --- /dev/null +++ b/d.ts/src/demux/pgs-data.d.ts @@ -0,0 +1,9 @@ +export declare class PGSData { + pid: number; + stream_id: number; + pts?: number; + dts?: number; + lang: string; + data: Uint8Array; + len: number; +} diff --git a/d.ts/src/demux/ts-demuxer.d.ts b/d.ts/src/demux/ts-demuxer.d.ts index c419c45c..b61c4d6c 100644 --- a/d.ts/src/demux/ts-demuxer.d.ts +++ b/d.ts/src/demux/ts-demuxer.d.ts @@ -85,6 +85,7 @@ declare class TSDemuxer extends BaseDemuxer { private dispatchPESPrivateDataDescriptor; private parsePESPrivateDataPayload; private parseTimedID3MetadataPayload; + private parsePGSPayload; private parseSynchronousKLVMetadataPayload; private parseAsynchronousKLVMetadataPayload; private parseSMPTE2038MetadataPayload; diff --git a/d.ts/src/player/player-engine-worker-msg-def.d.ts b/d.ts/src/player/player-engine-worker-msg-def.d.ts index 4a1bc58c..839c3695 100644 --- a/d.ts/src/player/player-engine-worker-msg-def.d.ts +++ b/d.ts/src/player/player-engine-worker-msg-def.d.ts @@ -26,7 +26,7 @@ export type WorkerMessagePacketPlayerEventError = WorkerMessagePacketPlayerEvent }; export type WorkerMessagePacketPlayerEventExtraData = WorkerMessagePacketPlayerEvent & { msg: 'player_event'; - event: PlayerEvents.METADATA_ARRIVED | PlayerEvents.SCRIPTDATA_ARRIVED | PlayerEvents.TIMED_ID3_METADATA_ARRIVED | PlayerEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED | PlayerEvents.ASYNCHRONOUS_KLV_METADATA_ARRIVED | PlayerEvents.SMPTE2038_METADATA_ARRIVED | PlayerEvents.SCTE35_METADATA_ARRIVED | PlayerEvents.PES_PRIVATE_DATA_DESCRIPTOR | PlayerEvents.PES_PRIVATE_DATA_ARRIVED; + event: PlayerEvents.METADATA_ARRIVED | PlayerEvents.SCRIPTDATA_ARRIVED | PlayerEvents.TIMED_ID3_METADATA_ARRIVED | PlayerEvents.PGS_SUBTITLE_ARRIVED | PlayerEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED | PlayerEvents.ASYNCHRONOUS_KLV_METADATA_ARRIVED | PlayerEvents.SMPTE2038_METADATA_ARRIVED | PlayerEvents.SCTE35_METADATA_ARRIVED | PlayerEvents.PES_PRIVATE_DATA_DESCRIPTOR | PlayerEvents.PES_PRIVATE_DATA_ARRIVED; extraData: any; }; export type WorkerMessagePacketTransmuxingEvent = WorkerMessagePacket & { diff --git a/d.ts/src/player/player-events.d.ts b/d.ts/src/player/player-events.d.ts index 704ec83f..004e42ec 100644 --- a/d.ts/src/player/player-events.d.ts +++ b/d.ts/src/player/player-events.d.ts @@ -6,6 +6,7 @@ declare enum PlayerEvents { METADATA_ARRIVED = "metadata_arrived", SCRIPTDATA_ARRIVED = "scriptdata_arrived", TIMED_ID3_METADATA_ARRIVED = "timed_id3_metadata_arrived", + PGS_SUBTITLE_ARRIVED = "pgs_subtitle_arrived", SYNCHRONOUS_KLV_METADATA_ARRIVED = "synchronous_klv_metadata_arrived", ASYNCHRONOUS_KLV_METADATA_ARRIVED = "asynchronous_klv_metadata_arrived", SMPTE2038_METADATA_ARRIVED = "smpte2038_metadata_arrived", diff --git a/docs/api.md b/docs/api.md index 41db38f2..ceaa619d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -203,6 +203,7 @@ A series of constants that can be used with `Player.on()` / `Player.off()`. They | METADATA_ARRIVED | Provides metadata which FLV file(stream) can contain with an "onMetaData" marker. | | SCRIPTDATA_ARRIVED | Provides scriptdata (OnCuePoint / OnTextData) which FLV file(stream) can contain. | | TIMED_ID3_METADATA_ARRIVED | Provides Timed ID3 Metadata packets containing private data (stream_type=0x15) callback | +| PGS_SUBTITLE_ARRIVED | Provides PGS Subtitle data (stream_type=0x90) callback | | SYNCHRONOUS_KLV_METADATA_ARRIVED | Provides Synchronous KLV Metadata packets containing private data (stream_type=0x15) callback | | ASYNCHRONOUS_KLV_METADATA_ARRIVED | Provides Asynchronous KLV Metadata packets containing private data (stream_type=0x06) callback | | SMPTE2038_METADATA_ARRIVED | Provides SMPTE2038 Metadata packets containing private data callback | diff --git a/package-lock.json b/package-lock.json index c2ace693..08dd0b67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,8 +9,7 @@ "version": "1.8.0", "license": "Apache-2.0", "dependencies": { - "es6-promise": "^4.2.5", - "webworkify-webpack": "xqq/webworkify-webpack" + "es6-promise": "^4.2.5" }, "devDependencies": { "@types/express-serve-static-core": "4.17.37", @@ -8836,11 +8835,6 @@ "node": ">=0.8.0" } }, - "node_modules/webworkify-webpack": { - "version": "2.1.5", - "resolved": "git+ssh://git@github.com/xqq/webworkify-webpack.git#24d1e719b4a6cac37a518b2bb10fe124527ef4ef", - "license": "MIT" - }, "node_modules/which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -16090,10 +16084,6 @@ "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", "dev": true }, - "webworkify-webpack": { - "version": "git+ssh://git@github.com/xqq/webworkify-webpack.git#24d1e719b4a6cac37a518b2bb10fe124527ef4ef", - "from": "webworkify-webpack@xqq/webworkify-webpack" - }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/package.json b/package.json index c7f09592..71f541ac 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,7 @@ "serve": "webpack serve --mode=development --progress" }, "dependencies": { - "es6-promise": "^4.2.5", - "webworkify-webpack": "xqq/webworkify-webpack" + "es6-promise": "^4.2.5" }, "devDependencies": { "@types/node": "^10.12.18", diff --git a/src/core/transmuxer.js b/src/core/transmuxer.js index f313cfaa..1128dff9 100644 --- a/src/core/transmuxer.js +++ b/src/core/transmuxer.js @@ -17,7 +17,7 @@ */ import EventEmitter from 'events'; -import work from 'webworkify-webpack'; +import work from '../utils/webworkify-webpack'; import Log from '../utils/logger.js'; import LoggingControl from '../utils/logging-control.js'; import TransmuxingController from './transmuxing-controller.js'; @@ -188,7 +188,13 @@ class Transmuxer { _onTimedID3MetadataArrived (data) { Promise.resolve().then(() => { this._emitter.emit(TransmuxingEvents.TIMED_ID3_METADATA_ARRIVED, data); - }) + }); + } + + _onPGSSubtitleArrived (data) { + Promise.resolve().then(() => { + this._emitter.emit(TransmuxingEvents.PGS_SUBTITLE_ARRIVED, data); + }); } _onSynchronousKLVMetadataArrived (data) { @@ -284,6 +290,7 @@ class Transmuxer { case TransmuxingEvents.METADATA_ARRIVED: case TransmuxingEvents.SCRIPTDATA_ARRIVED: case TransmuxingEvents.TIMED_ID3_METADATA_ARRIVED: + case TransmuxingEvents.PGS_SUBTITLE_ARRIVED: case TransmuxingEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED: case TransmuxingEvents.ASYNCHRONOUS_KLV_METADATA_ARRIVED: case TransmuxingEvents.SMPTE2038_METADATA_ARRIVED: diff --git a/src/core/transmuxing-controller.js b/src/core/transmuxing-controller.js index f1b6bcc0..b84cebb3 100644 --- a/src/core/transmuxing-controller.js +++ b/src/core/transmuxing-controller.js @@ -323,6 +323,7 @@ class TransmuxingController { demuxer.onMediaInfo = this._onMediaInfo.bind(this); demuxer.onMetaDataArrived = this._onMetaDataArrived.bind(this); demuxer.onTimedID3Metadata = this._onTimedID3Metadata.bind(this); + demuxer.onPGSSubtitleData = this._onPGSSubtitle.bind(this); demuxer.onSynchronousKLVMetadata = this._onSynchronousKLVMetadata.bind(this); demuxer.onAsynchronousKLVMetadata = this._onAsynchronousKLVMetadata.bind(this); demuxer.onSMPTE2038Metadata = this._onSMPTE2038Metadata.bind(this); @@ -386,6 +387,21 @@ class TransmuxingController { this._emitter.emit(TransmuxingEvents.TIMED_ID3_METADATA_ARRIVED, timed_id3_metadata); } + _onPGSSubtitle(pgs_data) { + let timestamp_base = this._remuxer.getTimestampBase(); + if (timestamp_base == undefined) { return; } + + if (pgs_data.pts != undefined) { + pgs_data.pts -= timestamp_base; + } + + if (pgs_data.dts != undefined) { + pgs_data.dts -= timestamp_base; + } + + this._emitter.emit(TransmuxingEvents.PGS_SUBTITLE_ARRIVED, pgs_data); + } + _onSynchronousKLVMetadata(synchronous_klv_metadata) { let timestamp_base = this._remuxer.getTimestampBase(); if (timestamp_base == undefined) { return; } diff --git a/src/core/transmuxing-events.ts b/src/core/transmuxing-events.ts index 6d2fc04e..c8beaeae 100644 --- a/src/core/transmuxing-events.ts +++ b/src/core/transmuxing-events.ts @@ -27,6 +27,7 @@ enum TransmuxingEvents { METADATA_ARRIVED = 'metadata_arrived', SCRIPTDATA_ARRIVED = 'scriptdata_arrived', TIMED_ID3_METADATA_ARRIVED = 'timed_id3_metadata_arrived', + PGS_SUBTITLE_ARRIVED = 'pgs_subtitle_arrived', SYNCHRONOUS_KLV_METADATA_ARRIVED = 'synchronous_klv_metadata_arrived', ASYNCHRONOUS_KLV_METADATA_ARRIVED = 'asynchronous_klv_metadata_arrived', SMPTE2038_METADATA_ARRIVED = 'smpte2038_metadata_arrived', diff --git a/src/core/transmuxing-worker.js b/src/core/transmuxing-worker.js index 525cc8a9..31ef968f 100644 --- a/src/core/transmuxing-worker.js +++ b/src/core/transmuxing-worker.js @@ -57,6 +57,7 @@ let TransmuxingWorker = function (self) { controller.on(TransmuxingEvents.METADATA_ARRIVED, onMetaDataArrived.bind(this)); controller.on(TransmuxingEvents.SCRIPTDATA_ARRIVED, onScriptDataArrived.bind(this)); controller.on(TransmuxingEvents.TIMED_ID3_METADATA_ARRIVED, onTimedID3MetadataArrived.bind(this)); + controller.on(TransmuxingEvents.PGS_SUBTITLE_ARRIVED, onPGSSubtitleDataArrived.bind(this)); controller.on(TransmuxingEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED, onSynchronousKLVMetadataArrived.bind(this)); controller.on(TransmuxingEvents.ASYNCHRONOUS_KLV_METADATA_ARRIVED, onAsynchronousKLVMetadataArrived.bind(this)); controller.on(TransmuxingEvents.SMPTE2038_METADATA_ARRIVED, onSMPTE2038MetadataArrived.bind(this)); @@ -170,6 +171,14 @@ let TransmuxingWorker = function (self) { self.postMessage(obj); } + function onPGSSubtitleDataArrived (data) { + let obj = { + msg: TransmuxingEvents.PGS_SUBTITLE_ARRIVED, + data: data + }; + self.postMessage(obj); + } + function onSynchronousKLVMetadataArrived (data) { let obj = { msg: TransmuxingEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED, diff --git a/src/demux/base-demuxer.ts b/src/demux/base-demuxer.ts index b5fa0935..f372fbbf 100644 --- a/src/demux/base-demuxer.ts +++ b/src/demux/base-demuxer.ts @@ -3,6 +3,7 @@ import { PESPrivateData, PESPrivateDataDescriptor } from './pes-private-data'; import { SMPTE2038Data } from './smpte2038'; import { SCTE35Data } from './scte35'; import { KLVData } from './klv'; +import { PGSData } from './pgs-data'; type OnErrorCallback = (type: string, info: string) => void; type OnMediaInfoCallback = (mediaInfo: MediaInfo) => void; @@ -10,6 +11,7 @@ type OnMetaDataArrivedCallback = (metadata: any) => void; type OnTrackMetadataCallback = (type: string, metadata: any) => void; type OnDataAvailableCallback = (audioTrack: any, videoTrack: any) => void; type OnTimedID3MetadataCallback = (timed_id3_data: PESPrivateData) => void; +type onPGSSubitleDataCallback = (pgs_data: PGSData) => void; type OnSynchronousKLVMetadataCallback = (synchronous_klv_data: KLVData) => void; type OnAsynchronousKLVMetadataCallback = (asynchronous_klv_data: PESPrivateData) => void; type OnSMPTE2038MetadataCallback = (smpte2038_data: SMPTE2038Data) => void; @@ -25,6 +27,7 @@ export default abstract class BaseDemuxer { public onTrackMetadata: OnTrackMetadataCallback; public onDataAvailable: OnDataAvailableCallback; public onTimedID3Metadata: OnTimedID3MetadataCallback; + public onPGSSubtitleData: onPGSSubitleDataCallback; public onSynchronousKLVMetadata: OnSynchronousKLVMetadataCallback public onAsynchronousKLVMetadata: OnAsynchronousKLVMetadataCallback; public onSMPTE2038Metadata: OnSMPTE2038MetadataCallback; @@ -41,6 +44,7 @@ export default abstract class BaseDemuxer { this.onTrackMetadata = null; this.onDataAvailable = null; this.onTimedID3Metadata = null; + this.onPGSSubtitleData = null; this.onSynchronousKLVMetadata = null; this.onAsynchronousKLVMetadata = null; this.onSMPTE2038Metadata = null; diff --git a/src/demux/flv-demuxer.js b/src/demux/flv-demuxer.js index 93ed1ae3..f3681e7d 100644 --- a/src/demux/flv-demuxer.js +++ b/src/demux/flv-demuxer.js @@ -516,7 +516,7 @@ class FLVDemuxer { } // Legacy FLV - if (soundFormat !== 2 && soundFormat !== 10) { // MP3 or AAC + if (soundFormat !== 2 && soundFormat !== 3 && soundFormat !== 10) { // PCM or MP3 or AAC this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec idx: ' + soundFormat); return; } @@ -533,7 +533,6 @@ class FLVDemuxer { let soundSize = (soundSpec & 2) >>> 1; // unused let soundType = (soundSpec & 1); - let meta = this._audioMetadata; let track = this._audioTrack; @@ -656,6 +655,39 @@ class FLVDemuxer { let mp3Sample = {unit: data, length: data.byteLength, dts: dts, pts: dts}; track.samples.push(mp3Sample); track.length += data.length; + } else if (soundFormat === 3) { + if (!meta.codec) { + meta.audioSampleRate = soundRate; + meta.sampleSize = (soundSize + 1) * 8; + meta.littleEndian = true; + meta.codec = 'ipcm'; + meta.originalCodec = 'ipcm'; + + this._audioInitialMetadataDispatched = true; + this._onTrackMetadata('audio', meta); + + let mi = this._mediaInfo; + mi.audioCodec = meta.codec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + mi.audioDataRate = meta.sampleSize * meta.audioSampleRate; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; + } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + } + if (mi.isComplete()) { + this._onMediaInfo(mi); + } + } + + let data = new Uint8Array(arrayBuffer, dataOffset + 1, dataSize - 1); + let dts = this._timestampBase + tagTimestamp; + let pcmSample = {unit: data, length: data.byteLength, dts: dts, pts: dts}; + track.samples.push(pcmSample); + track.length += data.length; } } diff --git a/src/demux/pat-pmt-pes.ts b/src/demux/pat-pmt-pes.ts index c4de2b0a..5777cb3b 100644 --- a/src/demux/pat-pmt-pes.ts +++ b/src/demux/pat-pmt-pes.ts @@ -19,6 +19,7 @@ export enum StreamType { kEAC3 = 0x87, kMetadata = 0x15, kSCTE35 = 0x86, + kPGS = 0x90, kH264 = 0x1b, kH265 = 0x24 } @@ -64,6 +65,13 @@ export class PMT { [pid: number]: boolean } = {}; + pgs_pids: { + [pid: number]: boolean; + } = {}; + pgs_langs: { + [pid: number]: string; + } = {}; + synchronous_klv_pids: { [pid: number]: boolean } = {}; diff --git a/src/demux/pgs-data.ts b/src/demux/pgs-data.ts new file mode 100644 index 00000000..8d37ce70 --- /dev/null +++ b/src/demux/pgs-data.ts @@ -0,0 +1,11 @@ +// ISO/IEC 13818-1 PES packets containing private data (stream_type=0x06) +export class PGSData { + pid: number; + stream_id: number; + pts?: number; + dts?: number; + lang: string; + data: Uint8Array; + len: number; +} + diff --git a/src/demux/ts-demuxer.ts b/src/demux/ts-demuxer.ts index 25503de6..43c95d94 100644 --- a/src/demux/ts-demuxer.ts +++ b/src/demux/ts-demuxer.ts @@ -36,6 +36,7 @@ import { AC3Config, AC3Frame, AC3Parser, EAC3Config, EAC3Frame, EAC3Parser } fro import { KLVData, klv_parse } from './klv'; import AV1OBUInMpegTsParser from './av1'; import AV1OBUParser from './av1-parser'; +import { PGSData } from './pgs-data'; type AdaptationFieldInfo = { discontinuity_indicator?: number; @@ -353,6 +354,7 @@ class TSDemuxer extends BaseDemuxer { || pid === this.pmt_.common_pids.mp3 || this.pmt_.pes_private_data_pids[pid] === true || this.pmt_.timed_id3_pids[pid] === true + || this.pmt_.pgs_pids[pid] === true || this.pmt_.synchronous_klv_pids[pid] === true || this.pmt_.asynchronous_klv_pids[pid] === true ) { @@ -637,6 +639,9 @@ class TSDemuxer extends BaseDemuxer { this.parseSynchronousKLVMetadataPayload(payload, pts, dts, pes_data.pid, stream_id); } break; + case StreamType.kPGS: + this.parsePGSPayload(payload, pts, dts, pes_data.pid, stream_id, this.pmt_.pgs_langs[pes_data.pid]); + break; case StreamType.kH264: this.parseH264Payload(payload, pts, dts, pes_data.file_position, pes_data.random_access_indicator); break; @@ -894,6 +899,21 @@ class TSDemuxer extends BaseDemuxer { } } else if (stream_type === StreamType.kSCTE35) { pmt.scte_35_pids[elementary_PID] = true; + } else if (stream_type === StreamType.kPGS) { + pmt.pgs_langs[elementary_PID] = 'und'; + if (ES_info_length > 0) { + // parse descriptor + for (let offset = i + 5; offset < i + 5 + ES_info_length; ) { + let tag = data[offset + 0]; + let length = data[offset + 1]; + if (tag === 0x0a) { // ISO_639_LANGUAGE_DESCRIPTOR + const lang = String.fromCharCode(... Array.from(data.slice(offset + 2, offset + 5))); + pmt.pgs_langs[elementary_PID] = lang; + } + offset += 2 + length; + } + } + pmt.pgs_pids[elementary_PID] = true; } i += 5 + ES_info_length; @@ -2004,6 +2024,30 @@ class TSDemuxer extends BaseDemuxer { } } + private parsePGSPayload(data: Uint8Array, pts: number, dts: number, pid: number, stream_id: number, lang: string) { + let pgs_data = new PGSData(); + + pgs_data.pid = pid; + pgs_data.lang = lang; + pgs_data.stream_id = stream_id; + pgs_data.len = data.byteLength; + pgs_data.data = data; + + if (pts != undefined) { + let pts_ms = Math.floor(pts / this.timescale_); + pgs_data.pts = pts_ms; + } + + if (dts != undefined) { + let dts_ms = Math.floor(dts / this.timescale_); + pgs_data.dts = dts_ms; + } + + if (this.onPGSSubtitleData) { + this.onPGSSubtitleData(pgs_data); + } + } + private parseSynchronousKLVMetadataPayload(data: Uint8Array, pts: number, dts: number, pid: number, stream_id: number) { let synchronous_klv_metadata = new KLVData(); diff --git a/src/player/player-engine-dedicated-thread.ts b/src/player/player-engine-dedicated-thread.ts index 377deadd..297652b5 100644 --- a/src/player/player-engine-dedicated-thread.ts +++ b/src/player/player-engine-dedicated-thread.ts @@ -17,7 +17,7 @@ */ import * as EventEmitter from 'events'; -import * as work from 'webworkify-webpack'; +import * as work from '../utils/webworkify-webpack'; import type PlayerEngine from './player-engine'; import Log from '../utils/logger'; import LoggingControl from '../utils/logging-control.js'; diff --git a/src/player/player-engine-main-thread.ts b/src/player/player-engine-main-thread.ts index 91e39bd8..bbfd1197 100644 --- a/src/player/player-engine-main-thread.ts +++ b/src/player/player-engine-main-thread.ts @@ -231,6 +231,9 @@ class PlayerEngineMainThread implements PlayerEngine { this._transmuxer.on(TransmuxingEvents.TIMED_ID3_METADATA_ARRIVED, (timed_id3_metadata: any) => { this._emitter.emit(PlayerEvents.TIMED_ID3_METADATA_ARRIVED, timed_id3_metadata); }); + this._transmuxer.on(TransmuxingEvents.PGS_SUBTITLE_ARRIVED, (pgs_data: any) => { + this._emitter.emit(PlayerEvents.PGS_SUBTITLE_ARRIVED, pgs_data); + }); this._transmuxer.on(TransmuxingEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED, (synchronous_klv_metadata: any) => { this._emitter.emit(PlayerEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED, synchronous_klv_metadata); }); diff --git a/src/player/player-engine-worker-msg-def.ts b/src/player/player-engine-worker-msg-def.ts index 714d6e3f..f13c445c 100644 --- a/src/player/player-engine-worker-msg-def.ts +++ b/src/player/player-engine-worker-msg-def.ts @@ -62,6 +62,7 @@ export type WorkerMessagePacketPlayerEventExtraData = WorkerMessagePacketPlayerE | PlayerEvents.METADATA_ARRIVED | PlayerEvents.SCRIPTDATA_ARRIVED | PlayerEvents.TIMED_ID3_METADATA_ARRIVED + | PlayerEvents.PGS_SUBTITLE_ARRIVED | PlayerEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED | PlayerEvents.ASYNCHRONOUS_KLV_METADATA_ARRIVED | PlayerEvents.SMPTE2038_METADATA_ARRIVED diff --git a/src/player/player-engine-worker.ts b/src/player/player-engine-worker.ts index 9e4da036..127a2bac 100644 --- a/src/player/player-engine-worker.ts +++ b/src/player/player-engine-worker.ts @@ -252,6 +252,9 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { transmuxer.on(TransmuxingEvents.TIMED_ID3_METADATA_ARRIVED, (timed_id3_metadata: any) => { emitPlayerEventsExtraData(PlayerEvents.TIMED_ID3_METADATA_ARRIVED, timed_id3_metadata); }); + transmuxer.on(TransmuxingEvents.PGS_SUBTITLE_ARRIVED, (pgs_data: any) => { + emitPlayerEventsExtraData(PlayerEvents.PGS_SUBTITLE_ARRIVED, pgs_data); + }); transmuxer.on(TransmuxingEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED, (synchronous_klv_metadata: any) => { emitPlayerEventsExtraData(PlayerEvents.SYNCHRONOUS_KLV_METADATA_ARRIVED, synchronous_klv_metadata); }); diff --git a/src/player/player-events.ts b/src/player/player-events.ts index 6766a096..ec608a7d 100644 --- a/src/player/player-events.ts +++ b/src/player/player-events.ts @@ -24,6 +24,7 @@ enum PlayerEvents { METADATA_ARRIVED = 'metadata_arrived', SCRIPTDATA_ARRIVED = 'scriptdata_arrived', TIMED_ID3_METADATA_ARRIVED = 'timed_id3_metadata_arrived', + PGS_SUBTITLE_ARRIVED = 'pgs_subtitle_arrived', SYNCHRONOUS_KLV_METADATA_ARRIVED = 'synchronous_klv_metadata_arrived', ASYNCHRONOUS_KLV_METADATA_ARRIVED = 'asynchronous_klv_metadata_arrived', SMPTE2038_METADATA_ARRIVED = 'smpte2038_metadata_arrived', diff --git a/src/remux/mp4-generator.js b/src/remux/mp4-generator.js index 7ccfad43..fa608a73 100644 --- a/src/remux/mp4-generator.js +++ b/src/remux/mp4-generator.js @@ -31,9 +31,11 @@ class MP4 { stco: [], stsc: [], stsd: [], stsz: [], stts: [], tfdt: [], tfhd: [], traf: [], trak: [], trun: [], trex: [], tkhd: [], - vmhd: [], smhd: [], '.mp3': [], - Opus: [], dOps: [], 'ac-3': [], dac3: [], 'ec-3': [], dec3: [], - fLaC: [], dfLa: [], + vmhd: [], smhd: [], chnl: [], + '.mp3': [], + Opus: [], dOps: [], fLaC: [], dfLa: [], + ipcm: [], pcmC: [], + 'ac-3': [], dac3: [], 'ec-3': [], dec3: [], }; for (let name in MP4.types) { @@ -331,6 +333,8 @@ class MP4 { return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.Opus(meta)); } else if (meta.codec == 'flac') { return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.fLaC(meta)); + } else if (meta.codec == 'ipcm') { + return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.ipcm(meta)); } // else: aac -> mp4a return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.mp4a(meta)); @@ -578,6 +582,51 @@ class MP4 { return MP4.box(MP4.types.dfLa, data); } + static ipcm(meta) { + let channelCount = meta.channelCount; + let sampleRate = Math.min(meta.audioSampleRate, 65535); + let sampleSize = meta.sampleSize; + + let data = new Uint8Array([ + 0x00, 0x00, 0x00, 0x00, // reserved(4) + 0x00, 0x00, 0x00, 0x01, // reserved(2) + data_reference_index(2) + 0x00, 0x00, 0x00, 0x00, // reserved: 2 * 4 bytes + 0x00, 0x00, 0x00, 0x00, + 0x00, channelCount, // channelCount(2) + 0x00, (sampleSize), // sampleSize(2) + 0x00, 0x00, 0x00, 0x00, // reserved(4) + (sampleRate >>> 8) & 0xFF, // Audio sample rate + (sampleRate) & 0xFF, + 0x00, 0x00 + ]); + + if (meta.channelCount === 1) { + return MP4.box(MP4.types.ipcm, data, MP4.pcmC(meta)); + } else { + return MP4.box(MP4.types.ipcm, data, MP4.chnl(meta), MP4.pcmC(meta)); + } + } + + static chnl(meta) { + let data = new Uint8Array([ + 0x00, 0x00, 0x00, 0x00, // version, flag + 0x01, // Channel Based Layout + meta.channelCount, // AudioConfiguration + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // omittedChannelsMap + ]); + return MP4.box(MP4.types.chnl, data); + } + + static pcmC(meta) { + let littleEndian = meta.littleEndian ? 0x01 : 0x00 + let sampleSize = meta.sampleSize; + let data = new Uint8Array([ + 0x00, 0x00, 0x00, 0x00, // version, flag + littleEndian, sampleSize + ]); + return MP4.box(MP4.types.pcmC, data); + } + static avc1(meta) { let avcc = meta.avcc; let width = meta.codecWidth, height = meta.codecHeight; diff --git a/src/utils/webworkify-webpack.js b/src/utils/webworkify-webpack.js new file mode 100644 index 00000000..8907e47c --- /dev/null +++ b/src/utils/webworkify-webpack.js @@ -0,0 +1,202 @@ +function webpackBootstrapFunc (modules) { +/******/ // The module cache +/******/ var installedModules = {}; + +/******/ // The require function +/******/ function __webpack_require__(moduleId) { + +/******/ // Check if module is in cache +/******/ if(installedModules[moduleId]) +/******/ return installedModules[moduleId].exports; + +/******/ // Create a new module (and put it into the cache) +/******/ var module = installedModules[moduleId] = { +/******/ i: moduleId, +/******/ l: false, +/******/ exports: {} +/******/ }; + +/******/ // Execute the module function +/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); + +/******/ // Flag the module as loaded +/******/ module.l = true; + +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } + +/******/ // expose the modules object (__webpack_modules__) +/******/ __webpack_require__.m = modules; + +/******/ // expose the module cache +/******/ __webpack_require__.c = installedModules; + +/******/ // identity function for calling harmony imports with the correct context +/******/ __webpack_require__.i = function(value) { return value; }; + +/******/ // define getter function for harmony exports +/******/ __webpack_require__.d = function(exports, name, getter) { +/******/ if(!__webpack_require__.o(exports, name)) { +/******/ Object.defineProperty(exports, name, { +/******/ configurable: false, +/******/ enumerable: true, +/******/ get: getter +/******/ }); +/******/ } +/******/ }; + +/******/ // define __esModule on exports +/******/ __webpack_require__.r = function(exports) { +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; + +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = function(module) { +/******/ var getter = module && module.__esModule ? +/******/ function getDefault() { return module['default']; } : +/******/ function getModuleExports() { return module; }; +/******/ __webpack_require__.d(getter, 'a', getter); +/******/ return getter; +/******/ }; + +/******/ // Object.prototype.hasOwnProperty.call +/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; + +/******/ // __webpack_public_path__ +/******/ __webpack_require__.p = "/"; + +/******/ // on error function for async loading +/******/ __webpack_require__.oe = function(err) { console.error(err); throw err; }; + + var f = __webpack_require__(__webpack_require__.s = ENTRY_MODULE) + return f.default || f // try to call default if defined to also support babel esmodule exports +} + +var moduleNameReqExp = '[\\.|\\-|\\+|\\w|\/|@]+' +var dependencyRegExp = '\\(\\s*(\/\\*.*?\\*\/)?\\s*.*?(' + moduleNameReqExp + ').*?\\)' // additional chars when output.pathinfo is true + +// http://stackoverflow.com/a/2593661/130442 +function quoteRegExp (str) { + return (str + '').replace(/[.?*+^$[\]\\(){}|-]/g, '\\$&') +} + +function isNumeric(n) { + return !isNaN(1 * n); // 1 * n converts integers, integers as string ("123"), 1e3 and "1e3" to integers and strings to NaN +} + +function getModuleDependencies (sources, module, queueName) { + var retval = {} + retval[queueName] = [] + + var fnString = module.toString() + var wrapperSignature = fnString.match(/^function\s?\w*\(\w+,\s*\w+,\s*(\w+)\)/) + if (!wrapperSignature) return retval + var webpackRequireName = wrapperSignature[1] + + // main bundle deps + var re = new RegExp('(\\\\n|\\W)' + quoteRegExp(webpackRequireName) + dependencyRegExp, 'g') + var match + while ((match = re.exec(fnString))) { + if (match[3] === 'dll-reference') continue + retval[queueName].push(match[3]) + } + + // dll deps + re = new RegExp('\\(' + quoteRegExp(webpackRequireName) + '\\("(dll-reference\\s(' + moduleNameReqExp + '))"\\)\\)' + dependencyRegExp, 'g') + while ((match = re.exec(fnString))) { + if (!sources[match[2]]) { + retval[queueName].push(match[1]) + sources[match[2]] = __webpack_require__(match[1]).m + } + retval[match[2]] = retval[match[2]] || [] + retval[match[2]].push(match[4]) + } + + // convert 1e3 back to 1000 - this can be important after uglify-js converted 1000 to 1e3 + var keys = Object.keys(retval); + for (var i = 0; i < keys.length; i++) { + for (var j = 0; j < retval[keys[i]].length; j++) { + if (isNumeric(retval[keys[i]][j])) { + retval[keys[i]][j] = 1 * retval[keys[i]][j]; + } + } + } + + return retval +} + +function hasValuesInQueues (queues) { + var keys = Object.keys(queues) + return keys.reduce(function (hasValues, key) { + return hasValues || queues[key].length > 0 + }, false) +} + +function getRequiredModules (sources, moduleId) { + var modulesQueue = { + main: [moduleId] + } + var requiredModules = { + main: [] + } + var seenModules = { + main: {} + } + + while (hasValuesInQueues(modulesQueue)) { + var queues = Object.keys(modulesQueue) + for (var i = 0; i < queues.length; i++) { + var queueName = queues[i] + var queue = modulesQueue[queueName] + var moduleToCheck = queue.pop() + seenModules[queueName] = seenModules[queueName] || {} + if (seenModules[queueName][moduleToCheck] || !sources[queueName][moduleToCheck]) continue + seenModules[queueName][moduleToCheck] = true + requiredModules[queueName] = requiredModules[queueName] || [] + requiredModules[queueName].push(moduleToCheck) + var newModules = getModuleDependencies(sources, sources[queueName][moduleToCheck], queueName) + var newModulesKeys = Object.keys(newModules) + for (var j = 0; j < newModulesKeys.length; j++) { + modulesQueue[newModulesKeys[j]] = modulesQueue[newModulesKeys[j]] || [] + modulesQueue[newModulesKeys[j]] = modulesQueue[newModulesKeys[j]].concat(newModules[newModulesKeys[j]]) + } + } + } + + return requiredModules +} + +module.exports = function (moduleId, options) { + options = options || {} + var sources = { + main: __webpack_modules__ + } + + var requiredModules = options.all ? { main: Object.keys(sources.main) } : getRequiredModules(sources, moduleId) + + var src = '' + + Object.keys(requiredModules).filter(function (m) { return m !== 'main' }).forEach(function (module) { + var entryModule = 0 + while (requiredModules[module][entryModule]) { + entryModule++ + } + requiredModules[module].push(entryModule) + sources[module][entryModule] = '(function(module, exports, __webpack_require__) { module.exports = __webpack_require__; })' + src = src + 'var ' + module + ' = (' + webpackBootstrapFunc.toString().replace('ENTRY_MODULE', JSON.stringify(entryModule)) + ')({' + requiredModules[module].map(function (id) { return '' + JSON.stringify(id) + ': ' + sources[module][id].toString() }).join(',') + '});\n' + }) + + src = src + 'new ((' + webpackBootstrapFunc.toString().replace('ENTRY_MODULE', JSON.stringify(moduleId)) + ')({' + requiredModules.main.map(function (id) { return '' + JSON.stringify(id) + ': ' + sources.main[id].toString() }).join(',') + '}))(self);' + + var blob = new self.Blob([src], { type: 'text/javascript' }) + if (options.bare) { return blob } + + var URL = self.URL || self.webkitURL || self.mozURL || self.msURL + + var workerUrl = URL.createObjectURL(blob) + var worker = new self.Worker(workerUrl) + worker.objectURL = workerUrl + + return worker +}