Source: lib/text/mp4_vtt_parser.js

  1. /**
  2.  * @license
  3.  * Copyright 2016 Google Inc.
  4.  *
  5.  * Licensed under the Apache License, Version 2.0 (the "License");
  6.  * you may not use this file except in compliance with the License.
  7.  * You may obtain a copy of the License at
  8.  *
  9.  *     http://www.apache.org/licenses/LICENSE-2.0
  10.  *
  11.  * Unless required by applicable law or agreed to in writing, software
  12.  * distributed under the License is distributed on an "AS IS" BASIS,
  13.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14.  * See the License for the specific language governing permissions and
  15.  * limitations under the License.
  16.  */
  18. goog.provide('shaka.text.Mp4VttParser');
  20. goog.require('goog.asserts');
  21. goog.require('shaka.log');
  22. goog.require('shaka.text.Cue');
  23. goog.require('shaka.text.TextEngine');
  24. goog.require('shaka.text.VttTextParser');
  25. goog.require('shaka.util.DataViewReader');
  26. goog.require('shaka.util.Error');
  27. goog.require('shaka.util.Functional');
  28. goog.require('shaka.util.Mp4Parser');
  29. goog.require('shaka.util.StringUtils');
  30. goog.require('shaka.util.TextParser');
  33. /**
  34.  * @struct
  35.  * @constructor
  36.  * @implements {shaka.extern.TextParser}
  37.  */
  38. shaka.text.Mp4VttParser = function() {
  39.   /**
  40.    * The current time scale used by the VTT parser.
  41.    *
  42.    * @type {?number}
  43.    * @private
  44.    */
  45.   this.timescale_ = null;
  46. };
  49. /** @override */
  50. shaka.text.Mp4VttParser.prototype.parseInit = function(data) {
  51.   const Mp4Parser = shaka.util.Mp4Parser;
  53.   let sawWVTT = false;
  55.   new Mp4Parser()
  56.       .box('moov', Mp4Parser.children)
  57.       .box('trak', Mp4Parser.children)
  58.       .box('mdia', Mp4Parser.children)
  59.       .fullBox('mdhd', function(box) {
  60.         goog.asserts.assert(
  61.             box.version == 0 || box.version == 1,
  62.             'MDHD version can only be 0 or 1');
  63.         if (box.version == 0) {
  64.           box.reader.skip(4); // Skip "creation_time".
  65.           box.reader.skip(4); // Skip "modification_time".
  66.           this.timescale_ = box.reader.readUint32();
  67.           box.reader.skip(4); // Skip "duration".
  68.         } else {
  69.           box.reader.skip(8); // Skip "creation_time".
  70.           box.reader.skip(8); // Skip "modification_time".
  71.           this.timescale_ = box.reader.readUint32();
  72.           box.reader.skip(8); // Skip "duration".
  73.         }
  74.         box.reader.skip(4); // Skip "pad", "language", and "pre-defined".
  75.       }.bind(this))
  76.       .box('minf', Mp4Parser.children)
  77.       .box('stbl', Mp4Parser.children)
  78.       .fullBox('stsd', Mp4Parser.sampleDescription)
  79.       .box('wvtt', function(box) {
  80.         // A valid vtt init segment, though we have no actual subtitles yet.
  81.         sawWVTT = true;
  82.       }).parse(data);
  84.   if (!this.timescale_) {
  85.     // Missing timescale for VTT content. It should be located in the MDHD.
  86.     throw new shaka.util.Error(
  87.         shaka.util.Error.Severity.CRITICAL,
  88.         shaka.util.Error.Category.TEXT,
  89.         shaka.util.Error.Code.INVALID_MP4_VTT);
  90.   }
  92.   if (!sawWVTT) {
  93.     // A WVTT box should have been seen (a valid vtt init segment with no
  94.     // actual subtitles).
  95.     throw new shaka.util.Error(
  96.         shaka.util.Error.Severity.CRITICAL,
  97.         shaka.util.Error.Category.TEXT,
  98.         shaka.util.Error.Code.INVALID_MP4_VTT);
  99.   }
  100. };
  103. /** @override */
  104. shaka.text.Mp4VttParser.prototype.parseMedia = function(data, time) {
  105.   if (!this.timescale_) {
  106.     // Missing timescale for VTT content. We should have seen the init segment.
  107.     shaka.log.error('No init segment for MP4+VTT!');
  108.     throw new shaka.util.Error(
  109.         shaka.util.Error.Severity.CRITICAL,
  110.         shaka.util.Error.Category.TEXT,
  111.         shaka.util.Error.Code.INVALID_MP4_VTT);
  112.   }
  114.   const Mp4VttParser = shaka.text.Mp4VttParser;
  115.   const Mp4Parser = shaka.util.Mp4Parser;
  117.   let baseTime = 0;
  118.   /** @type {!Array.<shaka.text.Mp4VttParser.TimeSegment>} */
  119.   let presentations = [];
  120.   /** @type {Uint8Array} */
  121.   let rawPayload;
  122.   /** @type {!Array.<shaka.text.Cue>} */
  123.   let cues = [];
  125.   let sawTFDT = false;
  126.   let sawTRUN = false;
  127.   let sawMDAT = false;
  128.   let defaultDuration = null;
  130.   new Mp4Parser()
  131.       .box('moof', Mp4Parser.children)
  132.       .box('traf', Mp4Parser.children)
  133.       .fullBox('tfdt', function(box) {
  134.         sawTFDT = true;
  135.         goog.asserts.assert(
  136.             box.version == 0 || box.version == 1,
  137.             'TFDT version can only be 0 or 1');
  138.         baseTime = (box.version == 0) ?
  139.             box.reader.readUint32() :
  140.             box.reader.readUint64();
  141.       })
  142.       .fullBox('tfhd', function(box) {
  143.         goog.asserts.assert(
  144.             box.flags != null,
  145.             'A TFHD box should have a valid flags value');
  146.         defaultDuration = Mp4VttParser.parseTFHD_(
  147.             box.flags, box.reader);
  148.       })
  149.       .fullBox('trun', function(box) {
  150.         sawTRUN = true;
  151.         goog.asserts.assert(
  152.             box.version != null,
  153.             'A TRUN box should have a valid version value');
  154.         goog.asserts.assert(
  155.             box.flags != null,
  156.             'A TRUN box should have a valid flags value');
  157.         presentations = Mp4VttParser.parseTRUN_(
  158.             box.version, box.flags, box.reader);
  159.       })
  160.       .box('mdat', Mp4Parser.allData(function(data) {
  161.         goog.asserts.assert(!sawMDAT,
  162.             'VTT cues in mp4 with multiple MDAT are not currently supported!');
  163.         sawMDAT = true;
  164.         rawPayload = data;
  165.       })).parse(data);
  167.   if (!sawMDAT && !sawTFDT && !sawTRUN) {
  168.     // A required box is missing.
  169.     throw new shaka.util.Error(
  170.         shaka.util.Error.Severity.CRITICAL,
  171.         shaka.util.Error.Category.TEXT,
  172.         shaka.util.Error.Code.INVALID_MP4_VTT);
  173.   }
  175.   let currentTime = baseTime;
  177.   let dataView = new DataView(
  178.       rawPayload.buffer, rawPayload.byteOffset, rawPayload.byteLength);
  179.   /** @type {!shaka.util.DataViewReader} */
  180.   let reader = new shaka.util.DataViewReader(
  181.       dataView, shaka.util.DataViewReader.Endianness.BIG_ENDIAN);
  183.   presentations.forEach((presentation) => {
  184.     // If one presentation corresponds to multiple payloads, it is assumed
  185.     // that all of those payloads have the same start time and duration.
  186.     let duration = presentation.duration || defaultDuration;
  187.     let startTime = presentation.timeOffset ?
  188.                     baseTime + presentation.timeOffset :
  189.                     currentTime;
  190.     currentTime = startTime + (duration || 0);
  192.     // Read samples until it adds up to the given size.
  193.     let totalSize = 0;
  194.     do {
  195.       // Read the payload size.
  196.       let payloadSize = reader.readUint32();
  197.       totalSize += payloadSize;
  199.       // Skip the type.
  200.       let payloadType = reader.readUint32();
  201.       let payloadName = shaka.util.Mp4Parser.typeToString(payloadType);
  203.       // Read the data payload.
  204.       /** @type {Uint8Array} */
  205.       let payload = null;
  206.       if (payloadName == 'vttc') {
  207.         if (payloadSize > 8) {
  208.           payload = reader.readBytes(payloadSize - 8);
  209.         }
  210.       } else if (payloadName == 'vtte') {
  211.         // It's a vtte, which is a vtt cue that is empty. Ignore any data that
  212.         // does exist.
  213.         reader.skip(payloadSize - 8);
  214.       } else {
  215.         shaka.log.error('Unknown box ' + payloadName + '! Skipping!');
  216.         reader.skip(payloadSize - 8);
  217.       }
  219.       if (duration) {
  220.         if (payload) {
  221.           cues.push(shaka.text.Mp4VttParser.parseVTTC_(
  222.               payload,
  223.               time.periodStart + startTime / this.timescale_,
  224.               time.periodStart + currentTime / this.timescale_));
  225.         }
  226.       } else {
  227.         shaka.log.error('WVTT sample duration unknown, and no default found!');
  228.       }
  230.       goog.asserts.assert(
  231.           !presentation.sampleSize || totalSize <= presentation.sampleSize,
  232.           'The samples do not fit evenly into the sample sizes given in the ' +
  233.           'TRUN box!');
  235.       // If no sampleSize was specified, it's assumed that this presentation
  236.       // corresponds to only a single cue.
  237.     } while (presentation.sampleSize && (totalSize < presentation.sampleSize));
  238.   });
  240.   goog.asserts.assert(
  241.       !reader.hasMoreData(),
  242.       'MDAT which contain VTT cues and non-VTT data are not currently ' +
  243.       'supported!');
  245.   return /** @type {!Array.<!shaka.extern.Cue>} */ (
  246.       cues.filter(shaka.util.Functional.isNotNull));
  247. };
  250. /**
  251.  * @typedef {{
  252.  *    duration: ?number,
  253.  *    sampleSize: ?number,
  254.  *    timeOffset: ?number
  255.  *  }}
  256.  *
  257.  * @property {?number} duration
  258.  *    The length of the segment in timescale units.
  259.  * @property {?number} sampleSize
  260.  *    The size of the segment in bytes.
  261.  * @property {?number} timeOffset
  262.  *    The time since the start of the segment in timescale units. Time
  263.  *    offset is based of the start of the segment. If this value is
  264.  *    missing, the accumated durations preceeding this time segment will
  265.  *    be used to create the start time.
  266.  */
  267. shaka.text.Mp4VttParser.TimeSegment;
  270. /**
  271.  * @param {number} flags
  272.  * @param {!shaka.util.DataViewReader} reader
  273.  * @return {?number} The default_sample_duration field, if present.
  274.  * @private
  275.  */
  276. shaka.text.Mp4VttParser.parseTFHD_ = function(flags, reader) {
  277.   // Skip "track_ID".
  278.   reader.skip(4);
  280.   // Skip "base_data_offset" if present.
  281.   if (flags & 0x000001) { reader.skip(8); }
  283.   // Skip "sample_description_index" if present.
  284.   if (flags & 0x000002) { reader.skip(4); }
  286.   // Read and return "default_sample_duration" if present.
  287.   if (flags & 0x000008) { return reader.readUint32(); }
  289.   // There is no "default_sample_duration".
  290.   return null;
  291. };
  294. /**
  295.  * @param {number} version
  296.  * @param {number} flags
  297.  * @param {!shaka.util.DataViewReader} reader
  298.  * @return {!Array.<shaka.text.Mp4VttParser.TimeSegment>}
  299.  * @private
  300.  */
  301. shaka.text.Mp4VttParser.parseTRUN_ = function(version, flags, reader) {
  302.   let sampleCount = reader.readUint32();
  304.   // Skip "data_offset" if present.
  305.   if (flags & 0x000001) { reader.skip(4); }
  307.   // Skip "first_sample_flags" if present.
  308.   if (flags & 0x000004) { reader.skip(4); }
  310.   let samples = [];
  312.   for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) {
  313.     /** @type {shaka.text.Mp4VttParser.TimeSegment} */
  314.     let sample = {
  315.       duration: null,
  316.       sampleSize: null,
  317.       timeOffset: null,
  318.     };
  320.     // Read "sample duration" if present.
  321.     if (flags & 0x000100) { sample.duration = reader.readUint32(); }
  323.     // Read "sample_size" if present.
  324.     if (flags & 0x000200) { sample.sampleSize = reader.readUint32(); }
  326.     // Skip "sample_flags" if present.
  327.     if (flags & 0x000400) { reader.skip(4); }
  329.     // Read "sample_time_offset" if present.
  330.     if (flags & 0x000800) {
  331.       sample.timeOffset = version == 0 ?
  332.           reader.readUint32() :
  333.           reader.readInt32();
  334.     }
  336.     samples.push(sample);
  337.   }
  339.   return samples;
  340. };
  343. /**
  344.  * Parses a vttc box into a cue.
  345.  *
  346.  * @param {!Uint8Array} data
  347.  * @param {number} startTime
  348.  * @param {number} endTime
  349.  * @return {shaka.text.Cue}
  350.  * @private
  351.  */
  352. shaka.text.Mp4VttParser.parseVTTC_ = function(data, startTime, endTime) {
  353.   let payload;
  354.   let id;
  355.   let settings;
  357.   new shaka.util.Mp4Parser()
  358.       .box('payl', shaka.util.Mp4Parser.allData(function(data) {
  359.         payload = shaka.util.StringUtils.fromUTF8(data);
  360.       }))
  361.       .box('iden', shaka.util.Mp4Parser.allData(function(data) {
  362.         id = shaka.util.StringUtils.fromUTF8(data);
  363.       }))
  364.       .box('sttg', shaka.util.Mp4Parser.allData(function(data) {
  365.         settings = shaka.util.StringUtils.fromUTF8(data);
  366.       }))
  367.       .parse(data);
  369.   if (payload) {
  370.     return shaka.text.Mp4VttParser.assembleCue_(payload,
  371.                                                 id,
  372.                                                 settings,
  373.                                                 startTime,
  374.                                                 endTime);
  375.   } else {
  376.     return null;
  377.   }
  378. };
  381. /**
  382.  * Take the individual components that make a cue and create a vttc cue.
  383.  *
  384.  * @param {string} payload
  385.  * @param {?string} id
  386.  * @param {?string} settings
  387.  * @param {number} startTime
  388.  * @param {number} endTime
  389.  * @return {!shaka.text.Cue}
  390.  * @private
  391.  */
  392. shaka.text.Mp4VttParser.assembleCue_ = function(payload,
  393.                                                 id,
  394.                                                 settings,
  395.                                                 startTime,
  396.                                                 endTime) {
  397.   let cue = new shaka.text.Cue(
  398.       startTime,
  399.       endTime,
  400.       payload);
  402.   if (id) {
  403.     cue.id = id;
  404.   }
  406.   if (settings) {
  407.     let parser = new shaka.util.TextParser(settings);
  409.     let word = parser.readWord();
  411.     while (word) {
  412.       // TODO: Check WebVTTConfigurationBox for region info.
  413.       if (!shaka.text.VttTextParser.parseCueSetting(cue, word,
  414.                                                     /* VTTRegions */ [])) {
  415.         shaka.log.warning('VTT parser encountered an invalid VTT setting: ',
  416.                           word,
  417.                           ' The setting will be ignored.');
  418.       }
  420.       parser.skipWhitespace();
  421.       word = parser.readWord();
  422.     }
  423.   }
  425.   return cue;
  426. };
  429. shaka.text.TextEngine.registerParser(
  430.     'application/mp4; codecs="wvtt"',
  431.     shaka.text.Mp4VttParser);