diff options
29 files changed, 504 insertions, 169 deletions
diff --git a/Cargo.lock b/Cargo.lock index 35444b693a6..11dbb0af8e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,16 +98,6 @@ dependencies = [ ] [[package]] -name = "audio-video-metadata" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "mp3-metadata 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "mp4parse 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ogg_metadata 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] name = "azure" version = "0.34.0" source = "git+https://github.com/servo/rust-azure#b3c8ddc64f2760cfb5c29818c93b9a02fac50e5e" @@ -194,11 +184,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "bitreader" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] name = "block" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2242,22 +2227,6 @@ dependencies = [ ] [[package]] -name = "mp3-metadata" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "mp4parse" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitreader 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] name = "msdos_time" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2498,23 +2467,6 @@ dependencies = [ ] [[package]] -name = "ogg" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ogg_metadata" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ogg 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] name = "openssl" version = "0.9.24" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2981,7 +2933,6 @@ name = "script" version = "0.0.1" dependencies = [ "app_units 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "audio-video-metadata 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3273,7 +3224,7 @@ dependencies = [ [[package]] name = "servo-media" version = "0.1.0" -source = "git+https://github.com/servo/media#e700a0834c3f38d49890d846591fd699e7405a48" +source = "git+https://github.com/servo/media#ce50f1332cc2b70e859b793425c9ec7086137303" dependencies = [ "servo-media-audio 0.1.0 (git+https://github.com/servo/media)", "servo-media-gstreamer 0.1.0 (git+https://github.com/servo/media)", @@ -3283,11 +3234,12 @@ dependencies = [ [[package]] name = "servo-media-audio" version = "0.1.0" -source = "git+https://github.com/servo/media#e700a0834c3f38d49890d846591fd699e7405a48" +source = "git+https://github.com/servo/media#ce50f1332cc2b70e859b793425c9ec7086137303" dependencies = [ "boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "euclid 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3299,7 +3251,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer" version = "0.1.0" -source = "git+https://github.com/servo/media#e700a0834c3f38d49890d846591fd699e7405a48" +source = "git+https://github.com/servo/media#ce50f1332cc2b70e859b793425c9ec7086137303" dependencies = [ "byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "glib 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3317,7 +3269,7 @@ dependencies = [ [[package]] name = "servo-media-player" version = "0.1.0" -source = "git+https://github.com/servo/media#e700a0834c3f38d49890d846591fd699e7405a48" +source = "git+https://github.com/servo/media#ce50f1332cc2b70e859b793425c9ec7086137303" dependencies = [ "ipc-channel 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3413,7 +3365,7 @@ dependencies = [ [[package]] name = "servo_media_derive" version = "0.1.0" -source = "git+https://github.com/servo/media#e700a0834c3f38d49890d846591fd699e7405a48" +source = "git+https://github.com/servo/media#ce50f1332cc2b70e859b793425c9ec7086137303" dependencies = [ "quote 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4344,7 +4296,6 @@ dependencies = [ "checksum ascii 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae7d751998c189c1d4468cf0a39bb2eae052a9c58d50ebb3b9591ee3813ad50" "checksum atomic_refcell 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fb2dcb6e6d35f20276943cc04bb98e538b348d525a04ac79c10021561d202f21" "checksum atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "21e50800ec991574876040fff8ee46b136a53e985286fbe6a3bdfe6421b78860" -"checksum audio-video-metadata 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3c23600291b35b9cd381f5cfca636f5d7d75e7d7192eddf867de84a6732cad5c" "checksum azure 0.34.0 (git+https://github.com/servo/rust-azure)" = "<none>" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" @@ -4354,7 +4305,6 @@ dependencies = [ "checksum bindgen 0.39.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eac4ed5f2de9efc3c87cb722468fa49d0763e98f999d539bfc5e452c13d85c91" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" "checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" -"checksum bitreader 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80b13e2ab064ff3aa0bdbf1eff533f9822dc37899821f5f98c67f263eab51707" "checksum block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" "checksum blurdroid 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "19b23557dd27704797128f9db2816416bef20dad62d4a9768714eeb65f07d296" "checksum blurmac 0.1.0 (git+https://github.com/servo/devices)" = "<none>" @@ -4519,8 +4469,6 @@ dependencies = [ "checksum mozangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "45a8a18a41cfab0fde25cc2f43ea89064d211a0fbb33225b8ff93ab20406e0e7" "checksum mozjs 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b9f85c1120b07d7a2acc9d1d62df1fe16f64162399448fb5307bf2bc3bd066c9" "checksum mozjs_sys 0.61.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff07b0f0a2371dc08d75d55371ca311be67e1fdfa6c146fc8ad154c340f70c9" -"checksum mp3-metadata 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ab5f1d2693586420208d1200ce5a51cd44726f055b635176188137aff42c7de" -"checksum mp4parse 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7316728464443fe5793a805dde3257864e9690cf46374daff3ce93de1df2f254" "checksum msdos_time 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "aad9dfe950c057b1bfe9c1f2aa51583a8468ef2a5baba2ebbe06d775efeb7729" "checksum muldiv 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "451a9a05d2a32c566c897835e0ea95cf79ed2fdfe957924045a1721a36c9980f" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" @@ -4539,8 +4487,6 @@ dependencies = [ "checksum objc-foundation 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" "checksum objc_id 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4730aa1c64d722db45f7ccc4113a3e2c465d018de6db4d3e7dfe031e8c8a297" "checksum offscreen_gl_context 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95f2e39e3b8c95495cfec835b6fefee3f1e7d63c6f81d99796b4f9926c02db3c" -"checksum ogg 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7137bf02687385302f4c0aecd77cfce052b69f5b4ee937be778e125c62f67e30" -"checksum ogg_metadata 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc665717454399cba557c55ad226148996e9266ee291f8a37a98bb2cded0a490" "checksum openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "a3605c298474a3aa69de92d21139fb5e2a81688d308262359d85cdd0d12a7985" "checksum openssl-sys 0.9.27 (registry+https://github.com/rust-lang/crates.io-index)" = "d6fdc5c4a02e69ce65046f1763a0181107038e02176233acb0b3351d7cc588f9" "checksum ordered-float 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a3c8db0fca1fdb34404f0b1286db252f23930b9f7a481e376c16c0d5c309d4" diff --git a/components/atoms/static_atoms.txt b/components/atoms/static_atoms.txt index 7fbd043dc0f..4281c7004d0 100644 --- a/components/atoms/static_atoms.txt +++ b/components/atoms/static_atoms.txt @@ -18,6 +18,7 @@ cursive date datetime-local dir +durationchange email emptied ended @@ -65,6 +66,7 @@ range readystatechange reftest-wait reset +resize right rtl sans-serif diff --git a/components/constellation/pipeline.rs b/components/constellation/pipeline.rs index d110bbfcdce..d37d4d80f07 100644 --- a/components/constellation/pipeline.rs +++ b/components/constellation/pipeline.rs @@ -517,6 +517,7 @@ impl UnprivilegedPipelineContent { webgl_chan: self.webgl_chan, webvr_chan: self.webvr_chan, webrender_document: self.webrender_document, + webrender_api_sender: self.webrender_api_sender.clone(), }, self.load_data.clone(), ); diff --git a/components/layout/construct.rs b/components/layout/construct.rs index 36ab116d0ca..ba86416f935 100644 --- a/components/layout/construct.rs +++ b/components/layout/construct.rs @@ -23,11 +23,10 @@ use floats::FloatKind; use flow::{AbsoluteDescendants, Flow, FlowClass, GetBaseFlow, ImmutableFlowUtils}; use flow::{FlowFlags, MutableFlowUtils, MutableOwnedFlowUtils}; use flow_ref::FlowRef; -use fragment::{CanvasFragmentInfo, ImageFragmentInfo, InlineAbsoluteFragmentInfo, SvgFragmentInfo}; -use fragment::{Fragment, GeneratedContentInfo, IframeFragmentInfo, FragmentFlags}; -use fragment::{InlineAbsoluteHypotheticalFragmentInfo, TableColumnFragmentInfo}; -use fragment::{InlineBlockFragmentInfo, SpecificFragmentInfo, UnscannedTextFragmentInfo}; -use fragment::WhitespaceStrippingResult; +use fragment::{CanvasFragmentInfo, Fragment, FragmentFlags, GeneratedContentInfo, IframeFragmentInfo}; +use fragment::{ImageFragmentInfo, InlineAbsoluteFragmentInfo, InlineAbsoluteHypotheticalFragmentInfo}; +use fragment::{InlineBlockFragmentInfo, MediaFragmentInfo, SpecificFragmentInfo, SvgFragmentInfo}; +use fragment::{TableColumnFragmentInfo, UnscannedTextFragmentInfo, WhitespaceStrippingResult}; use inline::{InlineFlow, InlineFragmentNodeInfo, InlineFragmentNodeFlags}; use linked_list::prepend_from; use list_item::{ListItemFlow, ListStyleTypeContent}; @@ -405,6 +404,10 @@ impl<'a, ConcreteThreadSafeLayoutNode: ThreadSafeLayoutNode> &self.layout_context, )); SpecificFragmentInfo::Image(image_info) + } + Some(LayoutNodeType::Element(LayoutElementType::HTMLMediaElement)) => { + let data = node.media_data().unwrap(); + SpecificFragmentInfo::Media(Box::new(MediaFragmentInfo::new(data))) }, Some(LayoutNodeType::Element(LayoutElementType::HTMLObjectElement)) => { let image_info = Box::new(ImageFragmentInfo::new( @@ -1956,6 +1959,7 @@ where match self.type_id() { Some(LayoutNodeType::Text) | Some(LayoutNodeType::Element(LayoutElementType::HTMLImageElement)) | + Some(LayoutNodeType::Element(LayoutElementType::HTMLMediaElement)) | Some(LayoutNodeType::Element(LayoutElementType::HTMLIFrameElement)) | Some(LayoutNodeType::Element(LayoutElementType::HTMLCanvasElement)) | Some(LayoutNodeType::Element(LayoutElementType::SVGSVGElement)) => true, diff --git a/components/layout/display_list/builder.rs b/components/layout/display_list/builder.rs index f473b98584a..3d49fdbb61c 100644 --- a/components/layout/display_list/builder.rs +++ b/components/layout/display_list/builder.rs @@ -1999,6 +1999,22 @@ impl FragmentDisplayListBuilding for Fragment { } } }, + SpecificFragmentInfo::Media(ref fragment_info) => { + if let Some((ref image_key, _, _)) = fragment_info.current_frame + { + let base = create_base_display_item(state); + state.add_image_item( + base, + webrender_api::ImageDisplayItem { + image_key: *image_key, + stretch_size: stacking_relative_border_box.size.to_layout(), + tile_spacing: LayoutSize::zero(), + image_rendering: ImageRendering::Auto, + alpha_type: webrender_api::AlphaType::PremultipliedAlpha, + }, + ); + } + } SpecificFragmentInfo::Canvas(ref canvas_fragment_info) => { let image_key = match canvas_fragment_info.source { CanvasFragmentSource::WebGL(image_key) => image_key, diff --git a/components/layout/fragment.rs b/components/layout/fragment.rs index 65fd277fdde..33a6ac3dd3b 100644 --- a/components/layout/fragment.rs +++ b/components/layout/fragment.rs @@ -31,8 +31,7 @@ use msg::constellation_msg::{BrowsingContextId, PipelineId}; use net_traits::image::base::{Image, ImageMetadata}; use net_traits::image_cache::{ImageOrMetadataAvailable, UsePlaceholder}; use range::*; -use script_layout_interface::{HTMLCanvasData, HTMLCanvasDataSource}; -use script_layout_interface::SVGSVGData; +use script_layout_interface::{HTMLCanvasData, HTMLCanvasDataSource, HTMLMediaData, SVGSVGData}; use script_layout_interface::wrapper_traits::{PseudoElementType, ThreadSafeLayoutElement, ThreadSafeLayoutNode}; use serde::ser::{Serialize, SerializeStruct, Serializer}; use servo_url::ServoUrl; @@ -182,6 +181,7 @@ pub enum SpecificFragmentInfo { Iframe(IframeFragmentInfo), Image(Box<ImageFragmentInfo>), + Media(Box<MediaFragmentInfo>), Canvas(Box<CanvasFragmentInfo>), Svg(Box<SvgFragmentInfo>), @@ -219,6 +219,7 @@ impl SpecificFragmentInfo { SpecificFragmentInfo::GeneratedContent(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::ScannedText(_) | SpecificFragmentInfo::Svg(_) | SpecificFragmentInfo::Table | @@ -242,6 +243,7 @@ impl SpecificFragmentInfo { pub fn get_type(&self) -> &'static str { match *self { SpecificFragmentInfo::Canvas(_) => "SpecificFragmentInfo::Canvas", + SpecificFragmentInfo::Media(_) => "SpecificFragmentInfo::Media", SpecificFragmentInfo::Generic => "SpecificFragmentInfo::Generic", SpecificFragmentInfo::GeneratedContent(_) => "SpecificFragmentInfo::GeneratedContent", SpecificFragmentInfo::Iframe(_) => "SpecificFragmentInfo::Iframe", @@ -365,6 +367,19 @@ impl CanvasFragmentInfo { } #[derive(Clone)] +pub struct MediaFragmentInfo { + pub current_frame: Option<(webrender_api::ImageKey, i32, i32)>, +} + +impl MediaFragmentInfo { + pub fn new(data: HTMLMediaData) -> MediaFragmentInfo { + MediaFragmentInfo { + current_frame: data.current_frame, + } + } +} + +#[derive(Clone)] pub struct SvgFragmentInfo { pub dom_width: Au, pub dom_height: Au, @@ -834,6 +849,7 @@ impl Fragment { ) -> QuantitiesIncludedInIntrinsicInlineSizes { match self.specific { SpecificFragmentInfo::Canvas(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::Generic | SpecificFragmentInfo::GeneratedContent(_) | SpecificFragmentInfo::Iframe(_) | @@ -978,6 +994,13 @@ impl Fragment { } else { Au(0) } + } + SpecificFragmentInfo::Media(ref info) => { + if let Some((_, width, _)) = info.current_frame { + Au::from_px(width as i32) + } else { + Au(0) + } }, SpecificFragmentInfo::Canvas(ref info) => info.dom_width, SpecificFragmentInfo::Svg(ref info) => info.dom_width, @@ -1001,6 +1024,13 @@ impl Fragment { } else { Au(0) } + } + SpecificFragmentInfo::Media(ref info) => { + if let Some((_, _, height)) = info.current_frame { + Au::from_px(height as i32) + } else { + Au(0) + } }, SpecificFragmentInfo::Canvas(ref info) => info.dom_height, SpecificFragmentInfo::Svg(ref info) => info.dom_height, @@ -1014,6 +1044,7 @@ impl Fragment { match self.specific { SpecificFragmentInfo::Image(_) | SpecificFragmentInfo::Canvas(_) | + SpecificFragmentInfo::Media(_) | // TODO(stshine): According to the SVG spec, whether a SVG element has intrinsic // aspect ratio is determined by the `preserveAspectRatio` attribute. Since for // now SVG is far from implemented, we simply choose the default behavior that @@ -1549,6 +1580,7 @@ impl Fragment { result.union_block(&block_flow.base.intrinsic_inline_sizes) }, SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::Canvas(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Svg(_) => { @@ -2024,6 +2056,7 @@ impl Fragment { }, SpecificFragmentInfo::Canvas(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::InlineBlock(_) | SpecificFragmentInfo::InlineAbsoluteHypothetical(_) | @@ -2115,6 +2148,7 @@ impl Fragment { SpecificFragmentInfo::Canvas(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::InlineBlock(_) | SpecificFragmentInfo::InlineAbsoluteHypothetical(_) | SpecificFragmentInfo::InlineAbsolute(_) | @@ -2171,6 +2205,7 @@ impl Fragment { SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Canvas(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::Svg(_) => true, _ => false, } @@ -2202,6 +2237,7 @@ impl Fragment { SpecificFragmentInfo::Canvas(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::Svg(_) | SpecificFragmentInfo::Generic | SpecificFragmentInfo::GeneratedContent(_) => { @@ -2530,6 +2566,7 @@ impl Fragment { SpecificFragmentInfo::GeneratedContent(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::ScannedText(_) | SpecificFragmentInfo::Svg(_) | SpecificFragmentInfo::Table | @@ -3057,6 +3094,7 @@ impl Fragment { SpecificFragmentInfo::GeneratedContent(_) | SpecificFragmentInfo::Iframe(_) | SpecificFragmentInfo::Image(_) | + SpecificFragmentInfo::Media(_) | SpecificFragmentInfo::ScannedText(_) | SpecificFragmentInfo::TruncatedFragment(_) | SpecificFragmentInfo::Svg(_) | diff --git a/components/layout_thread/dom_wrapper.rs b/components/layout_thread/dom_wrapper.rs index 5900b912e73..3c3495ee6fe 100644 --- a/components/layout_thread/dom_wrapper.rs +++ b/components/layout_thread/dom_wrapper.rs @@ -43,8 +43,8 @@ use script::layout_exports::{LayoutCharacterDataHelpers, LayoutDocumentHelpers}; use script::layout_exports::{LayoutElementHelpers, LayoutNodeHelpers, LayoutDom, RawLayoutElementHelpers}; use script::layout_exports::NodeFlags; use script::layout_exports::PendingRestyle; -use script_layout_interface::{HTMLCanvasData, LayoutNodeType, SVGSVGData, TrustedNodeAddress}; -use script_layout_interface::{OpaqueStyleAndLayoutData, StyleData}; +use script_layout_interface::{HTMLCanvasData, HTMLMediaData, LayoutNodeType, OpaqueStyleAndLayoutData}; +use script_layout_interface::{SVGSVGData, StyleData, TrustedNodeAddress}; use script_layout_interface::wrapper_traits::{DangerousThreadSafeLayoutNode, GetLayoutData, LayoutNode}; use script_layout_interface::wrapper_traits::{PseudoElementType, ThreadSafeLayoutElement, ThreadSafeLayoutNode}; use selectors::attr::{AttrSelectorOperation, NamespaceConstraint, CaseSensitivity}; @@ -1053,6 +1053,11 @@ impl<'ln> ThreadSafeLayoutNode for ServoThreadSafeLayoutNode<'ln> { this.canvas_data() } + fn media_data(&self) -> Option<HTMLMediaData> { + let this = unsafe { self.get_jsmanaged() }; + this.media_data() + } + fn svg_data(&self) -> Option<SVGSVGData> { let this = unsafe { self.get_jsmanaged() }; this.svg_data() diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml index 913edf25f2e..52108a89c59 100644 --- a/components/script/Cargo.toml +++ b/components/script/Cargo.toml @@ -30,7 +30,6 @@ tinyfiledialogs = "3.0" [dependencies] app_units = "0.7" -audio-video-metadata = "0.1.4" backtrace = {version = "0.3", optional = true} base64 = "0.6" bitflags = "1.0" diff --git a/components/script/dom/baseaudiocontext.rs b/components/script/dom/baseaudiocontext.rs index b892049a8ec..0cb60a2fd80 100644 --- a/components/script/dom/baseaudiocontext.rs +++ b/components/script/dom/baseaudiocontext.rs @@ -461,7 +461,7 @@ impl BaseAudioContextMethods for BaseAudioContext { }), &canceller, ); - }).error(move || { + }).error(move |error| { let _ = task_source_.queue_with_canceller( task!(audio_decode_eos: move || { let this = this_.root(); @@ -473,7 +473,8 @@ impl BaseAudioContextMethods for BaseAudioContext { &DOMException::new(&this.global(), DOMErrorName::DataCloneError), ExceptionHandling::Report); } - resolver.promise.reject_error(Error::Type("Audio decode error".to_owned())); + let error = format!("Audio decode error {:?}", error); + resolver.promise.reject_error(Error::Type(error)); }), &canceller_, ); diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index a3c4c7814ea..8e720f28e25 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -48,6 +48,7 @@ use dom::bindings::str::{DOMString, USVString}; use dom::bindings::utils::WindowProxyHandler; use dom::document::PendingRestyle; use dom::htmlimageelement::SourceSet; +use dom::htmlmediaelement::MediaFrameRenderer; use encoding_rs::{Decoder, Encoding}; use euclid::{Transform2D, Transform3D, Point2D, Vector2D, Rect, TypedSize2D, TypedScale}; use euclid::Length as EuclidLength; @@ -90,12 +91,14 @@ use servo_arc::Arc as ServoArc; use servo_atoms::Atom; use servo_channel::{Receiver, Sender}; use servo_media::Backend; +use servo_media::Error as ServoMediaError; use servo_media::audio::analyser_node::AnalysisEngine; use servo_media::audio::buffer_source_node::AudioBuffer; use servo_media::audio::context::AudioContext; use servo_media::audio::graph::NodeId; use servo_media::audio::panner_node::{DistanceModel, PanningModel}; use servo_media::audio::param::ParamType; +use servo_media::player::Player; use servo_url::{ImmutableOrigin, MutableOrigin, ServoUrl}; use smallvec::SmallVec; use std::cell::{Cell, RefCell, UnsafeCell}; @@ -104,7 +107,7 @@ use std::hash::{BuildHasher, Hash}; use std::ops::{Deref, DerefMut}; use std::path::PathBuf; use std::rc::Rc; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicBool, AtomicUsize}; use std::time::{SystemTime, Instant}; use style::attr::{AttrIdentifier, AttrValue, LengthOrPercentageOrAuto}; @@ -121,7 +124,7 @@ use style::stylesheets::keyframes_rule::Keyframe; use style::values::specified::Length; use time::Duration; use uuid::Uuid; -use webrender_api::{DocumentId, ImageKey}; +use webrender_api::{DocumentId, ImageKey, RenderApiSender}; use webvr_traits::WebVRGamepadHand; /// A trait to allow tracing (only) DOM objects. @@ -454,6 +457,9 @@ unsafe_no_jsmanaged_fields!(AudioBuffer); unsafe_no_jsmanaged_fields!(AudioContext<Backend>); unsafe_no_jsmanaged_fields!(NodeId); unsafe_no_jsmanaged_fields!(AnalysisEngine, DistanceModel, PanningModel, ParamType); +unsafe_no_jsmanaged_fields!(Player<Error=ServoMediaError>); +unsafe_no_jsmanaged_fields!(Mutex<MediaFrameRenderer>); +unsafe_no_jsmanaged_fields!(RenderApiSender); unsafe impl<'a> JSTraceable for &'a str { #[inline] diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index ad2a487a16f..84380c8d11f 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -2,7 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use audio_video_metadata; use document_loader::{LoadBlocker, LoadType}; use dom::attr::Attr; use dom::bindings::cell::DomRefCell; @@ -18,7 +17,7 @@ use dom::bindings::error::{Error, ErrorResult}; use dom::bindings::inheritance::Castable; use dom::bindings::refcounted::Trusted; use dom::bindings::reflector::DomObject; -use dom::bindings::root::{DomRoot, MutNullableDom}; +use dom::bindings::root::{DomRoot, LayoutDom, MutNullableDom}; use dom::bindings::str::DOMString; use dom::blob::Blob; use dom::document::Document; @@ -26,28 +25,106 @@ use dom::element::{Element, AttributeMutation}; use dom::eventtarget::EventTarget; use dom::htmlelement::HTMLElement; use dom::htmlsourceelement::HTMLSourceElement; +use dom::htmlvideoelement::HTMLVideoElement; use dom::mediaerror::MediaError; -use dom::node::{window_from_node, document_from_node, Node, UnbindContext}; +use dom::node::{document_from_node, window_from_node, Node, NodeDamage, UnbindContext}; use dom::promise::Promise; use dom::virtualmethods::VirtualMethods; use dom_struct::dom_struct; +use fetch::FetchCanceller; use html5ever::{LocalName, Prefix}; +use hyper::header::ContentLength; use ipc_channel::ipc; use ipc_channel::router::ROUTER; use microtask::{Microtask, MicrotaskRunnable}; use mime::{Mime, SubLevel, TopLevel}; -use net_traits::{FetchResponseListener, FetchMetadata, Metadata, NetworkError}; +use net_traits::{CoreResourceMsg, FetchChannels, FetchResponseListener, FetchMetadata, Metadata}; +use net_traits::NetworkError; use net_traits::request::{CredentialsMode, Destination, RequestInit}; use network_listener::{NetworkListener, PreInvoke}; +use script_layout_interface::HTMLMediaData; use script_thread::ScriptThread; +use servo_media::Error as ServoMediaError; +use servo_media::ServoMedia; +use servo_media::player::{PlaybackState, Player, PlayerEvent}; +use servo_media::player::frame::{Frame, FrameRenderer}; use servo_url::ServoUrl; use std::cell::Cell; use std::collections::VecDeque; +use std::f64; use std::mem; use std::rc::Rc; use std::sync::{Arc, Mutex}; use task_source::{TaskSource, TaskSourceName}; use time::{self, Timespec, Duration}; +use webrender_api::{ImageData, ImageDescriptor, ImageFormat, ImageKey, RenderApi}; +use webrender_api::{RenderApiSender, Transaction}; + +pub struct MediaFrameRenderer { + api: RenderApi, + current_frame: Option<(ImageKey, i32, i32)>, + old_frame: Option<ImageKey>, + very_old_frame: Option<ImageKey>, +} + +impl MediaFrameRenderer { + fn new(render_api_sender: RenderApiSender) -> Self { + Self { + api: render_api_sender.create_api(), + current_frame: None, + old_frame: None, + very_old_frame: None, + } + } +} + +impl FrameRenderer for MediaFrameRenderer { + fn render(&mut self, frame: Frame) { + let descriptor = ImageDescriptor::new( + frame.get_width() as u32, + frame.get_height() as u32, + ImageFormat::BGRA8, + false, + false, + ); + + let mut txn = Transaction::new(); + + let image_data = ImageData::Raw(frame.get_data().clone()); + + if let Some(old_image_key) = mem::replace(&mut self.very_old_frame, self.old_frame.take()) { + txn.delete_image(old_image_key); + } + + match self.current_frame { + Some((ref image_key, ref mut width, ref mut height)) + if *width == frame.get_width() && *height == frame.get_height() => + { + txn.update_image(*image_key, descriptor, image_data, None); + + if let Some(old_image_key) = self.old_frame.take() { + txn.delete_image(old_image_key); + } + } + Some((ref mut image_key, ref mut width, ref mut height)) => { + self.old_frame = Some(*image_key); + + let new_image_key = self.api.generate_image_key(); + txn.add_image(new_image_key, descriptor, image_data, None); + *image_key = new_image_key; + *width = frame.get_width(); + *height = frame.get_height(); + }, + None => { + let image_key = self.api.generate_image_key(); + txn.add_image(image_key, descriptor, image_data, None); + self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); + }, + } + + self.api.update_resources(txn.resource_updates); + } +} #[dom_struct] // FIXME(nox): A lot of tasks queued for this element should probably be in the @@ -82,6 +159,15 @@ pub struct HTMLMediaElement { /// Play promises which are soon to be fulfilled by a queued task. #[ignore_malloc_size_of = "promises are hard"] in_flight_play_promises_queue: DomRefCell<VecDeque<(Box<[Rc<Promise>]>, ErrorResult)>>, + #[ignore_malloc_size_of = "servo_media"] + player: Box<Player<Error=ServoMediaError>>, + #[ignore_malloc_size_of = "Arc"] + frame_renderer: Arc<Mutex<MediaFrameRenderer>>, + fetch_canceller: DomRefCell<FetchCanceller>, + /// https://html.spec.whatwg.org/multipage/#show-poster-flag + show_poster: Cell<bool>, + /// https://html.spec.whatwg.org/multipage/#dom-media-duration + duration: Cell<f64>, } /// <https://html.spec.whatwg.org/multipage/#dom-media-networkstate> @@ -95,9 +181,9 @@ pub enum NetworkState { } /// <https://html.spec.whatwg.org/multipage/#dom-media-readystate> -#[derive(Clone, Copy, JSTraceable, MallocSizeOf, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, JSTraceable, MallocSizeOf, PartialEq, PartialOrd)] #[repr(u8)] -enum ReadyState { +pub enum ReadyState { HaveNothing = HTMLMediaElementConstants::HAVE_NOTHING as u8, HaveMetadata = HTMLMediaElementConstants::HAVE_METADATA as u8, HaveCurrentData = HTMLMediaElementConstants::HAVE_CURRENT_DATA as u8, @@ -122,9 +208,20 @@ impl HTMLMediaElement { delaying_the_load_event_flag: Default::default(), pending_play_promises: Default::default(), in_flight_play_promises_queue: Default::default(), + player: ServoMedia::get().unwrap().create_player(), + frame_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new( + document.window().get_webrender_api_sender(), + ))), + fetch_canceller: DomRefCell::new(Default::default()), + show_poster: Cell::new(true), + duration: Cell::new(f64::NAN), } } + pub fn get_ready_state(&self) -> ReadyState { + self.ready_state.get() + } + fn media_type_id(&self) -> HTMLMediaElementTypeId { match self.upcast::<Node>().type_id() { NodeTypeId::Element(ElementTypeId::HTMLElement( @@ -187,8 +284,10 @@ impl HTMLMediaElement { self.paused.set(false); // Step 6.2. - // FIXME(nox): Set show poster flag to false and run time marches on - // steps if show poster flag is true. + if self.show_poster.get() { + self.show_poster.set(false); + self.time_marches_on(); + } // Step 6.3. task_source.queue_simple_event(self.upcast(), atom!("play"), &window); @@ -217,10 +316,15 @@ impl HTMLMediaElement { return; } - this.fulfill_in_flight_play_promises(|| ()); + this.fulfill_in_flight_play_promises(|| { + if let Err(e) = this.player.play() { + eprintln!("Could not play media {:?}", e); + } + }); }), window.upcast(), - ).unwrap(); + ) + .unwrap(); } // Step 8. @@ -230,6 +334,11 @@ impl HTMLMediaElement { // Not applicable here, the promise is returned from Play. } + /// https://html.spec.whatwg.org/multipage/#time-marches-on + fn time_marches_on(&self) { + // TODO: implement this. + } + /// <https://html.spec.whatwg.org/multipage/#internal-pause-steps> fn internal_pause_steps(&self) { // Step 1. @@ -263,6 +372,10 @@ impl HTMLMediaElement { // Step 2.3.2. this.upcast::<EventTarget>().fire_event(atom!("pause")); + if let Err(e) = this.player.pause() { + eprintln!("Could not pause player {:?}", e); + } + // Step 2.3.3. // Done after running this closure in // `fulfill_in_flight_play_promises`. @@ -298,6 +411,9 @@ impl HTMLMediaElement { this.fulfill_in_flight_play_promises(|| { // Step 2.1. this.upcast::<EventTarget>().fire_event(atom!("playing")); + if let Err(e) = this.player.play() { + eprintln!("Could not play media {:?}", e); + } // Step 2.2. // Done after running this closure in @@ -377,7 +493,11 @@ impl HTMLMediaElement { if self.autoplaying.get() && self.Paused() && self.Autoplay() { // Step 1 self.paused.set(false); - // TODO step 2: show poster + // Step 2 + if self.show_poster.get() { + self.show_poster.set(false); + self.time_marches_on(); + } // Step 3 task_source.queue_simple_event(self.upcast(), atom!("play"), &window); // Step 4 @@ -398,7 +518,7 @@ impl HTMLMediaElement { self.network_state.set(NetworkState::NoSource); // Step 2. - // FIXME(nox): Set show poster flag to true. + self.show_poster.set(true); // Step 3. self.delay_load_event(true); @@ -416,7 +536,7 @@ impl HTMLMediaElement { base_url: doc.base_url(), }; - // FIXME(nox): This will later call the resource_selection_algorith_sync + // FIXME(nox): This will later call the resource_selection_algorithm_sync // method from below, if microtasks were trait objects, we would be able // to put the code directly in this method, without the boilerplate // indirections. @@ -426,6 +546,8 @@ impl HTMLMediaElement { // https://html.spec.whatwg.org/multipage/#concept-media-load-algorithm fn resource_selection_algorithm_sync(&self, base_url: ServoUrl) { // Step 5. + // FIXME(ferjm): Implement blocked_on_parser logic + // https://html.spec.whatwg.org/multipage/#blocked-on-parser // FIXME(nox): Maybe populate the list of pending text tracks. // Step 6. @@ -515,6 +637,7 @@ impl HTMLMediaElement { }, Mode::Children(_source) => { // Step 9.children. + // FIXME: https://github.com/servo/servo/issues/21481 self.queue_dedicated_media_source_failure_steps() }, } @@ -522,6 +645,11 @@ impl HTMLMediaElement { // https://html.spec.whatwg.org/multipage/#concept-media-load-resource fn resource_fetch_algorithm(&self, resource: Resource) { + if let Err(e) = self.setup_media_player() { + eprintln!("Setup media player error {:?}", e); + self.queue_dedicated_media_source_failure_steps(); + return; + } // Steps 1-2. // Unapplicable, the `resource` variable already conveys which mode // is in use. @@ -554,7 +682,8 @@ impl HTMLMediaElement { this.root().delay_load_event(false); }), window.upcast(), - ).unwrap(); + ) + .unwrap(); // Steps 4.remote.1.4. // FIXME(nox): Somehow we should wait for the task from previous @@ -600,9 +729,15 @@ impl HTMLMediaElement { listener.notify_fetch(message.to().unwrap()); }), ); - document - .loader_mut() - .fetch_async_background(request, action_sender); + let cancel_receiver = self.fetch_canceller.borrow_mut().initialize(); + let global = self.global(); + global + .core_resource_thread() + .send(CoreResourceMsg::Fetch( + request, + FetchChannels::ResponseMsg(action_sender, Some(cancel_receiver)), + )) + .unwrap(); }, Resource::Object => { // FIXME(nox): Actually do something with the object. @@ -642,11 +777,15 @@ impl HTMLMediaElement { this.network_state.set(NetworkState::NoSource); // Step 4. - // FIXME(nox): Set show poster flag to true. + this.show_poster.set(true); // Step 5. this.upcast::<EventTarget>().fire_event(atom!("error")); + if let Err(e) = this.player.stop() { + eprintln!("Could not stop player {:?}", e); + } + // Step 6. // Done after running this closure in // `fulfill_in_flight_play_promises`. @@ -688,7 +827,7 @@ impl HTMLMediaElement { task_source.queue_simple_event(self.upcast(), atom!("emptied"), &window); // Step 6.2. - // FIXME(nox): Abort in-progress fetching process. + self.fetch_canceller.borrow_mut().cancel(); // Step 6.3. // FIXME(nox): Detach MediaSource media provider object. @@ -722,7 +861,7 @@ impl HTMLMediaElement { // FIXME(nox): Set timeline offset to NaN. // Step 6.10. - // FIXME(nox): Set duration to NaN. + self.duration.set(f64::NAN); } // Step 7. @@ -805,6 +944,104 @@ impl HTMLMediaElement { } self.media_element_load_algorithm(); } + + // servo media player + fn setup_media_player(&self) -> Result<(), ServoMediaError>{ + let (action_sender, action_receiver) = ipc::channel().unwrap(); + + self.player.register_event_handler(action_sender)?; + self.player + .register_frame_renderer(self.frame_renderer.clone())?; + + let trusted_node = Trusted::new(self); + let window = window_from_node(self); + let task_source = window.dom_manipulation_task_source(); + let task_canceller = window.task_canceller(TaskSourceName::DOMManipulation); + ROUTER.add_route( + action_receiver.to_opaque(), + Box::new(move |message| { + let event: PlayerEvent = message.to().unwrap(); + let this = trusted_node.clone(); + task_source + .queue_with_canceller( + task!(handle_player_event: move || { + this.root().handle_player_event(&event); + }), + &task_canceller, + ) + .unwrap(); + }), + ); + + Ok(()) + } + + fn handle_player_event(&self, event: &PlayerEvent) { + match *event { + PlayerEvent::MetadataUpdated(ref metadata) => { + // https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list + // => "Once enough of the media data has been fetched to determine the duration..." + // Step 1. + // servo-media owns the media timeline. + + // Step 2. + // XXX(ferjm) Update the timeline offset. + + // Step 3. + // XXX(ferjm) Set the current and official playback positions + // to the earliest possible position. + + // Step 4. + if let Some(duration) = metadata.duration { + self.duration.set(duration.as_secs() as f64); + } else { + self.duration.set(f64::INFINITY); + } + let window = window_from_node(self); + let task_source = window.dom_manipulation_task_source(); + task_source.queue_simple_event(self.upcast(), atom!("durationchange"), &window); + + // Step 5. + if self.is::<HTMLVideoElement>() { + let video_elem = self.downcast::<HTMLVideoElement>().unwrap(); + video_elem.set_video_width(metadata.width); + video_elem.set_video_height(metadata.height); + task_source.queue_simple_event(self.upcast(), atom!("resize"), &window); + } + + // Step 6. + self.change_ready_state(ReadyState::HaveMetadata); + + // XXX(ferjm) Steps 7 to 13. + }, + PlayerEvent::StateChanged(ref state) => match *state { + PlaybackState::Paused => { + if self.ready_state.get() == ReadyState::HaveMetadata { + self.change_ready_state(ReadyState::HaveEnoughData); + } + }, + _ => {}, + }, + PlayerEvent::EndOfStream => { + // https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list + // => "If the media data can be fetched but is found by inspection to be in + // an unsupported format, or can otherwise not be rendered at all" + if self.ready_state.get() < ReadyState::HaveMetadata { + self.queue_dedicated_media_source_failure_steps(); + } + }, + PlayerEvent::FrameUpdated => { + self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage); + }, + PlayerEvent::Error => { + self.error.set(Some(&*MediaError::new( + &*window_from_node(self), + MEDIA_ERR_DECODE, + ))); + self.upcast::<EventTarget>().fire_event(atom!("error")); + }, + } + } } impl HTMLMediaElementMethods for HTMLMediaElement { @@ -894,6 +1131,11 @@ impl HTMLMediaElementMethods for HTMLMediaElement { fn Paused(&self) -> bool { self.paused.get() } + + // https://html.spec.whatwg.org/multipage/#dom-media-duration + fn Duration(&self) -> f64 { + self.duration.get() + } } impl VirtualMethods for HTMLMediaElement { @@ -927,6 +1169,20 @@ impl VirtualMethods for HTMLMediaElement { } } +pub trait LayoutHTMLMediaElementHelpers { + fn data(&self) -> HTMLMediaData; +} + +impl LayoutHTMLMediaElementHelpers for LayoutDom<HTMLMediaElement> { + #[allow(unsafe_code)] + fn data(&self) -> HTMLMediaData { + let media = unsafe { &*self.unsafe_get() }; + HTMLMediaData { + current_frame: media.frame_renderer.lock().unwrap().current_frame.clone(), + } + } +} + #[derive(JSTraceable, MallocSizeOf)] pub enum MediaElementMicrotask { ResourceSelectionTask { @@ -968,16 +1224,12 @@ enum Resource { struct HTMLMediaElementContext { /// The element that initiated the request. elem: Trusted<HTMLMediaElement>, - /// The response body received to date. - data: Vec<u8>, /// The response metadata received to date. metadata: Option<Metadata>, /// The generation of the media element when this fetch started. generation_id: u32, /// Time of last progress notification. next_progress_event: Timespec, - /// Whether the media metadata has been completely received. - have_metadata: bool, /// True if this response is invalid and should be ignored. ignore_response: bool, } @@ -994,6 +1246,16 @@ impl FetchResponseListener for HTMLMediaElementContext { FetchMetadata::Filtered { unsafe_, .. } => unsafe_, }); + if let Some(metadata) = self.metadata.as_ref() { + if let Some(headers) = metadata.headers.as_ref() { + if let Some(content_length) = headers.get::<ContentLength>() { + if let Err(e) = self.elem.root().player.set_input_size(**content_length) { + eprintln!("Could not set player input size {:?}", e); + } + } + } + } + let status_is_ok = self .metadata .as_ref() @@ -1005,28 +1267,24 @@ impl FetchResponseListener for HTMLMediaElementContext { // Ensure that the element doesn't receive any further notifications // of the aborted fetch. self.ignore_response = true; - self.elem - .root() - .queue_dedicated_media_source_failure_steps(); + let elem = self.elem.root(); + elem.fetch_canceller.borrow_mut().cancel(); + elem.queue_dedicated_media_source_failure_steps(); } } - fn process_response_chunk(&mut self, mut payload: Vec<u8>) { + fn process_response_chunk(&mut self, payload: Vec<u8>) { if self.ignore_response { // An error was received previously, skip processing the payload. return; } - self.data.append(&mut payload); - let elem = self.elem.root(); - // https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list - // => "Once enough of the media data has been fetched to determine the duration..." - if !self.have_metadata { - self.check_metadata(&elem); - } else { - elem.change_ready_state(ReadyState::HaveCurrentData); + // Push input data into the player. + if let Err(e) = elem.player.push_data(payload) { + eprintln!("Could not push input data to player {:?}", e); + return; } // https://html.spec.whatwg.org/multipage/#concept-media-load-resource step 4, @@ -1050,13 +1308,17 @@ impl FetchResponseListener for HTMLMediaElementContext { } let elem = self.elem.root(); - // => "If the media data can be fetched but is found by inspection to be in an unsupported - // format, or can otherwise not be rendered at all" - if !self.have_metadata { - elem.queue_dedicated_media_source_failure_steps(); + // Signal the eos to player. + if let Err(e) = elem.player.end_of_stream() { + eprintln!("Could not signal EOS to player {:?}", e); } - // => "Once the entire media resource has been fetched..." - else if status.is_ok() { + + if status.is_ok() { + if elem.ready_state.get() == ReadyState::HaveNothing { + // Make sure that we don't skip the HaveMetadata and HaveCurrentData + // states for short streams. + elem.change_ready_state(ReadyState::HaveMetadata); + } elem.change_ready_state(ReadyState::HaveEnoughData); elem.upcast::<EventTarget>().fire_event(atom!("progress")); @@ -1067,6 +1329,9 @@ impl FetchResponseListener for HTMLMediaElementContext { } // => "If the connection is interrupted after some media data has been received..." else if elem.ready_state.get() != ReadyState::HaveNothing { + // Step 1 + elem.fetch_canceller.borrow_mut().cancel(); + // Step 2 elem.error.set(Some(&*MediaError::new( &*window_from_node(&*elem), @@ -1099,20 +1364,10 @@ impl HTMLMediaElementContext { fn new(elem: &HTMLMediaElement) -> HTMLMediaElementContext { HTMLMediaElementContext { elem: Trusted::new(elem), - data: vec![], metadata: None, generation_id: elem.generation_id.get(), next_progress_event: time::get_time() + Duration::milliseconds(350), - have_metadata: false, ignore_response: false, } } - - fn check_metadata(&mut self, elem: &HTMLMediaElement) { - if audio_video_metadata::get_format_from_slice(&self.data).is_ok() { - // Step 6. - elem.change_ready_state(ReadyState::HaveMetadata); - self.have_metadata = true; - } - } } diff --git a/components/script/dom/htmlvideoelement.rs b/components/script/dom/htmlvideoelement.rs index 8561aa051e1..59c93bfea6e 100644 --- a/components/script/dom/htmlvideoelement.rs +++ b/components/script/dom/htmlvideoelement.rs @@ -3,16 +3,22 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use dom::bindings::codegen::Bindings::HTMLVideoElementBinding; +use dom::bindings::codegen::Bindings::HTMLVideoElementBinding::HTMLVideoElementMethods; use dom::bindings::root::DomRoot; use dom::document::Document; -use dom::htmlmediaelement::HTMLMediaElement; +use dom::htmlmediaelement::{HTMLMediaElement, ReadyState}; use dom::node::Node; use dom_struct::dom_struct; use html5ever::{LocalName, Prefix}; +use std::cell::Cell; #[dom_struct] pub struct HTMLVideoElement { htmlmediaelement: HTMLMediaElement, + /// https://html.spec.whatwg.org/multipage/#dom-video-videowidth + video_width: Cell<u32>, + /// https://html.spec.whatwg.org/multipage/#dom-video-videoheight + video_height: Cell<u32>, } impl HTMLVideoElement { @@ -23,6 +29,8 @@ impl HTMLVideoElement { ) -> HTMLVideoElement { HTMLVideoElement { htmlmediaelement: HTMLMediaElement::new_inherited(local_name, prefix, document), + video_width: Cell::new(0), + video_height: Cell::new(0), } } @@ -40,4 +48,30 @@ impl HTMLVideoElement { HTMLVideoElementBinding::Wrap, ) } + + pub fn set_video_width(&self, width: u32) { + self.video_width.set(width); + } + + pub fn set_video_height(&self, height: u32) { + self.video_height.set(height); + } +} + +impl HTMLVideoElementMethods for HTMLVideoElement { + // https://html.spec.whatwg.org/multipage/#dom-video-videowidth + fn VideoWidth(&self) -> u32 { + if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing { + return 0; + } + self.video_width.get() + } + + // https://html.spec.whatwg.org/multipage/#dom-video-videoheight + fn VideoHeight(&self) -> u32 { + if self.htmlmediaelement.get_ready_state() == ReadyState::HaveNothing { + return 0; + } + self.video_height.get() + } } diff --git a/components/script/dom/node.rs b/components/script/dom/node.rs index 8fa494767f6..7899e494341 100644 --- a/components/script/dom/node.rs +++ b/components/script/dom/node.rs @@ -43,6 +43,7 @@ use dom::htmliframeelement::{HTMLIFrameElement, HTMLIFrameElementLayoutMethods}; use dom::htmlimageelement::{HTMLImageElement, LayoutHTMLImageElementHelpers}; use dom::htmlinputelement::{HTMLInputElement, LayoutHTMLInputElementHelpers}; use dom::htmllinkelement::HTMLLinkElement; +use dom::htmlmediaelement::{HTMLMediaElement, LayoutHTMLMediaElementHelpers}; use dom::htmlmetaelement::HTMLMetaElement; use dom::htmlstyleelement::HTMLStyleElement; use dom::htmltextareaelement::{HTMLTextAreaElement, LayoutHTMLTextAreaElementHelpers}; @@ -62,8 +63,8 @@ use libc::{self, c_void, uintptr_t}; use malloc_size_of::{MallocSizeOf, MallocSizeOfOps}; use msg::constellation_msg::{BrowsingContextId, PipelineId}; use ref_slice::ref_slice; -use script_layout_interface::{HTMLCanvasData, OpaqueStyleAndLayoutData, SVGSVGData}; -use script_layout_interface::{LayoutElementType, LayoutNodeType, TrustedNodeAddress}; +use script_layout_interface::{HTMLCanvasData, HTMLMediaData, LayoutElementType, LayoutNodeType}; +use script_layout_interface::{OpaqueStyleAndLayoutData, SVGSVGData, TrustedNodeAddress}; use script_layout_interface::message::Msg; use script_thread::ScriptThread; use script_traits::DocumentActivity; @@ -1086,6 +1087,7 @@ pub trait LayoutNodeHelpers { fn image_url(&self) -> Option<ServoUrl>; fn image_density(&self) -> Option<f64>; fn canvas_data(&self) -> Option<HTMLCanvasData>; + fn media_data(&self) -> Option<HTMLMediaData>; fn svg_data(&self) -> Option<SVGSVGData>; fn iframe_browsing_context_id(&self) -> Option<BrowsingContextId>; fn iframe_pipeline_id(&self) -> Option<PipelineId>; @@ -1245,6 +1247,11 @@ impl LayoutNodeHelpers for LayoutDom<Node> { .map(|canvas| canvas.data()) } + fn media_data(&self) -> Option<HTMLMediaData> { + self.downcast::<HTMLMediaElement>() + .map(|media| media.data()) + } + fn svg_data(&self) -> Option<SVGSVGData> { self.downcast::<SVGSVGElement>().map(|svg| svg.data()) } @@ -2911,6 +2918,9 @@ impl Into<LayoutElementType> for ElementTypeId { ElementTypeId::HTMLElement(HTMLElementTypeId::HTMLImageElement) => { LayoutElementType::HTMLImageElement }, + ElementTypeId::HTMLElement(HTMLElementTypeId::HTMLMediaElement(_)) => { + LayoutElementType::HTMLMediaElement + }, ElementTypeId::HTMLElement(HTMLElementTypeId::HTMLInputElement) => { LayoutElementType::HTMLInputElement }, diff --git a/components/script/dom/webidls/HTMLMediaElement.webidl b/components/script/dom/webidls/HTMLMediaElement.webidl index b67a5e3c1fe..e00b1de7c4f 100644 --- a/components/script/dom/webidls/HTMLMediaElement.webidl +++ b/components/script/dom/webidls/HTMLMediaElement.webidl @@ -39,7 +39,7 @@ interface HTMLMediaElement : HTMLElement { // playback state // attribute double currentTime; // void fastSeek(double time); - // readonly attribute unrestricted double duration; + readonly attribute unrestricted double duration; // Date getStartDate(); readonly attribute boolean paused; // attribute double defaultPlaybackRate; diff --git a/components/script/dom/webidls/HTMLVideoElement.webidl b/components/script/dom/webidls/HTMLVideoElement.webidl index bfd1be006ea..a3a34e7c47e 100644 --- a/components/script/dom/webidls/HTMLVideoElement.webidl +++ b/components/script/dom/webidls/HTMLVideoElement.webidl @@ -9,8 +9,8 @@ interface HTMLVideoElement : HTMLMediaElement { // attribute unsigned long width; // [CEReactions] // attribute unsigned long height; - // readonly attribute unsigned long videoWidth; - // readonly attribute unsigned long videoHeight; + readonly attribute unsigned long videoWidth; + readonly attribute unsigned long videoHeight; // [CEReactions] // attribute DOMString poster; }; diff --git a/components/script/dom/window.rs b/components/script/dom/window.rs index ff905ab55b5..c50aae0aec7 100644 --- a/components/script/dom/window.rs +++ b/components/script/dom/window.rs @@ -132,7 +132,7 @@ use time; use timers::{IsInterval, TimerCallback}; use url::Position; use webdriver_handlers::jsval_to_webdriver; -use webrender_api::{ExternalScrollId, DeviceIntPoint, DeviceUintSize, DocumentId}; +use webrender_api::{DeviceIntPoint, DeviceUintSize, DocumentId, ExternalScrollId, RenderApiSender}; use webvr_traits::WebVRMsg; /// Current state of the window object @@ -308,6 +308,9 @@ pub struct Window { /// Flag to identify whether mutation observers are present(true)/absent(false) exists_mut_observer: Cell<bool>, + /// Webrender API Sender + #[ignore_malloc_size_of = "defined in webrender_api"] + webrender_api_sender: RenderApiSender, } impl Window { @@ -483,6 +486,10 @@ impl Window { } self.add_pending_reflow(); } + + pub fn get_webrender_api_sender(&self) -> RenderApiSender { + self.webrender_api_sender.clone() + } } // https://html.spec.whatwg.org/multipage/#atob @@ -2083,6 +2090,7 @@ impl Window { webvr_chan: Option<IpcSender<WebVRMsg>>, microtask_queue: Rc<MicrotaskQueue>, webrender_document: DocumentId, + webrender_api_sender: RenderApiSender, ) -> DomRoot<Self> { let layout_rpc: Box<LayoutRPC + Send> = { let (rpc_send, rpc_recv) = channel(); @@ -2161,6 +2169,7 @@ impl Window { paint_worklet: Default::default(), webrender_document, exists_mut_observer: Cell::new(false), + webrender_api_sender, }); unsafe { WindowBinding::Wrap(runtime.cx(), win) } diff --git a/components/script/lib.rs b/components/script/lib.rs index 097f69d7808..eda29279bae 100644 --- a/components/script/lib.rs +++ b/components/script/lib.rs @@ -18,7 +18,6 @@ )] extern crate app_units; -extern crate audio_video_metadata; #[cfg(any(feature = "webgl_backtrace", feature = "js_backtrace"))] extern crate backtrace; extern crate base64; diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 748b6fc63e4..687dd32e1df 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -130,7 +130,7 @@ use time::{get_time, precise_time_ns, Tm}; use url::Position; use url::percent_encoding::percent_decode; use webdriver_handlers; -use webrender_api::DocumentId; +use webrender_api::{DocumentId, RenderApiSender}; use webvr_traits::{WebVREvent, WebVRMsg}; pub type ImageCacheMsg = (PipelineId, PendingImageResponse); @@ -591,6 +591,9 @@ pub struct ScriptThread { /// The Webrender Document ID associated with this thread. webrender_document: DocumentId, + + /// FIXME(victor): + webrender_api_sender: RenderApiSender, } /// In the event of thread panic, all data on the stack runs its destructor. However, there @@ -1063,6 +1066,7 @@ impl ScriptThread { custom_element_reaction_stack: CustomElementReactionStack::new(), webrender_document: state.webrender_document, + webrender_api_sender: state.webrender_api_sender, } } @@ -2584,6 +2588,7 @@ impl ScriptThread { self.webvr_chan.clone(), self.microtask_queue.clone(), self.webrender_document, + self.webrender_api_sender.clone(), ); // Initialize the browsing context for the window. diff --git a/components/script_layout_interface/lib.rs b/components/script_layout_interface/lib.rs index da1ab322fa9..bd48b2b7c51 100644 --- a/components/script_layout_interface/lib.rs +++ b/components/script_layout_interface/lib.rs @@ -115,6 +115,7 @@ pub enum LayoutElementType { HTMLIFrameElement, HTMLImageElement, HTMLInputElement, + HTMLMediaElement, HTMLObjectElement, HTMLParagraphElement, HTMLTableCellElement, @@ -170,3 +171,7 @@ pub struct PendingImage { pub node: UntrustedNodeAddress, pub id: PendingImageId, } + +pub struct HTMLMediaData { + pub current_frame: Option<(webrender_api::ImageKey, i32, i32)>, +} diff --git a/components/script_layout_interface/wrapper_traits.rs b/components/script_layout_interface/wrapper_traits.rs index 32cb4853fa4..9df6a7b44f4 100644 --- a/components/script_layout_interface/wrapper_traits.rs +++ b/components/script_layout_interface/wrapper_traits.rs @@ -5,6 +5,7 @@ #![allow(unsafe_code)] use HTMLCanvasData; +use HTMLMediaData; use LayoutNodeType; use OpaqueStyleAndLayoutData; use SVGSVGData; @@ -279,6 +280,8 @@ pub trait ThreadSafeLayoutNode: fn svg_data(&self) -> Option<SVGSVGData>; + fn media_data(&self) -> Option<HTMLMediaData>; + /// If this node is an iframe element, returns its browsing context ID. If this node is /// not an iframe element, fails. Returns None if there is no nested browsing context. fn iframe_browsing_context_id(&self) -> Option<BrowsingContextId>; diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 6c087690b5f..802e61ea172 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -72,7 +72,7 @@ use style_traits::CSSPixel; use style_traits::SpeculativePainter; use style_traits::cursor::CursorKind; use webdriver_msg::{LoadStatus, WebDriverScriptCommand}; -use webrender_api::{ExternalScrollId, DevicePixel, DeviceUintSize, DocumentId, ImageKey}; +use webrender_api::{DevicePixel, DeviceUintSize, DocumentId, ExternalScrollId, ImageKey, RenderApiSender}; use webvr_traits::{WebVREvent, WebVRMsg}; pub use script_msg::{LayoutMsg, ScriptMsg, EventResult, LogEntry}; @@ -588,6 +588,8 @@ pub struct InitialScriptState { pub webvr_chan: Option<IpcSender<WebVRMsg>>, /// The Webrender document ID associated with this thread. pub webrender_document: DocumentId, + /// FIXME(victor): The Webrender API sender in this constellation's pipeline + pub webrender_api_sender: RenderApiSender, } /// This trait allows creating a `ScriptThread` without depending on the `script` diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 908b813875b..da27cb4db1f 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -3,6 +3,7 @@ env: RUSTFLAGS: -Dwarnings CARGO_INCREMENTAL: "0" SCCACHE_IDLE_TIMEOUT: "1200" + GST_DEBUG: '3' mac-rel-wpt1: - ./mach clean-nightlies --keep 3 --force diff --git a/tests/wpt/metadata/html/dom/elements/the-innertext-idl-attribute/getter.html.ini b/tests/wpt/metadata/html/dom/elements/the-innertext-idl-attribute/getter.html.ini index ba5ca706e12..b87d4cead69 100644 --- a/tests/wpt/metadata/html/dom/elements/the-innertext-idl-attribute/getter.html.ini +++ b/tests/wpt/metadata/html/dom/elements/the-innertext-idl-attribute/getter.html.ini @@ -53,15 +53,9 @@ [opacity:0 child rendered ("<div>123<span style='opacity:0'>abc")] expected: FAIL - [<audio> contents ignored ("<audio style='display:block'>abc")] - expected: FAIL - [<audio> contents ignored ("<audio style='display:block'><source id='target' class='poke' style='display:block'>")] expected: FAIL - [<video> contents ignored ("<video>abc")] - expected: FAIL - [<video> contents ignored ("<video style='display:block'><source id='target' class='poke' style='display:block'>")] expected: FAIL @@ -104,12 +98,6 @@ [<select size='2'> contents of options preserved ("<div><select size='2'><option>abc</option><option>def")] expected: FAIL - [<audio> contents ignored ("<div><audio>abc")] - expected: FAIL - - [<video> contents ignored ("<div><video>abc")] - expected: FAIL - [Blank lines between <p>s separated by non-empty block ("<div><p>abc</p><div>123</div><p>def")] expected: FAIL @@ -260,3 +248,12 @@ [Whitespace around inline-block should not be collapsed ("<div>abc <span style='display:inline-block'></span> def")] expected: FAIL + [text-transform handles Turkish casing ("<div><div lang='tr' style='text-transform:uppercase'>i ı")] + expected: FAIL + + [<video> contents ok for element not being rendered ("<video style='display:block'><source id='target' class='poke' style='display:block'>")] + expected: FAIL + + [<audio> contents ok for element not being rendered ("<audio style='display:block'><source id='target' class='poke' style='display:block'>")] + expected: FAIL + diff --git a/tests/wpt/metadata/html/dom/interfaces.https.html.ini b/tests/wpt/metadata/html/dom/interfaces.https.html.ini index 3ddf0903edc..46e3fadd05e 100644 --- a/tests/wpt/metadata/html/dom/interfaces.https.html.ini +++ b/tests/wpt/metadata/html/dom/interfaces.https.html.ini @@ -6786,12 +6786,6 @@ [HTMLVideoElement interface: attribute height] expected: FAIL - [HTMLVideoElement interface: attribute videoWidth] - expected: FAIL - - [HTMLVideoElement interface: attribute videoHeight] - expected: FAIL - [HTMLVideoElement interface: attribute poster] expected: FAIL @@ -6804,12 +6798,6 @@ [HTMLVideoElement interface: document.createElement("video") must inherit property "height" with the proper type] expected: FAIL - [HTMLVideoElement interface: document.createElement("video") must inherit property "videoWidth" with the proper type] - expected: FAIL - - [HTMLVideoElement interface: document.createElement("video") must inherit property "videoHeight" with the proper type] - expected: FAIL - [HTMLVideoElement interface: document.createElement("video") must inherit property "poster" with the proper type] expected: FAIL @@ -6837,9 +6825,6 @@ [HTMLMediaElement interface: calling fastSeek(double) on document.createElement("video") with too few arguments must throw TypeError] expected: FAIL - [HTMLMediaElement interface: document.createElement("video") must inherit property "duration" with the proper type] - expected: FAIL - [HTMLMediaElement interface: document.createElement("video") must inherit property "getStartDate()" with the proper type] expected: FAIL @@ -6909,9 +6894,6 @@ [HTMLMediaElement interface: calling fastSeek(double) on document.createElement("audio") with too few arguments must throw TypeError] expected: FAIL - [HTMLMediaElement interface: document.createElement("audio") must inherit property "duration" with the proper type] - expected: FAIL - [HTMLMediaElement interface: document.createElement("audio") must inherit property "getStartDate()" with the proper type] expected: FAIL @@ -7197,9 +7179,6 @@ [HTMLMediaElement interface: operation fastSeek(double)] expected: FAIL - [HTMLMediaElement interface: attribute duration] - expected: FAIL - [HTMLMediaElement interface: operation getStartDate()] expected: FAIL diff --git a/tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_timeupdate_noautoplay.html.ini b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_timeupdate_noautoplay.html.ini index 22d1f1d6900..30b44071f96 100644 --- a/tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_timeupdate_noautoplay.html.ini +++ b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/event_timeupdate_noautoplay.html.ini @@ -1,5 +1,7 @@ [event_timeupdate_noautoplay.html] type: testharness + disabled: + if os == "mac": https://github.com/servo/saltfs/pull/898 expected: TIMEOUT [calling play() on a sufficiently long audio should trigger timeupdate event] expected: NOTRUN diff --git a/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/load-events-networkState.html.ini b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/load-events-networkState.html.ini new file mode 100644 index 00000000000..95be7126728 --- /dev/null +++ b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/load-events-networkState.html.ini @@ -0,0 +1,4 @@ +[load-events-networkState.html] + [NETWORK_NO_SOURCE] + expected: FAIL + diff --git a/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-insert-into-iframe.html.ini b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-insert-into-iframe.html.ini new file mode 100644 index 00000000000..dd5f01686e9 --- /dev/null +++ b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-insert-into-iframe.html.ini @@ -0,0 +1,4 @@ +[resource-selection-invoke-insert-into-iframe.html] + [NOT invoking resource selection by inserting into other document with src set] + expected: FAIL + diff --git a/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-pause-networkState.html.ini b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-pause-networkState.html.ini new file mode 100644 index 00000000000..80becaced04 --- /dev/null +++ b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-pause-networkState.html.ini @@ -0,0 +1,4 @@ +[resource-selection-invoke-pause-networkState.html] + [NOT invoking resource selection with pause() when networkState is not NETWORK_EMPTY] + expected: FAIL + diff --git a/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-remove-from-document-networkState.html.ini b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-remove-from-document-networkState.html.ini new file mode 100644 index 00000000000..cbd728cd5fd --- /dev/null +++ b/tests/wpt/metadata/html/semantics/embedded-content/media-elements/loading-the-media-resource/resource-selection-invoke-remove-from-document-networkState.html.ini @@ -0,0 +1,4 @@ +[resource-selection-invoke-remove-from-document-networkState.html] + [NOT invoking resource selection with implicit pause() when networkState is not NETWORK_EMPTY] + expected: FAIL + |