MediaStream playback through audio and video elements

4 files changed, 101 insertions, 61 deletions

diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs
index 4ac3505be84..7a0498c5e96 100644
--- a/components/script/dom/bindings/trace.rs
+++ b/components/script/dom/bindings/trace.rs
@@ -104,7 +104,7 @@ use servo_media::audio::graph::NodeId;
 use servo_media::audio::panner_node::{DistanceModel, PanningModel};
 use servo_media::audio::param::ParamType;
 use servo_media::player::Player;
-use servo_media::streams::MediaStream as BackendMediaStream;
+use servo_media::streams::registry::MediaStreamId;
 use servo_media::webrtc::WebRtcController;
 use servo_url::{ImmutableOrigin, MutableOrigin, ServoUrl};
 use smallvec::SmallVec;
@@ -488,7 +488,7 @@ unsafe_no_jsmanaged_fields!(NodeId);
 unsafe_no_jsmanaged_fields!(AnalysisEngine, DistanceModel, PanningModel, ParamType);
 unsafe_no_jsmanaged_fields!(dyn Player);
 unsafe_no_jsmanaged_fields!(WebRtcController);
-unsafe_no_jsmanaged_fields!(dyn BackendMediaStream);
+unsafe_no_jsmanaged_fields!(MediaStreamId);
 unsafe_no_jsmanaged_fields!(Mutex<MediaFrameRenderer>);
 unsafe_no_jsmanaged_fields!(RenderApiSender);
 unsafe_no_jsmanaged_fields!(ResourceFetchTiming);
diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs
index dbc952688b1..e89389fd253 100644
--- a/components/script/dom/htmlmediaelement.rs
+++ b/components/script/dom/htmlmediaelement.rs
@@ -215,7 +215,7 @@ pub struct HTMLMediaElement {
     #[ignore_malloc_size_of = "promises are hard"]
     in_flight_play_promises_queue: DomRefCell<VecDeque<(Box<[Rc<Promise>]>, ErrorResult)>>,
     #[ignore_malloc_size_of = "servo_media"]
-    player: Box<Player>,
+    player: DomRefCell<Option<Box<Player>>>,
     #[ignore_malloc_size_of = "Arc"]
     frame_renderer: Arc<Mutex<MediaFrameRenderer>>,
     /// https://html.spec.whatwg.org/multipage/#show-poster-flag
@@ -295,7 +295,7 @@ impl HTMLMediaElement {
             delaying_the_load_event_flag: Default::default(),
             pending_play_promises: Default::default(),
             in_flight_play_promises_queue: Default::default(),
-            player: ServoMedia::get().unwrap().create_player(),
+            player: Default::default(),
             frame_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new(
                 document.window().get_webrender_api_sender(),
             ))),
@@ -330,11 +330,13 @@ impl HTMLMediaElement {
     }
 
     fn play_media(&self) {
-        if let Err(e) = self.player.set_rate(self.playbackRate.get()) {
-            warn!("Could not set the playback rate {:?}", e);
-        }
-        if let Err(e) = self.player.play() {
-            warn!("Could not play media {:?}", e);
+        if let Some(ref player) = *self.player.borrow() {
+            if let Err(e) = player.set_rate(self.playbackRate.get()) {
+                warn!("Could not set the playback rate {:?}", e);
+            }
+            if let Err(e) = player.play() {
+                warn!("Could not play media {:?}", e);
+            }
         }
     }
 
@@ -398,8 +400,10 @@ impl HTMLMediaElement {
                         // Step 2.3.2.
                         this.upcast::<EventTarget>().fire_event(atom!("pause"));
 
-                        if let Err(e) = this.player.pause() {
-                            eprintln!("Could not pause player {:?}", e);
+                        if let Some(ref player) = *this.player.borrow() {
+                            if let Err(e) = player.pause() {
+                                eprintln!("Could not pause player {:?}", e);
+                            }
                         }
 
                         // Step 2.3.3.
@@ -762,7 +766,7 @@ impl HTMLMediaElement {
 
     // https://html.spec.whatwg.org/multipage/#concept-media-load-resource
     fn resource_fetch_algorithm(&self, resource: Resource) {
-        if let Err(e) = self.setup_media_player() {
+        if let Err(e) = self.setup_media_player(&resource) {
             eprintln!("Setup media player error {:?}", e);
             self.queue_dedicated_media_source_failure_steps();
             return;
@@ -827,8 +831,14 @@ impl HTMLMediaElement {
                                 Some(ServoUrl::parse(&blob_url).expect("infallible"));
                             self.fetch_request(None);
                         },
-                        SrcObject::MediaStream(_) => {
-                            self.queue_dedicated_media_source_failure_steps();
+                        SrcObject::MediaStream(ref stream) => {
+                            for stream in stream.get_tracks() {
+                                if let Err(_) =
+                                    self.player.borrow().as_ref().unwrap().set_stream(&stream)
+                                {
+                                    self.queue_dedicated_media_source_failure_steps();
+                                }
+                            }
                         },
                     }
                 }
@@ -872,8 +882,10 @@ impl HTMLMediaElement {
                     // Step 5.
                     this.upcast::<EventTarget>().fire_event(atom!("error"));
 
-                    if let Err(e) = this.player.stop() {
-                        eprintln!("Could not stop player {:?}", e);
+                    if let Some(ref player) = *this.player.borrow() {
+                        if let Err(e) = player.stop() {
+                            eprintln!("Could not stop player {:?}", e);
+                        }
                     }
 
                     // Step 6.
@@ -1126,8 +1138,10 @@ impl HTMLMediaElement {
         task_source.queue_simple_event(self.upcast(), atom!("seeking"), &window);
 
         // Step 11.
-        if let Err(e) = self.player.seek(time) {
-            eprintln!("Seek error {:?}", e);
+        if let Some(ref player) = *self.player.borrow() {
+            if let Err(e) = player.seek(time) {
+                eprintln!("Seek error {:?}", e);
+            }
         }
 
         // The rest of the steps are handled when the media engine signals a
@@ -1175,12 +1189,28 @@ impl HTMLMediaElement {
         }
     }
 
-    fn setup_media_player(&self) -> Result<(), PlayerError> {
+    fn setup_media_player(&self, resource: &Resource) -> Result<(), ()> {
+        let stream_type = match *resource {
+            Resource::Object => {
+                if let Some(ref src_object) = *self.src_object.borrow() {
+                    match src_object {
+                        SrcObject::MediaStream(_) => StreamType::Stream,
+                        _ => StreamType::Seekable,
+                    }
+                } else {
+                    return Err(());
+                }
+            },
+            _ => StreamType::Seekable,
+        };
+
+        let player = ServoMedia::get().unwrap().create_player(stream_type);
+
         let (action_sender, action_receiver) = ipc::channel().unwrap();
+        player.register_event_handler(action_sender);
+        player.register_frame_renderer(self.frame_renderer.clone());
 
-        self.player.register_event_handler(action_sender);
-        self.player
-            .register_frame_renderer(self.frame_renderer.clone());
+        *self.player.borrow_mut() = Some(player);
 
         let trusted_node = Trusted::new(self);
         let window = window_from_node(self);
@@ -1536,8 +1566,10 @@ impl HTMLMediaElement {
 
 impl Drop for HTMLMediaElement {
     fn drop(&mut self) {
-        if let Err(err) = self.player.shutdown() {
-            warn!("Error shutting down player {:?}", err);
+        if let Some(ref player) = *self.player.borrow() {
+            if let Err(err) = player.shutdown() {
+                warn!("Error shutting down player {:?}", err);
+            }
         }
     }
 }
@@ -1579,15 +1611,17 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
         if self.muted.get() == value {
             return;
         }
-        self.muted.set(value);
-        let _ = self.player.set_mute(value);
-        let window = window_from_node(self);
-        window
-            .task_manager()
-            .media_element_task_source()
-            .queue_simple_event(self.upcast(), atom!("volumechange"), &window);
-        if !self.is_allowed_to_play() {
-            self.internal_pause_steps();
+        if let Some(ref player) = *self.player.borrow() {
+            self.muted.set(value);
+            let _ = player.set_mute(value);
+            let window = window_from_node(self);
+            window
+                .task_manager()
+                .media_element_task_source()
+                .queue_simple_event(self.upcast(), atom!("volumechange"), &window);
+            if !self.is_allowed_to_play() {
+                self.internal_pause_steps();
+            }
         }
     }
 
@@ -1790,8 +1824,10 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
             self.playbackRate.set(*value);
             self.queue_ratechange_event();
             if self.is_potentially_playing() {
-                if let Err(e) = self.player.set_rate(*value) {
-                    warn!("Could not set the playback rate {:?}", e);
+                if let Some(ref player) = *self.player.borrow() {
+                    if let Err(e) = player.set_rate(*value) {
+                        warn!("Could not set the playback rate {:?}", e);
+                    }
                 }
             }
         }
@@ -1855,9 +1891,11 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
     // https://html.spec.whatwg.org/multipage/#dom-media-buffered
     fn Buffered(&self) -> DomRoot<TimeRanges> {
         let mut buffered = TimeRangesContainer::new();
-        if let Ok(ranges) = self.player.buffered() {
-            for range in ranges {
-                let _ = buffered.add(range.start as f64, range.end as f64);
+        if let Some(ref player) = *self.player.borrow() {
+            if let Ok(ranges) = player.buffered() {
+                for range in ranges {
+                    let _ = buffered.add(range.start as f64, range.end as f64);
+                }
             }
         }
         TimeRanges::new(self.global().as_window(), buffered)
@@ -2129,7 +2167,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
     fn process_response(&mut self, metadata: Result<FetchMetadata, NetworkError>) {
         let elem = self.elem.root();
 
-        if elem.generation_id.get() != self.generation_id {
+        if elem.generation_id.get() != self.generation_id || elem.player.borrow().is_none() {
             // A new fetch request was triggered, so we ignore this response.
             return;
         }
@@ -2155,7 +2193,13 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
                 // We only set the expected input size if it changes.
                 if content_length != self.expected_content_length {
                     if let Some(content_length) = content_length {
-                        if let Err(e) = elem.player.set_input_size(content_length) {
+                        if let Err(e) = elem
+                            .player
+                            .borrow()
+                            .as_ref()
+                            .unwrap()
+                            .set_input_size(content_length)
+                        {
                             warn!("Could not set player input size {:?}", e);
                         } else {
                             self.expected_content_length = Some(content_length);
@@ -2178,10 +2222,6 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
             if let Some(ref mut current_fetch_context) = *elem.current_fetch_context.borrow_mut() {
                 current_fetch_context.set_seekable(true);
             }
-            // and we can safely set the type of stream to Seekable.
-            if let Err(e) = elem.player.set_stream_type(StreamType::Seekable) {
-                warn!("Could not set stream type to Seekable. {:?}", e);
-            }
         }
 
         // => "If the media data cannot be fetched at all..."
@@ -2199,7 +2239,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
         let elem = self.elem.root();
         // If an error was received previously or if we triggered a new fetch request,
         // we skip processing the payload.
-        if elem.generation_id.get() != self.generation_id {
+        if elem.generation_id.get() != self.generation_id || elem.player.borrow().is_none() {
             return;
         }
         if let Some(ref current_fetch_context) = *elem.current_fetch_context.borrow() {
@@ -2211,7 +2251,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
         let payload_len = payload.len() as u64;
 
         // Push input data into the player.
-        if let Err(e) = elem.player.push_data(payload) {
+        if let Err(e) = elem.player.borrow().as_ref().unwrap().push_data(payload) {
             // If we are pushing too much data and we know that we can
             // restart the download later from where we left, we cancel
             // the current request. Otherwise, we continue the request
@@ -2247,13 +2287,18 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
     // https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list
     fn process_response_eof(&mut self, status: Result<ResourceFetchTiming, NetworkError>) {
         let elem = self.elem.root();
+
+        if elem.player.borrow().is_none() {
+            return;
+        }
+
         // If an error was previously received and no new fetch request was triggered,
         // we skip processing the payload and notify the media backend that we are done
         // pushing data.
         if elem.generation_id.get() == self.generation_id {
             if let Some(ref current_fetch_context) = *elem.current_fetch_context.borrow() {
                 if let Some(CancelReason::Error) = current_fetch_context.cancel_reason() {
-                    if let Err(e) = elem.player.end_of_stream() {
+                    if let Err(e) = elem.player.borrow().as_ref().unwrap().end_of_stream() {
                         warn!("Could not signal EOS to player {:?}", e);
                     }
                     return;
diff --git a/components/script/dom/mediastream.rs b/components/script/dom/mediastream.rs
index 787c266637a..42588afc476 100644
--- a/components/script/dom/mediastream.rs
+++ b/components/script/dom/mediastream.rs
@@ -9,25 +9,24 @@ use crate::dom::bindings::root::DomRoot;
 use crate::dom::eventtarget::EventTarget;
 use crate::dom::globalscope::GlobalScope;
 use dom_struct::dom_struct;
-use servo_media::streams::MediaStream as BackendMediaStream;
-use std::mem;
+use servo_media::streams::registry::MediaStreamId;
 
 #[dom_struct]
 pub struct MediaStream {
     eventtarget: EventTarget,
     #[ignore_malloc_size_of = "defined in servo-media"]
-    tracks: DomRefCell<Vec<Box<BackendMediaStream>>>,
+    tracks: DomRefCell<Vec<MediaStreamId>>,
 }
 
 impl MediaStream {
-    pub fn new_inherited(tracks: Vec<Box<BackendMediaStream>>) -> MediaStream {
+    pub fn new_inherited(tracks: Vec<MediaStreamId>) -> MediaStream {
         MediaStream {
             eventtarget: EventTarget::new_inherited(),
             tracks: DomRefCell::new(tracks),
         }
     }
 
-    pub fn new(global: &GlobalScope, tracks: Vec<Box<BackendMediaStream>>) -> DomRoot<MediaStream> {
+    pub fn new(global: &GlobalScope, tracks: Vec<MediaStreamId>) -> DomRoot<MediaStream> {
         reflect_dom_object(
             Box::new(MediaStream::new_inherited(tracks)),
             global,
@@ -35,11 +34,7 @@ impl MediaStream {
         )
     }
 
-    pub fn get_tracks(&self) -> Vec<Box<BackendMediaStream>> {
-        // XXXManishearth we have hard ownership constraints here so we actually empty the vec,
-        // ideally we should only have a media stream id here, or servo-media hands
-        // out Arcs
-        let mut tracks = self.tracks.borrow_mut();
-        mem::replace(&mut *tracks, vec![])
+    pub fn get_tracks(&self) -> Vec<MediaStreamId> {
+        self.tracks.borrow_mut().clone()
     }
 }
diff --git a/components/script/dom/rtcpeerconnection.rs b/components/script/dom/rtcpeerconnection.rs
index e4948e08d8e..aeed1cb24a5 100644
--- a/components/script/dom/rtcpeerconnection.rs
+++ b/components/script/dom/rtcpeerconnection.rs
@@ -35,7 +35,7 @@ use crate::task_source::networking::NetworkingTaskSource;
 use crate::task_source::TaskSource;
 use dom_struct::dom_struct;
 
-use servo_media::streams::MediaStream as BackendMediaStream;
+use servo_media::streams::registry::MediaStreamId;
 use servo_media::webrtc::{
     BundlePolicy, GatheringState, IceCandidate, IceConnectionState, SdpType, SessionDescription,
     SignalingState, WebRtcController, WebRtcSignaller,
@@ -128,7 +128,7 @@ impl WebRtcSignaller for RTCSignaller {
         );
     }
 
-    fn on_add_stream(&self, _: Box<BackendMediaStream>) {}
+    fn on_add_stream(&self, _: &MediaStreamId) {}
 
     fn close(&self) {
         // do nothing
@@ -567,7 +567,7 @@ impl RTCPeerConnectionMethods for RTCPeerConnection {
     fn AddStream(&self, stream: &MediaStream) {
         let mut tracks = stream.get_tracks();
 
-        for track in tracks.drain(..) {
+        for ref track in tracks.drain(..) {
             self.controller.borrow().as_ref().unwrap().add_stream(track);
         }
     }