Auto merge of #17061 - servo:utf8, r=nox

EventSource: decode UTF-8 code points across network packets

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/17061)
<!-- Reviewable:end -->

10 files changed, 48 insertions, 27 deletions

diff --git a/Cargo.lock b/Cargo.lock
index efdff886494..77a28b0929d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -632,7 +632,6 @@ name = "devtools"
 version = "0.0.1"
 dependencies = [
  "devtools_traits 0.0.1",
- "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.10.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper_serde 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2374,6 +2373,7 @@ dependencies = [
  "tinyfiledialogs 2.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-segmentation 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utf-8 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "uuid 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "webrender_traits 0.39.0 (git+https://github.com/servo/webrender)",
  "webvr 0.0.1",
diff --git a/components/devtools/Cargo.toml b/components/devtools/Cargo.toml
index 9b5a2164bb8..067d7750470 100644
--- a/components/devtools/Cargo.toml
+++ b/components/devtools/Cargo.toml
@@ -11,7 +11,6 @@ path = "lib.rs"
 
 [dependencies]
 devtools_traits = {path = "../devtools_traits"}
-encoding = "0.2"
 hyper = "0.10"
 hyper_serde = "0.6"
 ipc-channel = "0.7"
diff --git a/components/devtools/actors/network_event.rs b/components/devtools/actors/network_event.rs
index d05d21cc882..0906971821a 100644
--- a/components/devtools/actors/network_event.rs
+++ b/components/devtools/actors/network_event.rs
@@ -9,8 +9,6 @@
 use actor::{Actor, ActorMessageStatus, ActorRegistry};
 use devtools_traits::HttpRequest as DevtoolsHttpRequest;
 use devtools_traits::HttpResponse as DevtoolsHttpResponse;
-use encoding::all::UTF_8;
-use encoding::types::{DecoderTrap, Encoding};
 use hyper::header::{ContentType, Cookie};
 use hyper::header::Headers;
 use hyper::http::RawStatus;
@@ -361,7 +359,7 @@ impl NetworkEventActor {
     pub fn add_response(&mut self, response: DevtoolsHttpResponse) {
         self.response.headers = response.headers.clone();
         self.response.status = response.status.as_ref().map(|&(s, ref st)| {
-            let status_text = UTF_8.decode(st, DecoderTrap::Replace).unwrap();
+            let status_text = String::from_utf8_lossy(st).into_owned();
             RawStatus(s, Cow::from(status_text))
         });
         self.response.body = response.body.clone();
diff --git a/components/devtools/lib.rs b/components/devtools/lib.rs
index 380c5c85236..25697e9410a 100644
--- a/components/devtools/lib.rs
+++ b/components/devtools/lib.rs
@@ -15,7 +15,6 @@
 #![feature(box_syntax)]
 
 extern crate devtools_traits;
-extern crate encoding;
 extern crate hyper;
 extern crate ipc_channel;
 #[macro_use]
diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml
index bc6c17a8871..346ab0ab165 100644
--- a/components/script/Cargo.toml
+++ b/components/script/Cargo.toml
@@ -89,6 +89,7 @@ swapper = "0.0.4"
 time = "0.1.12"
 unicode-segmentation = "1.1.0"
 url = {version = "1.2", features = ["heap_size", "query_encoding"]}
+utf-8 = "0.7"
 uuid = {version = "0.4", features = ["v4"]}
 xml5ever = {version = "0.7", features = ["unstable"]}
 webrender_traits = {git = "https://github.com/servo/webrender", features = ["ipc"]}
diff --git a/components/script/body.rs b/components/script/body.rs
index 1967d69cf50..80c9be5d49f 100644
--- a/components/script/body.rs
+++ b/components/script/body.rs
@@ -11,8 +11,6 @@ use dom::blob::{Blob, BlobImpl};
 use dom::formdata::FormData;
 use dom::globalscope::GlobalScope;
 use dom::promise::Promise;
-use encoding::all::UTF_8;
-use encoding::types::{DecoderTrap, Encoding};
 use js::jsapi::JSContext;
 use js::jsapi::JS_ClearPendingException;
 use js::jsapi::JS_ParseJSON;
@@ -110,14 +108,13 @@ fn run_package_data_algorithm<T: BodyOperations + DomObject>(object: &T,
 }
 
 fn run_text_data_algorithm(bytes: Vec<u8>) -> Fallible<FetchedData> {
-    let text = UTF_8.decode(&bytes, DecoderTrap::Replace).unwrap();
-    Ok(FetchedData::Text(text))
+    Ok(FetchedData::Text(String::from_utf8_lossy(&bytes).into_owned()))
 }
 
 #[allow(unsafe_code)]
 fn run_json_data_algorithm(cx: *mut JSContext,
                            bytes: Vec<u8>) -> Fallible<FetchedData> {
-    let json_text = UTF_8.decode(&bytes, DecoderTrap::Replace).unwrap();
+    let json_text = String::from_utf8_lossy(&bytes);
     let json_text: Vec<u16> = json_text.encode_utf16().collect();
     rooted!(in(cx) let mut rval = UndefinedValue());
     unsafe {
diff --git a/components/script/dom/blob.rs b/components/script/dom/blob.rs
index c076282dcc1..ddf14fc8096 100644
--- a/components/script/dom/blob.rs
+++ b/components/script/dom/blob.rs
@@ -12,8 +12,6 @@ use dom::bindings::reflector::{DomObject, Reflector, reflect_dom_object};
 use dom::bindings::str::DOMString;
 use dom::globalscope::GlobalScope;
 use dom_struct::dom_struct;
-use encoding::all::UTF_8;
-use encoding::types::{EncoderTrap, Encoding};
 use ipc_channel::ipc;
 use net_traits::{CoreResourceMsg, IpcSend};
 use net_traits::blob_url_store::{BlobBuf, get_blob_origin};
@@ -337,12 +335,11 @@ pub fn blob_parts_to_bytes(blobparts: Vec<BlobOrString>) -> Result<Vec<u8>, ()>
     for blobpart in &blobparts {
         match blobpart {
             &BlobOrString::String(ref s) => {
-                let mut bytes = UTF_8.encode(s, EncoderTrap::Replace).map_err(|_|())?;
-                ret.append(&mut bytes);
+                ret.extend(s.as_bytes());
             },
             &BlobOrString::Blob(ref b) => {
-                let mut bytes = b.get_bytes().unwrap_or(vec![]);
-                ret.append(&mut bytes);
+                let bytes = b.get_bytes().unwrap_or(vec![]);
+                ret.extend(bytes);
             },
         }
     }
diff --git a/components/script/dom/eventsource.rs b/components/script/dom/eventsource.rs
index 7945eec0b7d..81a2bc87584 100644
--- a/components/script/dom/eventsource.rs
+++ b/components/script/dom/eventsource.rs
@@ -16,8 +16,6 @@ use dom::eventtarget::EventTarget;
 use dom::globalscope::GlobalScope;
 use dom::messageevent::MessageEvent;
 use dom_struct::dom_struct;
-use encoding::Encoding;
-use encoding::all::UTF_8;
 use euclid::length::Length;
 use hyper::header::{Accept, qitem};
 use ipc_channel::ipc;
@@ -39,6 +37,7 @@ use std::str::{Chars, FromStr};
 use std::sync::{Arc, Mutex};
 use task_source::TaskSource;
 use timers::OneshotTimerCallback;
+use utf8;
 
 header! { (LastEventId, "Last-Event-ID") => [String] }
 
@@ -76,6 +75,8 @@ enum ParserState {
 }
 
 struct EventSourceContext {
+    incomplete_utf8: Option<utf8::Incomplete>,
+
     event_source: Trusted<EventSource>,
     gen_id: GenerationId,
     action_sender: ipc::IpcSender<FetchResponseMsg>,
@@ -293,12 +294,41 @@ impl FetchResponseListener for EventSourceContext {
     }
 
     fn process_response_chunk(&mut self, chunk: Vec<u8>) {
-        let mut stream = String::new();
-        UTF_8.raw_decoder().raw_feed(&chunk, &mut stream);
-        self.parse(stream.chars())
+        let mut input = &*chunk;
+        if let Some(mut incomplete) = self.incomplete_utf8.take() {
+            match incomplete.try_complete(input) {
+                None => return,
+                Some((result, remaining_input)) => {
+                    self.parse(result.unwrap_or("\u{FFFD}").chars());
+                    input = remaining_input;
+                }
+            }
+        }
+
+        while !input.is_empty() {
+            match utf8::decode(&input) {
+                Ok(s) => {
+                    self.parse(s.chars());
+                    return
+                }
+                Err(utf8::DecodeError::Invalid { valid_prefix, remaining_input, .. }) => {
+                    self.parse(valid_prefix.chars());
+                    self.parse("\u{FFFD}".chars());
+                    input = remaining_input;
+                }
+                Err(utf8::DecodeError::Incomplete { valid_prefix, incomplete_suffix }) => {
+                    self.parse(valid_prefix.chars());
+                    self.incomplete_utf8 = Some(incomplete_suffix);
+                    return
+                }
+            }
+        }
     }
 
     fn process_response_eof(&mut self, _response: Result<(), NetworkError>) {
+        if let Some(_) = self.incomplete_utf8.take() {
+            self.parse("\u{FFFD}".chars());
+        }
         self.reestablish_the_connection();
     }
 }
@@ -378,6 +408,8 @@ impl EventSource {
         // Step 14
         let (action_sender, action_receiver) = ipc::channel().unwrap();
         let context = EventSourceContext {
+            incomplete_utf8: None,
+
             event_source: Trusted::new(&ev),
             gen_id: ev.generation_id.get(),
             action_sender: action_sender.clone(),
diff --git a/components/script/dom/textencoder.rs b/components/script/dom/textencoder.rs
index 765c95ade0b..5cdbeca1dd3 100644
--- a/components/script/dom/textencoder.rs
+++ b/components/script/dom/textencoder.rs
@@ -11,9 +11,6 @@ use dom::bindings::reflector::{Reflector, reflect_dom_object};
 use dom::bindings::str::{DOMString, USVString};
 use dom::globalscope::GlobalScope;
 use dom_struct::dom_struct;
-use encoding::EncoderTrap;
-use encoding::Encoding;
-use encoding::all::UTF_8;
 use js::jsapi::{JSContext, JSObject};
 use js::typedarray::{Uint8Array, CreateWith};
 use std::ptr;
@@ -45,13 +42,13 @@ impl TextEncoder {
 impl TextEncoderMethods for TextEncoder {
     // https://encoding.spec.whatwg.org/#dom-textencoder-encoding
     fn Encoding(&self) -> DOMString {
-        DOMString::from(UTF_8.name())
+        DOMString::from("utf-8")
     }
 
     #[allow(unsafe_code)]
     // https://encoding.spec.whatwg.org/#dom-textencoder-encode
     unsafe fn Encode(&self, cx: *mut JSContext, input: USVString) -> NonZero<*mut JSObject> {
-        let encoded = UTF_8.encode(&input.0, EncoderTrap::Strict).unwrap();
+        let encoded = input.0.as_bytes();
 
         rooted!(in(cx) let mut js_object = ptr::null_mut());
         assert!(Uint8Array::create(cx, CreateWith::Slice(&encoded), js_object.handle_mut()).is_ok());
diff --git a/components/script/lib.rs b/components/script/lib.rs
index 6e0b75d7ef9..1dc74905cc9 100644
--- a/components/script/lib.rs
+++ b/components/script/lib.rs
@@ -102,6 +102,7 @@ extern crate time;
 extern crate tinyfiledialogs;
 extern crate unicode_segmentation;
 extern crate url;
+extern crate utf8;
 extern crate uuid;
 extern crate webrender_traits;
 extern crate webvr_traits;