Auto merge of #8192 - nxnfufunezn:brotli-decompressor, r=jdm

 Accept Brotli-compressed HTTP responses #8156

r? @jdm

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/servo/8192)
<!-- Reviewable:end -->

11 files changed, 97 insertions, 9 deletions

diff --git a/components/net/Cargo.toml b/components/net/Cargo.toml
index 51573136d4e..620a10ca1b6 100644
--- a/components/net/Cargo.toml
+++ b/components/net/Cargo.toml
@@ -16,6 +16,9 @@ path = "../util"
 [dependencies.devtools_traits]
 path = "../devtools_traits"
 
+[dependencies.brotli]
+git = "https://github.com/ende76/brotli-rs"
+
 [dependencies.plugins]
 path = "../plugins"
 
diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs
index 521af527a37..62e461dbc21 100644
--- a/components/net/http_loader.rs
+++ b/components/net/http_loader.rs
@@ -3,6 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 
+use brotli::Decompressor;
 use cookie;
 use cookie_storage::CookieStorage;
 use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, HttpRequest as DevtoolsHttpRequest};
@@ -154,10 +155,9 @@ pub trait HttpResponse: Read {
                         Some(Encoding::Gzip)
                     } else if encodings.contains(&Encoding::Deflate) {
                         Some(Encoding::Deflate)
-                    } else {
-                        // TODO: Is this the correct behaviour?
-                        None
-                    }
+                    } else if encodings.contains(&Encoding::EncodingExt("br".to_owned())) {
+                        Some(Encoding::EncodingExt("br".to_owned()))
+                    } else { None }
                 }
             }
         })
@@ -302,7 +302,8 @@ fn set_default_accept_encoding(headers: &mut Headers) {
 
     headers.set(AcceptEncoding(vec![
         qitem(Encoding::Gzip),
-        qitem(Encoding::Deflate)
+        qitem(Encoding::Deflate),
+        qitem(Encoding::EncodingExt("br".to_owned()))
     ]));
 }
 
@@ -394,6 +395,7 @@ impl<R: HttpResponse> Read for StreamedResponse<R> {
         match self.decoder {
             Decoder::Gzip(ref mut d) => d.read(buf),
             Decoder::Deflate(ref mut d) => d.read(buf),
+            Decoder::Brotli(ref mut d) => d.read(buf),
             Decoder::Plain(ref mut d) => d.read(buf)
         }
     }
@@ -421,6 +423,10 @@ impl<R: HttpResponse> StreamedResponse<R> {
                 let response_decoding = DeflateDecoder::new(response);
                 Ok(StreamedResponse::new(m, Decoder::Deflate(response_decoding)))
             }
+            Some(Encoding::EncodingExt(ref ext)) if ext == "br" => {
+                let response_decoding = Decompressor::new(response);
+                Ok(StreamedResponse::new(m, Decoder::Brotli(response_decoding)))
+            }
             _ => {
                 Ok(StreamedResponse::new(m, Decoder::Plain(response)))
             }
@@ -431,6 +437,7 @@ impl<R: HttpResponse> StreamedResponse<R> {
 enum Decoder<R: Read> {
     Gzip(GzDecoder<R>),
     Deflate(DeflateDecoder<R>),
+    Brotli(Decompressor<R>),
     Plain(R)
 }
 
diff --git a/components/net/lib.rs b/components/net/lib.rs
index d09d28fa5ba..e0bae6b829d 100644
--- a/components/net/lib.rs
+++ b/components/net/lib.rs
@@ -17,6 +17,7 @@ extern crate cookie as cookie_rs;
 extern crate devtools_traits;
 extern crate euclid;
 extern crate flate2;
+extern crate brotli;
 extern crate hyper;
 extern crate ipc_channel;
 extern crate net_traits;
diff --git a/components/servo/Cargo.lock b/components/servo/Cargo.lock
index d62b48cec42..c4d436e5451 100644
--- a/components/servo/Cargo.lock
+++ b/components/servo/Cargo.lock
@@ -139,6 +139,11 @@ dependencies = [
 ]
 
 [[package]]
+name = "brotli"
+version = "0.3.17"
+source = "git+https://github.com/ende76/brotli-rs#4a8c42cce771ded65fe64d6816f5d7303006b2ea"
+
+[[package]]
 name = "byteorder"
 version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1200,6 +1205,7 @@ dependencies = [
 name = "net"
 version = "0.0.1"
 dependencies = [
+ "brotli 0.3.17 (git+https://github.com/ende76/brotli-rs)",
  "cookie 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "devtools_traits 0.0.1",
  "euclid 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/ports/cef/Cargo.lock b/ports/cef/Cargo.lock
index 1fa03c0f628..e95466b2112 100644
--- a/ports/cef/Cargo.lock
+++ b/ports/cef/Cargo.lock
@@ -129,6 +129,11 @@ dependencies = [
 ]
 
 [[package]]
+name = "brotli"
+version = "0.3.17"
+source = "git+https://github.com/ende76/brotli-rs#4a8c42cce771ded65fe64d6816f5d7303006b2ea"
+
+[[package]]
 name = "byteorder"
 version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1150,6 +1155,7 @@ dependencies = [
 name = "net"
 version = "0.0.1"
 dependencies = [
+ "brotli 0.3.17 (git+https://github.com/ende76/brotli-rs)",
  "cookie 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "devtools_traits 0.0.1",
  "euclid 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/ports/gonk/Cargo.lock b/ports/gonk/Cargo.lock
index 4004400466a..bf4ebfd0ee4 100644
--- a/ports/gonk/Cargo.lock
+++ b/ports/gonk/Cargo.lock
@@ -121,6 +121,11 @@ dependencies = [
 ]
 
 [[package]]
+name = "brotli"
+version = "0.3.17"
+source = "git+https://github.com/ende76/brotli-rs#4a8c42cce771ded65fe64d6816f5d7303006b2ea"
+
+[[package]]
 name = "byteorder"
 version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1099,6 +1104,7 @@ dependencies = [
 name = "net"
 version = "0.0.1"
 dependencies = [
+ "brotli 0.3.17 (git+https://github.com/ende76/brotli-rs)",
  "cookie 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "devtools_traits 0.0.1",
  "euclid 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/tests/unit/net/http_loader.rs b/tests/unit/net/http_loader.rs
index 27bd6341af0..ede8b7d8ed0 100644
--- a/tests/unit/net/http_loader.rs
+++ b/tests/unit/net/http_loader.rs
@@ -307,7 +307,9 @@ fn test_check_default_headers_loaded_in_every_request() {
     load_data.method = Method::Get;
 
     let mut headers = Headers::new();
-    headers.set(AcceptEncoding(vec![qitem(Encoding::Gzip), qitem(Encoding::Deflate)]));
+    headers.set(AcceptEncoding(vec![qitem(Encoding::Gzip),
+                                    qitem(Encoding::Deflate),
+                                    qitem(Encoding::EncodingExt("br".to_owned()))]));
     headers.set(Host { hostname: "mozilla.com".to_owned() , port: None });
     let accept = Accept(vec![
                             qitem(Mime(TopLevel::Text, SubLevel::Html, vec![])),
@@ -776,7 +778,9 @@ fn test_load_uses_explicit_accept_encoding_from_load_data_headers() {
 #[test]
 fn test_load_sets_default_accept_encoding_to_gzip_and_deflate() {
     let mut accept_encoding_headers = Headers::new();
-    accept_encoding_headers.set(AcceptEncoding(vec![qitem(Encoding::Gzip), qitem(Encoding::Deflate)]));
+    accept_encoding_headers.set(AcceptEncoding(vec![qitem(Encoding::Gzip),
+                                                    qitem(Encoding::Deflate),
+                                                    qitem(Encoding::EncodingExt("br".to_owned()))]));
 
     let url = Url::parse("http://mozilla.com").unwrap();
     let mut load_data = LoadData::new(url.clone(), None);
diff --git a/tests/wpt/metadata/MANIFEST.json b/tests/wpt/metadata/MANIFEST.json
index 3222090ad1d..f279b5f318f 100644
--- a/tests/wpt/metadata/MANIFEST.json
+++ b/tests/wpt/metadata/MANIFEST.json
@@ -35349,4 +35349,4 @@
   "rev": "7123012427f92f0dc38a120e6e86a75b6c03aab5",
   "url_base": "/",
   "version": 2
-}
\ No newline at end of file
+}
diff --git a/tests/wpt/mozilla/meta/MANIFEST.json b/tests/wpt/mozilla/meta/MANIFEST.json
index bc04fb431a2..e46ba1e0aa6 100644
--- a/tests/wpt/mozilla/meta/MANIFEST.json
+++ b/tests/wpt/mozilla/meta/MANIFEST.json
@@ -4567,6 +4567,12 @@
             "url": "/_mozilla/mozilla/proxy_setter.html"
           }
         ],
+        "mozilla/response-data-brotli.htm": [
+          {
+            "path": "mozilla/response-data-brotli.htm",
+            "url": "/_mozilla/mozilla/response-data-brotli.htm"
+          }
+        ],
         "mozilla/script_type.html": [
           {
             "path": "mozilla/script_type.html",
@@ -8730,4 +8736,4 @@
   "rev": null,
   "url_base": "/_mozilla/",
   "version": 2
-}
+}
\ No newline at end of file
diff --git a/tests/wpt/mozilla/tests/mozilla/resources/brotli.py b/tests/wpt/mozilla/tests/mozilla/resources/brotli.py
new file mode 100644
index 00000000000..7e27b5c7d17
--- /dev/null
+++ b/tests/wpt/mozilla/tests/mozilla/resources/brotli.py
@@ -0,0 +1,7 @@
+def main(request, response):
+    output = '\x1b\x03)\x00\xa4\xcc\xde\xe2\xb3 vA\x00\x0c'
+    headers = [("Content-type", "text/plain"),
+               ("Content-Encoding", "br"),
+               ("Content-Length", len(output))]
+
+    return headers, output
diff --git a/tests/wpt/mozilla/tests/mozilla/response-data-brotli.htm b/tests/wpt/mozilla/tests/mozilla/response-data-brotli.htm
new file mode 100644
index 00000000000..e46a0e5fb04
--- /dev/null
+++ b/tests/wpt/mozilla/tests/mozilla/response-data-brotli.htm
@@ -0,0 +1,42 @@
+<!doctype html>
+<html>
+  <head>
+    <title>XMLHttpRequest: Brotli response was correctly inflated</title>
+    <script src="/resources/testharness.js"></script>
+    <script src="/resources/testharnessreport.js"></script>
+    <link rel="help" href="https://dvcs.w3.org/hg/xhr/raw-file/tip/Overview.html#the-send()-method" data-tested-assertations="following::p[contains(text(),'content-encodings')]" />
+  </head>
+  <body>
+    <div id="log"></div>
+    <script>
+      function request(input) {
+        var test = async_test();
+        test.step(function() {
+          var client = new XMLHttpRequest()
+
+          client.open("POST", "resources/brotli.py", false);
+
+          client.onreadystatechange = test.step_func(function () {
+              if (client.readyState === 4) {
+                  var len = parseInt(client.getResponseHeader('content-length'), 10);
+
+                  assert_equals(client.getResponseHeader('content-encoding'), 'br');
+                  assert_true(len < input.length);
+                  assert_equals(client.responseText, input);
+                  test.done();
+              }
+          });
+
+          client.send(input);
+        }, document.title);
+      }
+
+      var wellCompressableData = '';
+      for (var i = 0; i < 500; i++) {
+          wellCompressableData += 'foofoofoofoofoofoofoo';
+      }
+
+      request(wellCompressableData);
+    </script>
+  </body>
+</html>