Refactor http_fetch to reflect the new standard

3 files changed, 152 insertions, 83 deletions

diff --git a/components/net/fetch/request.rs b/components/net/fetch/request.rs
index d5b2439553a..861729f5ebf 100644
--- a/components/net/fetch/request.rs
+++ b/components/net/fetch/request.rs
@@ -11,10 +11,13 @@ use hyper::header::{QualityItem, q, qitem};
 use hyper::method::Method;
 use hyper::mime::{Attr, Mime, SubLevel, TopLevel, Value};
 use hyper::status::StatusCode;
-use net_traits::{AsyncFetchListener, Response, ResponseType, Metadata};
+use net_traits::{AsyncFetchListener, Response};
+use net_traits::{ResponseType, Metadata};
 use std::ascii::AsciiExt;
+use std::cell::RefCell;
+use std::rc::Rc;
 use std::str::FromStr;
-use url::Url;
+use url::{Url, UrlParser};
 use util::task::spawn_named;
 
 /// A [request context](https://fetch.spec.whatwg.org/#concept-request-context)
@@ -90,7 +93,8 @@ pub enum ResponseTainting {
 /// A [Request](https://fetch.spec.whatwg.org/#requests) as defined by the Fetch spec
 pub struct Request {
     pub method: Method,
-    pub url: Url,
+    // Use the last method on url_list to act as spec url field
+    pub url_list: Vec<Url>,
     pub headers: Headers,
     pub unsafe_request: bool,
     pub body: Option<Vec<u8>>,
@@ -101,7 +105,7 @@ pub struct Request {
     pub skip_service_worker: bool,
     pub context: Context,
     pub context_frame_type: ContextFrameType,
-    pub origin: Option<Url>,
+    pub origin: Option<Url>, // FIXME: Use Url::Origin
     pub force_origin_header: bool,
     pub same_origin_data: bool,
     pub referer: Referer,
@@ -121,7 +125,7 @@ impl Request {
     pub fn new(url: Url, context: Context, is_service_worker_global_scope: bool) -> Request {
          Request {
             method: Method::Get,
-            url: url,
+            url_list: vec![url],
             headers: Headers::new(),
             unsafe_request: false,
             body: None,
@@ -147,13 +151,17 @@ impl Request {
         }
     }
 
+    fn get_last_url_string(&self) -> String {
+        self.url_list.last().unwrap().serialize()
+    }
+
     pub fn fetch_async(mut self,
                        cors_flag: bool,
                        listener: Box<AsyncFetchListener + Send>) {
-        spawn_named(format!("fetch for {:?}", self.url.serialize()), move || {
+        spawn_named(format!("fetch for {:?}", self.get_last_url_string()), move || {
             let res = self.fetch(cors_flag);
             listener.response_available(res);
-        });
+        })
     }
 
     /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch)
@@ -211,17 +219,21 @@ impl Request {
 
     /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch)
     pub fn basic_fetch(&mut self) -> Response {
-        match &*self.url.scheme {
-            "about" => match self.url.non_relative_scheme_data() {
-                Some(s) if &*s == "blank" => {
-                    let mut response = Response::new();
-                    response.headers.set(ContentType(Mime(
-                        TopLevel::Text, SubLevel::Html,
-                        vec![(Attr::Charset, Value::Utf8)])));
-                    response
-                },
-                _ => Response::network_error()
-            },
+        let scheme = self.url_list.last().unwrap().scheme.clone();
+        match &*scheme {
+            "about" => {
+                let url = self.url_list.last().unwrap();
+                match url.non_relative_scheme_data() {
+                    Some(s) if &*s == "blank" => {
+                        let mut response = Response::new();
+                        response.headers.set(ContentType(Mime(
+                            TopLevel::Text, SubLevel::Html,
+                            vec![(Attr::Charset, Value::Utf8)])));
+                        response
+                    },
+                    _ => Response::network_error()
+                }
+            }
             "http" | "https" => {
                 self.http_fetch(false, false, false)
             },
@@ -234,31 +246,64 @@ impl Request {
         }
     }
 
+    pub fn http_fetch_async(mut self, cors_flag: bool,
+                            cors_preflight_flag: bool,
+                            authentication_fetch_flag: bool,
+                            listener: Box<AsyncFetchListener + Send>) {
+        spawn_named(format!("http_fetch for {:?}", self.get_last_url_string()), move || {
+            let res = self.http_fetch(cors_flag, cors_preflight_flag,
+                                      authentication_fetch_flag);
+            listener.response_available(res);
+        });
+    }
+
     /// [HTTP fetch](https://fetch.spec.whatwg.org#http-fetch)
     pub fn http_fetch(&mut self, cors_flag: bool, cors_preflight_flag: bool,
                       authentication_fetch_flag: bool) -> Response {
         // Step 1
-        let mut response: Option<Response> = None;
+        let mut response: Option<Rc<RefCell<Response>>> = None;
         // Step 2
+        let mut actual_response: Option<Rc<RefCell<Response>>> = None;
+        // Step 3
         if !self.skip_service_worker && !self.is_service_worker_global_scope {
             // TODO: Substep 1 (handle fetch unimplemented)
-            // Substep 2
             if let Some(ref res) = response {
-                if (res.response_type == ResponseType::Opaque && self.mode != RequestMode::NoCORS) ||
-                   res.response_type == ResponseType::Error {
+                let resp = res.borrow();
+                // Substep 2
+                actual_response = match resp.internal_response {
+                    Some(ref internal_res) => Some(internal_res.clone()),
+                    None => Some(res.clone())
+                };
+                // Substep 3
+                if (resp.response_type == ResponseType::Opaque &&
+                    self.mode != RequestMode::NoCORS) ||
+                   (resp.response_type == ResponseType::OpaqueRedirect &&
+                    self.redirect_mode != RedirectMode::Manual) ||
+                   resp.response_type == ResponseType::Error {
                     return Response::network_error();
                 }
             }
+            // Substep 4
+            if let Some(ref res) = actual_response {
+                let mut resp = res.borrow_mut();
+                if resp.url_list.is_empty() {
+                    resp.url_list = self.url_list.clone();
+                }
+            }
+            // Substep 5
+            // TODO: set response's CSP list on actual_response
         }
-        // Step 3
+        // Step 4
         if response.is_none() {
             // Substep 1
             if cors_preflight_flag {
                 let mut method_mismatch = false;
                 let mut header_mismatch = false;
                 if let Some(ref mut cache) = self.cache {
+                    // FIXME: Once Url::Origin is available, rewrite origin to
+                    // take an Origin instead of a Url
                     let origin = self.origin.clone().unwrap_or(Url::parse("").unwrap());
-                    let url = self.url.clone();
+                    let url = self.url_list.last().unwrap().clone();
                     let credentials = self.credentials_mode == CredentialsMode::Include;
                     let method_cache_match = cache.match_method(CacheRequestDetails {
                         origin: origin.clone(),
@@ -280,7 +325,7 @@ impl Request {
                     if preflight_result.response_type == ResponseType::Error {
                         return Response::network_error();
                     }
-                    response = Some(preflight_result);
+                    response = Some(Rc::new(RefCell::new(preflight_result)));
                 }
             }
             // Substep 2
@@ -288,31 +333,24 @@ impl Request {
             // Substep 3
             let credentials = match self.credentials_mode {
                 CredentialsMode::Include => true,
-                CredentialsMode::CredentialsSameOrigin if !cors_flag => true,
+                CredentialsMode::CredentialsSameOrigin if (!cors_flag ||
+                    self.response_tainting == ResponseTainting::Opaque)
+                    => true,
                 _ => false
             };
             // Substep 4
-            if self.cache_mode == CacheMode::Default && is_no_store_cache(&self.headers) {
-                self.cache_mode = CacheMode::NoStore;
-            }
-            // Substep 5
             let fetch_result = self.http_network_or_cache_fetch(credentials, authentication_fetch_flag);
-            // Substep 6
+            // Substep 5
             if cors_flag && self.cors_check(&fetch_result).is_err() {
                 return Response::network_error();
             }
-            response = Some(fetch_result);
+            response = Some(Rc::new(RefCell::new(fetch_result)));
+            actual_response = response.clone();
         }
-        // Step 4
-        let mut response = response.unwrap();
-        match response.status.unwrap() {
-            // Code 304
-            StatusCode::NotModified => match self.cache_mode {
-                CacheMode::Default | CacheMode::NoCache => {
-                    // TODO: Check HTTP cache for request and response entry
-                }
-                _ => { }
-            },
+        // Step 5
+        let mut actual_response = Rc::try_unwrap(actual_response.unwrap()).ok().unwrap().into_inner();
+        let mut response = Rc::try_unwrap(response.unwrap()).ok().unwrap();
+        match actual_response.status.unwrap() {
             // Code 301, 302, 303, 307, 308
             StatusCode::MovedPermanently | StatusCode::Found | StatusCode::SeeOther |
             StatusCode::TemporaryRedirect | StatusCode::PermanentRedirect => {
@@ -321,19 +359,20 @@ impl Request {
                     return Response::network_error();
                 }
                 // Step 2-4
-                if !response.headers.has::<Location>() {
-                    return response;
+                if !actual_response.headers.has::<Location>() {
+                    return actual_response;
                 }
-                let location = match response.headers.get::<Location>() {
-                    None => return Response::network_error(),
-                    Some(location) => location,
+                let location = match actual_response.headers.get::<Location>() {
+                    Some(&Location(ref location)) => location.clone(),
+                    _ => return Response::network_error(),
                 };
                 // Step 5
-                let location_url = Url::parse(location);
+                let location_url = UrlParser::new().base_url(self.url_list.last().unwrap()).parse(&*location);
                 // Step 6
                 let location_url = match location_url {
+                    Ok(ref url) if url.scheme == "data" => { return Response::network_error(); }
                     Ok(url) => url,
-                    Err(_) => return Response::network_error()
+                    _ => { return Response::network_error(); }
                 };
                 // Step 7
                 if self.redirect_count == 20 {
@@ -341,62 +380,79 @@ impl Request {
                 }
                 // Step 8
                 self.redirect_count += 1;
-                // Step 9
-                self.same_origin_data = false;
-                // Step 10
-                if self.redirect_mode == RedirectMode::Follow {
-                    // FIXME: Origin method of the Url crate hasn't been implemented
-                    // https://github.com/servo/rust-url/issues/54
-
-                    // Substep 1
-                    // if cors_flag && location_url.origin() != self.url.origin() { self.origin = None; }
-                    // Substep 2
-                    if cors_flag && (!location_url.username().unwrap_or("").is_empty() ||
-                                      location_url.password().is_some()) {
-                        return Response::network_error();
+                match self.redirect_mode {
+                    // Step 9
+                    RedirectMode::Manual => {
+                        *response.borrow_mut() = actual_response.to_filtered(ResponseType::Opaque);
                     }
-                    // Substep 3
-                    if response.status.unwrap() == StatusCode::MovedPermanently ||
-                       response.status.unwrap() == StatusCode::SeeOther ||
-                       (response.status.unwrap() == StatusCode::Found && self.method == Method::Post) {
-                        self.method = Method::Get;
+                    // Step 10
+                    RedirectMode::Follow => {
+                        // Substep 1
+                        // FIXME: Use Url::origin
+                        // if (self.mode == RequestMode::CORSMode || self.mode == RequestMode::ForcedPreflightMode) &&
+                        //     location_url.origin() != self.url.origin() &&
+                        //     has_credentials(&location_url) {
+                        //     return Response::network_error();
+                        // }
+                        // Substep 2
+                        if cors_flag && has_credentials(&location_url) {
+                            return Response::network_error();
+                        }
+                        // Substep 3
+                        // FIXME: Use Url::origin
+                        // if cors_flag && location_url.origin() != self.url.origin() {
+                        //     self.origin = Origin::UID(OpaqueOrigin);
+                        // }
+                        // Substep 4
+                        if actual_response.status.unwrap() == StatusCode::SeeOther ||
+                           ((actual_response.status.unwrap() == StatusCode::MovedPermanently ||
+                             actual_response.status.unwrap() == StatusCode::Found) &&
+                            self.method == Method::Post) {
+                            self.method = Method::Get;
+                        }
+                        // Substep 5
+                        self.url_list.push(location_url);
+                        // Substep 6
+                        return self.main_fetch(cors_flag);
                     }
-                    // Substep 4
-                    self.url = location_url;
-                    // Substep 5
-                    return self.fetch(cors_flag);
+                    RedirectMode::Error => { panic!("RedirectMode is Error after step 8") }
                 }
             }
             // Code 401
             StatusCode::Unauthorized => {
                 // Step 1
-                if !self.authentication || cors_flag {
-                    return response;
+                // FIXME: Figure out what to do with request window objects
+                if cors_flag {
+                    return response.into_inner();
                 }
                 // Step 2
-                // TODO: Spec says requires testing
+                // TODO: Spec says requires testing on multiple WWW-Authenticate headers
                 // Step 3
                 if !self.use_url_credentials || authentication_fetch_flag {
-                    // TODO: Prompt the user for username and password
+                    // TODO: Prompt the user for username and password from the window
                 }
+                // Step 4
                 return self.http_fetch(cors_flag, cors_preflight_flag, true);
             }
             // Code 407
             StatusCode::ProxyAuthenticationRequired => {
                 // Step 1
-                // TODO: Spec says requires testing
+                // TODO: Figure out what to do with request window objects
                 // Step 2
-                // TODO: Prompt the user for proxy authentication credentials
+                // TODO: Spec says requires testing on Proxy-Authenticate headers
                 // Step 3
+                // TODO: Prompt the user for proxy authentication credentials
+                // Step 4
                 return self.http_fetch(cors_flag, cors_preflight_flag, authentication_fetch_flag);
             }
             _ => { }
         }
-        // Step 5
+        let mut response = response.into_inner();
+        // Step 6
         if authentication_fetch_flag {
             // TODO: Create authentication entry for this request
         }
-        // Step 6
+        // Step 7
         response
     }
 
@@ -421,6 +477,10 @@ impl Request {
     }
 }
 
+fn has_credentials(url: &Url) -> bool {
+    !url.username().unwrap_or("").is_empty() || url.password().is_some()
+}
+
 fn is_no_store_cache(headers: &Headers) -> bool {
     headers.has::<IfModifiedSince>() | headers.has::<IfNoneMatch>() |
     headers.has::<IfUnmodifiedSince>() | headers.has::<IfMatch>() |
diff --git a/components/net/fetch/response.rs b/components/net/fetch/response.rs
index fe39e1495c0..584e6e7fe40 100644
--- a/components/net/fetch/response.rs
+++ b/components/net/fetch/response.rs
@@ -6,6 +6,8 @@ use hyper::header::Headers;
 use hyper::status::StatusCode;
 use net_traits::{Response, ResponseBody, ResponseType};
 use std::ascii::AsciiExt;
+use std::cell::RefCell;
+use std::rc::Rc;
 use std::sync::mpsc::Receiver;
 use url::Url;
 
@@ -20,6 +22,7 @@ impl ResponseMethods for Response {
             response_type: ResponseType::Default,
             termination_reason: None,
             url: None,
+            url_list: Vec::new(),
             status: Some(StatusCode::Ok),
             headers: Headers::new(),
             body: ResponseBody::Empty,
@@ -37,7 +40,7 @@ impl ResponseMethods for Response {
         }
         let old_headers = self.headers.clone();
         let mut response = self.clone();
-        response.internal_response = Some(box self);
+        response.internal_response = Some(Rc::new(RefCell::new(self)));
         match filter_type {
             ResponseType::Default | ResponseType::Error => unreachable!(),
             ResponseType::Basic => {
@@ -62,7 +65,8 @@ impl ResponseMethods for Response {
                 response.headers = headers;
                 response.response_type = filter_type;
             },
-            ResponseType::Opaque => {
+            ResponseType::Opaque |
+            ResponseType::OpaqueRedirect => {
                 response.headers = Headers::new();
                 response.status = None;
                 response.body = ResponseBody::Empty;
diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs
index de59bf654d7..18078310d56 100644
--- a/components/net_traits/lib.rs
+++ b/components/net_traits/lib.rs
@@ -35,6 +35,8 @@ use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use msg::constellation_msg::{PipelineId};
 use regex::Regex;
 use serde::{Deserializer, Serializer};
+use std::cell::RefCell;
+use std::rc::Rc;
 use std::thread;
 use url::Url;
 use util::mem::HeapSizeOf;
@@ -55,7 +57,8 @@ pub enum ResponseType {
     CORS,
     Default,
     Error,
-    Opaque
+    Opaque,
+    OpaqueRedirect
 }
 
 /// [Response termination reason](https://fetch.spec.whatwg.org/#concept-response-termination-reason)
@@ -87,13 +90,14 @@ pub struct Response {
     pub response_type: ResponseType,
     pub termination_reason: Option<TerminationReason>,
     pub url: Option<Url>,
+    pub url_list: Vec<Url>,
     /// `None` can be considered a StatusCode of `0`.
     pub status: Option<StatusCode>,
     pub headers: Headers,
     pub body: ResponseBody,
     /// [Internal response](https://fetch.spec.whatwg.org/#concept-internal-response), only used if the Response
     /// is a filtered response
-    pub internal_response: Option<Box<Response>>,
+    pub internal_response: Option<Rc<RefCell<Response>>>,
 }
 
 impl Response {
@@ -102,6 +106,7 @@ impl Response {
             response_type: ResponseType::Error,
             termination_reason: None,
             url: None,
+            url_list: vec![],
             status: None,
             headers: Headers::new(),
             body: ResponseBody::Empty,