Test fixes; update for changes in spec

3 files changed, 151 insertions, 131 deletions

diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs
index 90d2d24cfe1..305eb1bdf83 100644
--- a/components/net/fetch/methods.rs
+++ b/components/net/fetch/methods.rs
@@ -7,6 +7,7 @@ use data_loader::decode;
 use fetch::cors_cache::CORSCache;
 use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies};
 use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block};
+use http_loader::{auth_from_cache, determine_request_referrer};
 use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials};
 use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods};
 use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic};
@@ -17,6 +18,7 @@ use hyper::method::Method;
 use hyper::mime::{Mime, SubLevel, TopLevel};
 use hyper::status::StatusCode;
 use mime_guess::guess_mime_type;
+use msg::constellation_msg::ReferrerPolicy;
 use net_traits::FetchTaskTarget;
 use net_traits::request::{CacheMode, CredentialsMode};
 use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting};
@@ -42,17 +44,22 @@ enum Data {
     Done,
 }
 
+pub struct FetchContext {
+    pub state: HttpState,
+    pub user_agent: String,
+}
+
 type DoneChannel = Option<(Sender<Data>, Receiver<Data>)>;
 
 /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch)
-pub fn fetch(request: Rc<Request>, target: &mut Target, state: HttpState) -> Response {
-    fetch_with_cors_cache(request, &mut CORSCache::new(), target, state)
+pub fn fetch(request: Rc<Request>, target: &mut Target, context: FetchContext) -> Response {
+    fetch_with_cors_cache(request, &mut CORSCache::new(), target, context)
 }
 
 pub fn fetch_with_cors_cache(request: Rc<Request>,
                              cache: &mut CORSCache,
                              target: &mut Target,
-                             state: HttpState) -> Response {
+                             context: FetchContext) -> Response {
     // Step 1
     if request.window.get() == Window::Client {
         // TODO: Set window to request's client object if client is a Window object
@@ -113,13 +120,13 @@ pub fn fetch_with_cors_cache(request: Rc<Request>,
     }
 
     // Step 7
-    main_fetch(request, cache, false, false, target, &mut None, &state)
+    main_fetch(request, cache, false, false, target, &mut None, &context)
 }
 
 /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch)
 fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
               recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel,
-              state: &HttpState) -> Response {
+              context: &FetchContext) -> Response {
     // TODO: Implement main fetch spec
 
     // Step 1
@@ -142,14 +149,25 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
     // Step 5
     // TODO this step (CSP port/content blocking)
 
-    // Step 6-7
+    // Step 6
     // TODO this step (referer policy)
+    // currently the clients themselves set referer policy in RequestInit
+
+    // Step 7
+    if request.referrer_policy.get().is_none() {
+        request.referrer_policy.set(Some(ReferrerPolicy::NoRefWhenDowngrade));
+    }
 
     // Step 8
     if *request.referer.borrow() != Referer::NoReferer {
-        // TODO be able to invoke "determine request's referer"
-        // once this is filled in be sure to update the match
-        // referer below to have an unreachable branch for client
+        // remove Referer headers set in past redirects/preflights
+        // this stops the assertion in determine_request_referrer from failing
+        request.headers.borrow_mut().remove::<RefererHeader>();
+        let referrer_url = determine_request_referrer(&mut *request.headers.borrow_mut(),
+                                                      request.referrer_policy.get(),
+                                                      request.referer.borrow_mut().take(),
+                                                      request.current_url().clone());
+        *request.referer.borrow_mut() = Referer::from_url(referrer_url);
     }
 
     // Step 9
@@ -174,14 +192,14 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
                 (current_url.scheme() == "file" && request.same_origin_data.get()) ||
                 current_url.scheme() == "about" ||
                 request.mode == RequestMode::Navigate {
-                basic_fetch(request.clone(), cache, target, done_chan, state)
+                basic_fetch(request.clone(), cache, target, done_chan, context)
 
             } else if request.mode == RequestMode::SameOrigin {
                 Response::network_error()
 
             } else if request.mode == RequestMode::NoCORS {
                 request.response_tainting.set(ResponseTainting::Opaque);
-                basic_fetch(request.clone(), cache, target, done_chan, state)
+                basic_fetch(request.clone(), cache, target, done_chan, context)
 
             } else if !matches!(current_url.scheme(), "http" | "https") {
                 Response::network_error()
@@ -192,7 +210,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
                   request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) {
                 request.response_tainting.set(ResponseTainting::CORSTainting);
                 request.redirect_mode.set(RedirectMode::Error);
-                let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, state);
+                let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, context);
                 if response.is_network_error() {
                     // TODO clear cache entries using request
                 }
@@ -200,7 +218,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
 
             } else {
                 request.response_tainting.set(ResponseTainting::CORSTainting);
-                http_fetch(request.clone(), cache, true, false, false, target, done_chan, state)
+                http_fetch(request.clone(), cache, true, false, false, target, done_chan, context)
             }
         }
     };
@@ -233,9 +251,14 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
         };
 
         // Step 15
-        // TODO this step (CSP/blocking)
+        if internal_response.url_list.borrow().is_empty() {
+            *internal_response.url_list.borrow_mut() = request.url_list.borrow().clone();
+        }
 
         // Step 16
+        // TODO this step (CSP/blocking)
+
+        // Step 17
         if !response.is_network_error() && (is_null_body_status(&internal_response.status) ||
             match *request.method.borrow() {
                 Method::Head | Method::Connect => true,
@@ -247,7 +270,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
             *body = ResponseBody::Empty;
         }
 
-        // Step 17
+        // Step 18
         // TODO be able to compare response integrity against request integrity metadata
         // if !response.is_network_error() {
 
@@ -262,7 +285,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
         // }
     }
 
-    // Step 18
+    // Step 19
     if request.synchronous {
         if let Some(ref mut target) = *target {
             // process_response is not supposed to be used
@@ -282,14 +305,12 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
                     Data::Done => break,
                 }
             }
-        } else {
-            if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
-                // in case there was no channel to wait for, the body was
-                // obtained synchronously via basic_fetch for data/file/about/etc
-                // We should still send the body across as a chunk
-                if let Some(ref mut target) = *target {
-                    target.process_response_chunk(vec.clone());
-                }
+        } else if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
+            // in case there was no channel to wait for, the body was
+            // obtained synchronously via basic_fetch for data/file/about/etc
+            // We should still send the body across as a chunk
+            if let Some(ref mut target) = *target {
+                target.process_response_chunk(vec.clone());
             }
         }
 
@@ -300,7 +321,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
         return response;
     }
 
-    // Step 19
+    // Step 20
     if request.body.borrow().is_some() && matches!(request.current_url().scheme(), "http" | "https") {
         if let Some(ref mut target) = *target {
             // XXXManishearth: We actually should be calling process_request
@@ -312,12 +333,12 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
         }
     }
 
-    // Step 20
+    // Step 21
     if let Some(ref mut target) = *target {
         target.process_response(&response);
     }
 
-    // Step 21
+    // Step 22
     if let Some(ref ch) = *done_chan {
         loop {
             match ch.1.recv()
@@ -330,21 +351,19 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
                 Data::Done => break,
             }
         }
-    } else {
-        if let Some(ref mut target) = *target {
-            if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
-                // in case there was no channel to wait for, the body was
-                // obtained synchronously via basic_fetch for data/file/about/etc
-                // We should still send the body across as a chunk
-                target.process_response_chunk(vec.clone());
-            }
+    } else if let Some(ref mut target) = *target {
+        if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
+            // in case there was no channel to wait for, the body was
+            // obtained synchronously via basic_fetch for data/file/about/etc
+            // We should still send the body across as a chunk
+            target.process_response_chunk(vec.clone());
         }
     }
 
-    // Step 22
+    // Step 23
     request.done.set(true);
 
-    // Step 23
+    // Step 24
     if let Some(ref mut target) = *target {
         target.process_response_eof(&response);
     }
@@ -356,7 +375,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
 /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch)
 fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache,
                target: &mut Target, done_chan: &mut DoneChannel,
-               state: &HttpState) -> Response {
+               context: &FetchContext) -> Response {
     let url = request.current_url();
 
     match url.scheme() {
@@ -370,7 +389,7 @@ fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache,
         },
 
         "http" | "https" => {
-            http_fetch(request.clone(), cache, false, false, false, target, done_chan, state)
+            http_fetch(request.clone(), cache, false, false, false, target, done_chan, context)
         },
 
         "data" => {
@@ -432,8 +451,7 @@ fn http_fetch(request: Rc<Request>,
               authentication_fetch_flag: bool,
               target: &mut Target,
               done_chan: &mut DoneChannel,
-              state: &HttpState) -> Response {
-
+              context: &FetchContext) -> Response {
     // This is a new async fetch, reset the channel we are waiting on
     *done_chan = None;
     // Step 1
@@ -463,17 +481,18 @@ fn http_fetch(request: Rc<Request>,
             }
 
             // Substep 4
-            let actual_response = res.actual_response();
-            if actual_response.url_list.borrow().is_empty() {
-                *actual_response.url_list.borrow_mut() = request.url_list.borrow().clone();
-            }
-
-            // Substep 5
             // TODO: set response's CSP list on actual_response
         }
     }
 
     // Step 4
+    let credentials = match request.credentials_mode {
+        CredentialsMode::Include => true,
+        CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic
+            => true,
+        _ => false
+    };
+    // Step 5
     if response.is_none() {
         // Substep 1
         if cors_preflight_flag {
@@ -488,7 +507,7 @@ fn http_fetch(request: Rc<Request>,
 
             // Sub-substep 1
             if method_mismatch || header_mismatch {
-                let preflight_result = cors_preflight_fetch(request.clone(), cache, state);
+                let preflight_result = cors_preflight_fetch(request.clone(), cache, context);
                 // Sub-substep 2
                 if preflight_result.response_type == ResponseType::Error {
                     return Response::network_error();
@@ -500,18 +519,10 @@ fn http_fetch(request: Rc<Request>,
         request.skip_service_worker.set(true);
 
         // Substep 3
-        let credentials = match request.credentials_mode {
-            CredentialsMode::Include => true,
-            CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic
-                => true,
-            _ => false
-        };
-
-        // Substep 4
         let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag,
-                                                       done_chan, state);
+                                                       done_chan, context);
 
-        // Substep 5
+        // Substep 4
         if cors_flag && cors_check(request.clone(), &fetch_result).is_err() {
             return Response::network_error();
         }
@@ -537,7 +548,7 @@ fn http_fetch(request: Rc<Request>,
                     // set back to default
                     response.return_internal.set(true);
                     http_redirect_fetch(request, cache, Rc::new(response),
-                                        cors_flag, target, done_chan, state)
+                                        cors_flag, target, done_chan, context)
                 }
             }
         },
@@ -546,7 +557,7 @@ fn http_fetch(request: Rc<Request>,
         StatusCode::Unauthorized => {
             // Step 1
             // FIXME: Figure out what to do with request window objects
-            if cors_flag || request.credentials_mode != CredentialsMode::Include {
+            if cors_flag || !credentials {
                 return response;
             }
 
@@ -556,11 +567,15 @@ fn http_fetch(request: Rc<Request>,
             // Step 3
             if !request.use_url_credentials || authentication_fetch_flag {
                 // TODO: Prompt the user for username and password from the window
+                // Wrong, but will have to do until we are able to prompt the user
+                // otherwise this creates an infinite loop
+                // We basically pretend that the user declined to enter credentials
+                return response;
             }
 
             // Step 4
             return http_fetch(request, cache, cors_flag, cors_preflight_flag,
-                              true, target, done_chan, state);
+                              true, target, done_chan, context);
         }
 
         // Code 407
@@ -573,12 +588,16 @@ fn http_fetch(request: Rc<Request>,
 
             // Step 3
             // TODO: Prompt the user for proxy authentication credentials
+            // Wrong, but will have to do until we are able to prompt the user
+            // otherwise this creates an infinite loop
+            // We basically pretend that the user declined to enter credentials
+            return response;
 
             // Step 4
-            return http_fetch(request, cache,
-                              cors_flag, cors_preflight_flag,
-                              authentication_fetch_flag, target,
-                              done_chan, state);
+            // return http_fetch(request, cache,
+            //                   cors_flag, cors_preflight_flag,
+            //                   authentication_fetch_flag, target,
+            //                   done_chan, context);
         }
 
         _ => { }
@@ -602,7 +621,7 @@ fn http_redirect_fetch(request: Rc<Request>,
                        cors_flag: bool,
                        target: &mut Target,
                        done_chan: &mut DoneChannel,
-                       state: &HttpState) -> Response {
+                       context: &FetchContext) -> Response {
     // Step 1
     assert_eq!(response.return_internal.get(), true);
 
@@ -676,7 +695,7 @@ fn http_redirect_fetch(request: Rc<Request>,
     request.url_list.borrow_mut().push(location_url);
 
     // Step 15
-    main_fetch(request, cache, cors_flag, true, target, done_chan, state)
+    main_fetch(request, cache, cors_flag, true, target, done_chan, context)
 }
 
 /// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch)
@@ -684,14 +703,14 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
                                credentials_flag: bool,
                                authentication_fetch_flag: bool,
                                done_chan: &mut DoneChannel,
-                               state: &HttpState) -> Response {
+                               context: &FetchContext) -> Response {
     // TODO: Implement Window enum for Request
     let request_has_no_window = true;
 
     // Step 1
     let http_request = if request_has_no_window &&
-        request.redirect_mode.get() != RedirectMode::Follow {
-        request.clone()
+        request.redirect_mode.get() == RedirectMode::Error {
+        request
     } else {
         Rc::new((*request).clone())
     };
@@ -716,16 +735,13 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 6
     match *http_request.referer.borrow() {
-        // Referer::Client should not be here, but we don't set
-        // the referer yet, so we club it with NoReferer
-        Referer::NoReferer | Referer::Client =>
-            http_request.headers.borrow_mut().set(RefererHeader("".to_owned())),
+        Referer::NoReferer => (),
         Referer::RefererUrl(ref http_request_referer) =>
             http_request.headers.borrow_mut().set(RefererHeader(http_request_referer.to_string())),
-        // Referer::Client =>
-        //     // it should be impossible for referer to be anything else during fetching
-        //     // https://fetch.spec.whatwg.org/#concept-request-referrer
-        //     unreachable!()
+        Referer::Client =>
+            // it should be impossible for referer to be anything else during fetching
+            // https://fetch.spec.whatwg.org/#concept-request-referrer
+            unreachable!()
     };
 
     // Step 7
@@ -736,7 +752,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 8
     if !http_request.headers.borrow().has::<UserAgent>() {
-        http_request.headers.borrow_mut().set(UserAgent(global_user_agent().to_owned()));
+        http_request.headers.borrow_mut().set(UserAgent(context.user_agent.clone()));
     }
 
     match http_request.cache_mode.get() {
@@ -780,7 +796,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
             port: current_url.port_or_known_default()
         };
         headers.set(host);
-        // accept header should not be set here, unlike http
+        // unlike http_loader, we should not set the accept header
+        // here, according to the fetch spec
         set_default_accept_encoding(headers);
     }
 
@@ -789,28 +806,27 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
     if credentials_flag {
         // Substep 1
         // TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504
-        // XXXManishearth http_loader has block_cookies, should we do this too?
+        // XXXManishearth http_loader has block_cookies: support content blocking here too
         set_request_cookies(&current_url,
                             &mut *http_request.headers.borrow_mut(),
-                            &state.cookie_jar);
+                            &context.state.cookie_jar);
         // Substep 2
         if !http_request.headers.borrow().has::<Authorization<String>>() {
             // Substep 3
             let mut authorization_value = None;
 
             // Substep 4
-            // TODO be able to retrieve https://fetch.spec.whatwg.org/#authentication-entry
-
-            // Substep 5
-            if authentication_fetch_flag {
-
-                authorization_value = if has_credentials(&current_url) {
-                    Some(Basic {
+            if let Some(basic) = auth_from_cache(&context.state.auth_cache, &current_url) {
+                if !http_request.use_url_credentials || !has_credentials(&current_url) {
+                    authorization_value = Some(basic);
+                }
+            } else if authentication_fetch_flag {
+                // Substep 5
+                if has_credentials(&current_url) {
+                    authorization_value = Some(Basic {
                         username: current_url.username().to_owned(),
                         password: current_url.password().map(str::to_owned)
                     })
-                } else {
-                    None
                 }
             }
 
@@ -866,7 +882,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 18
     if response.is_none() {
-        response = Some(http_network_fetch(http_request.clone(), http_request.clone(), credentials_flag, done_chan));
+        response = Some(http_network_fetch(http_request.clone(), credentials_flag, done_chan));
     }
     let response = response.unwrap();
 
@@ -901,7 +917,6 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
 /// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch)
 fn http_network_fetch(request: Rc<Request>,
-                      _http_request: Rc<Request>,
                       _credentials_flag: bool,
                       done_chan: &mut DoneChannel) -> Response {
     // TODO: Implement HTTP network fetch spec
@@ -926,12 +941,12 @@ fn http_network_fetch(request: Rc<Request>,
     let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(),
                                            &request.headers.borrow(),
                                            &cancellation_listener, &request.body.borrow(), &request.method.borrow(),
-                                           &None, request.redirect_count.get()+1, &None, "");
+                                           &None, request.redirect_count.get() + 1, &None, "");
 
     let mut response = Response::new();
     match wrapped_response {
-        Ok((mut res, _)) => {
-            response.url = Some(res.response.url.clone());
+        Ok((res, _)) => {
+            response.url = Some(url.clone());
             response.status = Some(res.response.status);
             response.raw_status = Some(res.response.status_raw().clone());
             response.headers = res.response.headers.clone();
@@ -940,17 +955,16 @@ fn http_network_fetch(request: Rc<Request>,
 
             // We're about to spawn a thread to be waited on here
             *done_chan = Some(channel());
-            let meta = response.metadata().expect("Response metadata should exist at this stage"); 
+            let meta = response.metadata().expect("Response metadata should exist at this stage");
             let done_sender = done_chan.as_ref().map(|ch| ch.0.clone());
             spawn_named(format!("fetch worker thread"), move || {
                 match StreamedResponse::from_http_response(box res, meta) {
                     Ok(mut res) => {
-                    *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]);
+                        *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]);
                         loop {
                             match read_block(&mut res) {
                                 Ok(ReadResult::Payload(chunk)) => {
                                     if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() {
-
                                         body.extend_from_slice(&chunk);
                                         if let Some(ref sender) = done_sender {
                                             let _ = sender.send(Data::Payload(chunk));
@@ -1003,7 +1017,10 @@ fn http_network_fetch(request: Rc<Request>,
 
     // TODO Read request
 
-    // Step 5
+    // Step 5-9
+    // (needs stream bodies)
+
+    // Step 10
     // TODO when https://bugzilla.mozilla.org/show_bug.cgi?id=1030660
     // is resolved, this step will become uneccesary
     // TODO this step
@@ -1015,22 +1032,19 @@ fn http_network_fetch(request: Rc<Request>,
         }
     };
 
-    // Step 6
-    *response.url_list.borrow_mut() = request.url_list.borrow().clone();
-
-    // Step 7
-    // TODO this step isn't possible yet
+    // Step 11
+    // TODO this step isn't possible yet (CSP)
 
-    // Step 8
+    // Step 12
     if response.is_network_error() && request.cache_mode.get() == CacheMode::NoStore {
         // TODO update response in the HTTP cache for request
     }
 
     // TODO this step isn't possible yet
-    // Step 9
+    // Step 13
 
     // TODO these steps
-    // Step 10
+    // Step 14
         // Substep 1
         // Substep 2
             // Sub-substep 1
@@ -1039,12 +1053,12 @@ fn http_network_fetch(request: Rc<Request>,
             // Sub-substep 4
         // Substep 3
 
-    // Step 11
+    // Step 15
     response
 }
 
 /// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch)
-fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, state: &HttpState) -> Response {
+fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, context: &FetchContext) -> Response {
     // Step 1
     let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()), false);
     *preflight.method.borrow_mut() = Method::Options;
@@ -1052,6 +1066,7 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, state: &Htt
     preflight.type_ = request.type_.clone();
     preflight.destination = request.destination.clone();
     *preflight.referer.borrow_mut() = request.referer.borrow().clone();
+    preflight.referrer_policy.set(preflight.referrer_policy.get());
 
     // Step 2
     preflight.headers.borrow_mut().set::<AccessControlRequestMethod>(
@@ -1072,7 +1087,7 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, state: &Htt
 
     // Step 6
     let preflight = Rc::new(preflight);
-    let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, state);
+    let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, context);
 
     // Step 7
     if cors_check(request.clone(), &response).is_ok() &&
@@ -1105,12 +1120,16 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, state: &Htt
         }
 
         // Substep 5
+        debug!("CORS check: Allowed methods: {:?}, current method: {:?}",
+                methods, request.method.borrow());
         if methods.iter().all(|method| *method != *request.method.borrow()) &&
             !is_simple_method(&*request.method.borrow()) {
             return Response::network_error();
         }
 
         // Substep 6
+        debug!("CORS check: Allowed headers: {:?}, current headers: {:?}",
+                header_names, request.headers.borrow());
         let set: HashSet<&UniCase<String>> = HashSet::from_iter(header_names.iter());
         if request.headers.borrow().iter().any(|ref hv| !set.contains(&UniCase(hv.name().to_owned())) &&
                                                         !is_simple_header(hv)) {
@@ -1183,12 +1202,6 @@ fn cors_check(request: Rc<Request>, response: &Response) -> Result<(), ()> {
     Err(())
 }
 
-fn global_user_agent() -> String {
-    // TODO have a better useragent string
-    const USER_AGENT_STRING: &'static str = "Servo";
-    USER_AGENT_STRING.to_owned()
-}
-
 fn has_credentials(url: &Url) -> bool {
     !url.username().is_empty() || url.password().is_some()
 }
@@ -1199,6 +1212,7 @@ fn is_no_store_cache(headers: &Headers) -> bool {
     headers.has::<IfRange>()
 }
 
+/// https://fetch.spec.whatwg.org/#cors-safelisted-request-header
 fn is_simple_header(h: &HeaderView) -> bool {
     if h.is::<ContentType>() {
         match h.value() {
diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs
index d647aa16717..1123af628ab 100644
--- a/components/net/http_loader.rs
+++ b/components/net/http_loader.rs
@@ -434,7 +434,7 @@ fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option<Url> {
 }
 
 /// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
-fn determine_request_referrer(headers: &mut Headers,
+pub fn determine_request_referrer(headers: &mut Headers,
                               referrer_policy: Option<ReferrerPolicy>,
                               referrer_url: Option<Url>,
                               url: Url) -> Option<Url> {
@@ -667,18 +667,21 @@ fn set_auth_header(headers: &mut Headers,
         if let Some(auth) = auth_from_url(url) {
             headers.set(auth);
         } else {
-            if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) {
-                auth_from_entry(&auth_entry, headers);
+            if let Some(basic) = auth_from_cache(auth_cache, url) {
+                headers.set(Authorization(basic));
             }
         }
     }
 }
 
-fn auth_from_entry(auth_entry: &AuthCacheEntry, headers: &mut Headers) {
-    let user_name = auth_entry.user_name.clone();
-    let password  = Some(auth_entry.password.clone());
-
-    headers.set(Authorization(Basic { username: user_name, password: password }));
+pub fn auth_from_cache(auth_cache: &Arc<RwLock<AuthCache>>, url: &Url) -> Option<Basic> {
+    if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) {
+        let user_name = auth_entry.user_name.clone();
+        let password  = Some(auth_entry.password.clone());
+        Some(Basic { username: user_name, password: password })
+    } else {
+        None
+    }
 }
 
 fn auth_from_url(doc_url: &Url) -> Option<Authorization<Basic>> {
@@ -955,6 +958,7 @@ pub fn load<A, B>(load_data: &LoadData,
             // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12
             set_auth_header(&mut request_headers, &doc_url, &http_state.auth_cache);
         }
+
         //if there is a new auth header then set the request headers with it
         if let Some(ref auth_header) = new_auth_header {
             request_headers.set(auth_header.clone());
diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs
index c7c6000844a..6e94fd1cf03 100644
--- a/components/net/resource_thread.rs
+++ b/components/net/resource_thread.rs
@@ -11,7 +11,7 @@ use cookie;
 use cookie_storage::CookieStorage;
 use data_loader;
 use devtools_traits::DevtoolsControlMsg;
-use fetch::methods::fetch;
+use fetch::methods::{fetch, FetchContext};
 use file_loader;
 use filemanager_thread::FileManagerThreadFactory;
 use hsts::HstsList;
@@ -23,11 +23,11 @@ use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag};
 use net_traits::LoadContext;
 use net_traits::ProgressMsg::Done;
+use net_traits::request::{Request, RequestInit};
 use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, CoreResourceThread};
 use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, FetchTaskTarget, LoadConsumer};
 use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId};
 use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads};
-use net_traits::request::{Request, RequestInit};
 use profile_traits::time::ProfilerChan;
 use rustc_serialize::json;
 use rustc_serialize::{Decodable, Encodable};
@@ -197,7 +197,7 @@ impl ResourceChannelManager {
             match self.from_client.recv().unwrap() {
                 CoreResourceMsg::Load(load_data, consumer, id_sender) =>
                     self.resource_manager.load(load_data, consumer, id_sender, control_sender.clone()),
-                CoreResourceMsg::Fetch(init, sender) => 
+                CoreResourceMsg::Fetch(init, sender) =>
                     self.resource_manager.fetch(init, sender),
                 CoreResourceMsg::WebsocketConnect(connect, connect_data) =>
                     self.resource_manager.websocket_connect(connect, connect_data),
@@ -486,21 +486,23 @@ impl CoreResourceManager {
                          cancel_listener));
     }
 
-    fn fetch(&self, init: RequestInit, sender: IpcSender<FetchResponseMsg>) {        
+    fn fetch(&self, init: RequestInit, sender: IpcSender<FetchResponseMsg>) {
         let http_state = HttpState {
             hsts_list: self.hsts_list.clone(),
             cookie_jar: self.cookie_jar.clone(),
             auth_cache: self.auth_cache.clone(),
             blocked_content: BLOCKED_CONTENT_RULES.clone(),
         };
+        let ua = self.user_agent.clone();
         spawn_named(format!("fetch thread for {}", init.url), move || {
             let request = Request::from_init(init);
-            // XXXManishearth: Check origin against pipeline id
+            // XXXManishearth: Check origin against pipeline id (also ensure that the mode is allowed)
             // todo load context / mimesniff in fetch
             // todo referrer policy?
             // todo service worker stuff
             let mut target = Some(Box::new(sender) as Box<FetchTaskTarget + Send + 'static>);
-            fetch(Rc::new(request), &mut target, http_state);
+            let context = FetchContext { state: http_state, user_agent: ua };
+            fetch(Rc::new(request), &mut target, context);
         })
     }