diff options
author | bors-servo <lbergstrom+bors@mozilla.com> | 2016-06-11 19:30:28 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-06-11 19:30:28 -0500 |
commit | 0c11e8340b26aa86faf9ea40aae253392b338ba3 (patch) | |
tree | c64418a8dc4cc53dcec1766076fa5cf31782fcb3 | |
parent | 3fa0dca3a32552a83ae020c57535346d9cdf3bdd (diff) | |
parent | 9deecd793c6859e52c2ce9df7bba28eb3159cb97 (diff) | |
download | servo-0c11e8340b26aa86faf9ea40aae253392b338ba3.tar.gz servo-0c11e8340b26aa86faf9ea40aae253392b338ba3.zip |
Auto merge of #11556 - Manishearth:make-fetch-happen, r=jdm
Make fetch happen
<!-- Please describe your changes on the following line: -->
Moves XHR over to the fetch backend.
Previous PR: https://github.com/servo/servo/pull/114
---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [ ] `./mach test-tidy` does not report any errors (Will fix later)
<!-- Either: -->
- [x] There are tests for these changes OR
- [ ] These changes do not require tests because _____
<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->
<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/11556)
<!-- Reviewable:end -->
33 files changed, 859 insertions, 1083 deletions
diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 16d64a86a7c..b357edae05e 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -5,20 +5,24 @@ use connector::create_http_connector; use data_loader::decode; use fetch::cors_cache::CORSCache; -use http_loader::{NetworkHttpRequestFactory, ReadResult, obtain_response, read_block}; +use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies}; +use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block}; +use http_loader::{auth_from_cache, determine_request_referrer}; use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials}; use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods}; use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic}; use hyper::header::{CacheControl, CacheDirective, ContentEncoding, ContentLength, ContentLanguage, ContentType}; -use hyper::header::{Encoding, HeaderView, Headers, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince}; +use hyper::header::{Encoding, HeaderView, Headers, Host, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince}; use hyper::header::{IfNoneMatch, Pragma, Location, QualityItem, Referer as RefererHeader, UserAgent, q, qitem}; use hyper::method::Method; use hyper::mime::{Mime, SubLevel, TopLevel}; use hyper::status::StatusCode; use mime_guess::guess_mime_type; -use net_traits::AsyncFetchListener; -use net_traits::request::{CacheMode, CredentialsMode, Type, Origin, Window}; +use msg::constellation_msg::ReferrerPolicy; +use net_traits::FetchTaskTarget; +use net_traits::request::{CacheMode, CredentialsMode}; use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting}; +use net_traits::request::{Type, Origin, Window}; use net_traits::response::{HttpsState, TerminationReason}; use net_traits::response::{Response, ResponseBody, ResponseType}; use resource_thread::CancellationListener; @@ -26,27 +30,36 @@ use std::collections::HashSet; use std::fs::File; use std::io::Read; use std::iter::FromIterator; +use std::mem::swap; use std::rc::Rc; -use std::thread; +use std::sync::mpsc::{channel, Sender, Receiver}; use unicase::UniCase; use url::{Origin as UrlOrigin, Url}; use util::thread::spawn_named; -pub fn fetch_async(request: Request, listener: Box<AsyncFetchListener + Send>) { - spawn_named(format!("fetch for {:?}", request.current_url_string()), move || { - let request = Rc::new(request); - let fetch_response = fetch(request); - fetch_response.wait_until_done(); - listener.response_available(fetch_response); - }) +pub type Target = Option<Box<FetchTaskTarget + Send>>; + +enum Data { + Payload(Vec<u8>), + Done, +} + +pub struct FetchContext { + pub state: HttpState, + pub user_agent: String, } +type DoneChannel = Option<(Sender<Data>, Receiver<Data>)>; + /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch) -pub fn fetch(request: Rc<Request>) -> Response { - fetch_with_cors_cache(request, &mut CORSCache::new()) +pub fn fetch(request: Rc<Request>, target: &mut Target, context: FetchContext) -> Response { + fetch_with_cors_cache(request, &mut CORSCache::new(), target, context) } -pub fn fetch_with_cors_cache(request: Rc<Request>, cache: &mut CORSCache) -> Response { +pub fn fetch_with_cors_cache(request: Rc<Request>, + cache: &mut CORSCache, + target: &mut Target, + context: FetchContext) -> Response { // Step 1 if request.window.get() == Window::Client { // TODO: Set window to request's client object if client is a Window object @@ -105,12 +118,15 @@ pub fn fetch_with_cors_cache(request: Rc<Request>, cache: &mut CORSCache) -> Res if request.is_subresource_request() { // TODO: create a fetch record and append it to request's client's fetch group list } + // Step 7 - main_fetch(request, cache, false, false) + main_fetch(request, cache, false, false, target, &mut None, &context) } /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch) -fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recursive_flag: bool) -> Response { +fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, + recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel, + context: &FetchContext) -> Response { // TODO: Implement main fetch spec // Step 1 @@ -128,23 +144,39 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu // TODO be able to execute report CSP // Step 4 - // TODO this step, based off of http_loader.rs + // TODO this step, based off of http_loader.rs (upgrade) // Step 5 - // TODO this step + // TODO this step (CSP port/content blocking) // Step 6 - if request.referer != Referer::NoReferer { - // TODO be able to invoke "determine request's referer" - } + // TODO this step (referer policy) + // currently the clients themselves set referer policy in RequestInit // Step 7 - // TODO this step + if request.referrer_policy.get().is_none() { + request.referrer_policy.set(Some(ReferrerPolicy::NoRefWhenDowngrade)); + } // Step 8 - // this step is obsoleted by fetch_async + if *request.referer.borrow() != Referer::NoReferer { + // remove Referer headers set in past redirects/preflights + // this stops the assertion in determine_request_referrer from failing + request.headers.borrow_mut().remove::<RefererHeader>(); + let referrer_url = determine_request_referrer(&mut *request.headers.borrow_mut(), + request.referrer_policy.get(), + request.referer.borrow_mut().take(), + request.current_url().clone()); + *request.referer.borrow_mut() = Referer::from_url(referrer_url); + } // Step 9 + // TODO this step (HSTS) + + // Step 10 + // this step is obsoleted by fetch_async + + // Step 11 let response = match response { Some(response) => response, None => { @@ -160,14 +192,14 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu (current_url.scheme() == "file" && request.same_origin_data.get()) || current_url.scheme() == "about" || request.mode == RequestMode::Navigate { - basic_fetch(request.clone(), cache) + basic_fetch(request.clone(), cache, target, done_chan, context) } else if request.mode == RequestMode::SameOrigin { Response::network_error() } else if request.mode == RequestMode::NoCORS { request.response_tainting.set(ResponseTainting::Opaque); - basic_fetch(request.clone(), cache) + basic_fetch(request.clone(), cache, target, done_chan, context) } else if !matches!(current_url.scheme(), "http" | "https") { Response::network_error() @@ -178,7 +210,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) { request.response_tainting.set(ResponseTainting::CORSTainting); request.redirect_mode.set(RedirectMode::Error); - let response = http_fetch(request.clone(), cache, true, true, false); + let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, context); if response.is_network_error() { // TODO clear cache entries using request } @@ -186,17 +218,17 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu } else { request.response_tainting.set(ResponseTainting::CORSTainting); - http_fetch(request.clone(), cache, true, false, false) + http_fetch(request.clone(), cache, true, false, false, target, done_chan, context) } } }; - // Step 10 + // Step 12 if recursive_flag { return response; } - // Step 11 + // Step 13 // no need to check if response is a network error, since the type would not be `Default` let response = if response.response_type == ResponseType::Default { let response_type = match request.response_tainting.get() { @@ -210,7 +242,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu }; { - // Step 12 + // Step 14 let network_error_res = Response::network_error(); let internal_response = if response.is_network_error() { &network_error_res @@ -218,10 +250,15 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu response.actual_response() }; - // Step 13 - // TODO this step + // Step 15 + if internal_response.url_list.borrow().is_empty() { + *internal_response.url_list.borrow_mut() = request.url_list.borrow().clone(); + } + + // Step 16 + // TODO this step (CSP/blocking) - // Step 14 + // Step 17 if !response.is_network_error() && (is_null_body_status(&internal_response.status) || match *request.method.borrow() { Method::Head | Method::Connect => true, @@ -233,7 +270,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu *body = ResponseBody::Empty; } - // Step 15 + // Step 18 // TODO be able to compare response integrity against request integrity metadata // if !response.is_network_error() { @@ -248,34 +285,91 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu // } } - // Step 16 + // Step 19 if request.synchronous { - response.actual_response().wait_until_done(); + if let Some(ref mut target) = *target { + // process_response is not supposed to be used + // by sync fetch, but we overload it here for simplicity + target.process_response(&response); + } + + if let Some(ref ch) = *done_chan { + loop { + match ch.1.recv() + .expect("fetch worker should always send Done before terminating") { + Data::Payload(vec) => { + if let Some(ref mut target) = *target { + target.process_response_chunk(vec); + } + } + Data::Done => break, + } + } + } else if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { + // in case there was no channel to wait for, the body was + // obtained synchronously via basic_fetch for data/file/about/etc + // We should still send the body across as a chunk + if let Some(ref mut target) = *target { + target.process_response_chunk(vec.clone()); + } + } else { + assert!(*response.body.lock().unwrap() == ResponseBody::Empty) + } + + // overloaded similarly to process_response + if let Some(ref mut target) = *target { + target.process_response_eof(&response); + } return response; } - // Step 17 + // Step 20 if request.body.borrow().is_some() && matches!(request.current_url().scheme(), "http" | "https") { - // TODO queue a fetch task on request to process end-of-file + if let Some(ref mut target) = *target { + // XXXManishearth: We actually should be calling process_request + // in http_network_fetch. However, we can't yet follow the request + // upload progress, so I'm keeping it here for now and pretending + // the body got sent in one chunk + target.process_request_body(&request); + target.process_request_eof(&request); + } } - { - // Step 12 repeated to use internal_response - let network_error_res = Response::network_error(); - let internal_response = if response.is_network_error() { - &network_error_res - } else { - response.actual_response() - }; + // Step 21 + if let Some(ref mut target) = *target { + target.process_response(&response); + } - // Step 18 - // TODO this step + // Step 22 + if let Some(ref ch) = *done_chan { + loop { + match ch.1.recv() + .expect("fetch worker should always send Done before terminating") { + Data::Payload(vec) => { + if let Some(ref mut target) = *target { + target.process_response_chunk(vec); + } + } + Data::Done => break, + } + } + } else if let Some(ref mut target) = *target { + if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { + // in case there was no channel to wait for, the body was + // obtained synchronously via basic_fetch for data/file/about/etc + // We should still send the body across as a chunk + target.process_response_chunk(vec.clone()); + } else { + assert!(*response.body.lock().unwrap() == ResponseBody::Empty) + } + } - // Step 19 - internal_response.wait_until_done(); + // Step 23 + request.done.set(true); - // Step 20 - // TODO this step + // Step 24 + if let Some(ref mut target) = *target { + target.process_response_eof(&response); } // TODO remove this line when only asynchronous fetches are used @@ -283,19 +377,23 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu } /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch) -fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response { +fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache, + target: &mut Target, done_chan: &mut DoneChannel, + context: &FetchContext) -> Response { let url = request.current_url(); match url.scheme() { "about" if url.path() == "blank" => { let mut response = Response::new(); + // https://github.com/whatwg/fetch/issues/312 + response.url = Some(url); response.headers.set(ContentType(mime!(Text / Html; Charset = Utf8))); *response.body.lock().unwrap() = ResponseBody::Done(vec![]); response }, "http" | "https" => { - http_fetch(request.clone(), cache, false, false, false) + http_fetch(request.clone(), cache, false, false, false, target, done_chan, context) }, "data" => { @@ -303,6 +401,8 @@ fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response { match decode(&url) { Ok((mime, bytes)) => { let mut response = Response::new(); + // https://github.com/whatwg/fetch/issues/312 + response.url = Some(url.clone()); *response.body.lock().unwrap() = ResponseBody::Done(bytes); response.headers.set(ContentType(mime)); response @@ -324,6 +424,8 @@ fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response { let mime = guess_mime_type(file_path); let mut response = Response::new(); + // https://github.com/whatwg/fetch/issues/312 + response.url = Some(url.clone()); *response.body.lock().unwrap() = ResponseBody::Done(bytes); response.headers.set(ContentType(mime)); response @@ -350,7 +452,12 @@ fn http_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, cors_preflight_flag: bool, - authentication_fetch_flag: bool) -> Response { + authentication_fetch_flag: bool, + target: &mut Target, + done_chan: &mut DoneChannel, + context: &FetchContext) -> Response { + // This is a new async fetch, reset the channel we are waiting on + *done_chan = None; // Step 1 let mut response: Option<Response> = None; @@ -378,17 +485,18 @@ fn http_fetch(request: Rc<Request>, } // Substep 4 - let actual_response = res.actual_response(); - if actual_response.url_list.borrow().is_empty() { - *actual_response.url_list.borrow_mut() = request.url_list.borrow().clone(); - } - - // Substep 5 // TODO: set response's CSP list on actual_response } } // Step 4 + let credentials = match request.credentials_mode { + CredentialsMode::Include => true, + CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic + => true, + _ => false + }; + // Step 5 if response.is_none() { // Substep 1 if cors_preflight_flag { @@ -403,7 +511,7 @@ fn http_fetch(request: Rc<Request>, // Sub-substep 1 if method_mismatch || header_mismatch { - let preflight_result = cors_preflight_fetch(request.clone(), cache); + let preflight_result = cors_preflight_fetch(request.clone(), cache, context); // Sub-substep 2 if preflight_result.response_type == ResponseType::Error { return Response::network_error(); @@ -415,17 +523,10 @@ fn http_fetch(request: Rc<Request>, request.skip_service_worker.set(true); // Substep 3 - let credentials = match request.credentials_mode { - CredentialsMode::Include => true, - CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic - => true, - _ => false - }; + let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag, + done_chan, context); // Substep 4 - let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag); - - // Substep 5 if cors_flag && cors_check(request.clone(), &fetch_result).is_err() { return Response::network_error(); } @@ -450,7 +551,8 @@ fn http_fetch(request: Rc<Request>, RedirectMode::Follow => { // set back to default response.return_internal.set(true); - http_redirect_fetch(request, cache, Rc::new(response), cors_flag) + http_redirect_fetch(request, cache, Rc::new(response), + cors_flag, target, done_chan, context) } } }, @@ -459,7 +561,7 @@ fn http_fetch(request: Rc<Request>, StatusCode::Unauthorized => { // Step 1 // FIXME: Figure out what to do with request window objects - if cors_flag || request.credentials_mode != CredentialsMode::Include { + if cors_flag || !credentials { return response; } @@ -469,10 +571,15 @@ fn http_fetch(request: Rc<Request>, // Step 3 if !request.use_url_credentials || authentication_fetch_flag { // TODO: Prompt the user for username and password from the window + // Wrong, but will have to do until we are able to prompt the user + // otherwise this creates an infinite loop + // We basically pretend that the user declined to enter credentials + return response; } // Step 4 - return http_fetch(request, cache, cors_flag, cors_preflight_flag, true); + return http_fetch(request, cache, cors_flag, cors_preflight_flag, + true, target, done_chan, context); } // Code 407 @@ -485,11 +592,16 @@ fn http_fetch(request: Rc<Request>, // Step 3 // TODO: Prompt the user for proxy authentication credentials + // Wrong, but will have to do until we are able to prompt the user + // otherwise this creates an infinite loop + // We basically pretend that the user declined to enter credentials + return response; // Step 4 - return http_fetch(request, cache, - cors_flag, cors_preflight_flag, - authentication_fetch_flag); + // return http_fetch(request, cache, + // cors_flag, cors_preflight_flag, + // authentication_fetch_flag, target, + // done_chan, context); } _ => { } @@ -510,7 +622,10 @@ fn http_fetch(request: Rc<Request>, fn http_redirect_fetch(request: Rc<Request>, cache: &mut CORSCache, response: Rc<Response>, - cors_flag: bool) -> Response { + cors_flag: bool, + target: &mut Target, + done_chan: &mut DoneChannel, + context: &FetchContext) -> Response { // Step 1 assert_eq!(response.return_internal.get(), true); @@ -584,20 +699,22 @@ fn http_redirect_fetch(request: Rc<Request>, request.url_list.borrow_mut().push(location_url); // Step 15 - main_fetch(request, cache, cors_flag, true) + main_fetch(request, cache, cors_flag, true, target, done_chan, context) } /// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch) fn http_network_or_cache_fetch(request: Rc<Request>, credentials_flag: bool, - authentication_fetch_flag: bool) -> Response { + authentication_fetch_flag: bool, + done_chan: &mut DoneChannel, + context: &FetchContext) -> Response { // TODO: Implement Window enum for Request let request_has_no_window = true; // Step 1 let http_request = if request_has_no_window && - request.redirect_mode.get() != RedirectMode::Follow { - request.clone() + request.redirect_mode.get() == RedirectMode::Error { + request } else { Rc::new((*request).clone()) }; @@ -621,9 +738,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>, } // Step 6 - match http_request.referer { - Referer::NoReferer => - http_request.headers.borrow_mut().set(RefererHeader("".to_owned())), + match *http_request.referer.borrow() { + Referer::NoReferer => (), Referer::RefererUrl(ref http_request_referer) => http_request.headers.borrow_mut().set(RefererHeader(http_request_referer.to_string())), Referer::Client => @@ -640,7 +756,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>, // Step 8 if !http_request.headers.borrow().has::<UserAgent>() { - http_request.headers.borrow_mut().set(UserAgent(global_user_agent().to_owned())); + http_request.headers.borrow_mut().set(UserAgent(context.user_agent.clone())); } match http_request.cache_mode.get() { @@ -670,34 +786,53 @@ fn http_network_or_cache_fetch(request: Rc<Request>, _ => {} } + let current_url = http_request.current_url(); // Step 12 - // modify_request_headers(http_request.headers.borrow()); + // todo: pass referrer url and policy + // this can only be uncommented when the referer header is set, else it crashes + // in the meantime, we manually set the headers in the block below + // modify_request_headers(&mut http_request.headers.borrow_mut(), ¤t_url, + // None, None, None); + { + let headers = &mut *http_request.headers.borrow_mut(); + let host = Host { + hostname: current_url.host_str().unwrap().to_owned(), + port: current_url.port_or_known_default() + }; + headers.set(host); + // unlike http_loader, we should not set the accept header + // here, according to the fetch spec + set_default_accept_encoding(headers); + } // Step 13 // TODO some of this step can't be implemented yet if credentials_flag { // Substep 1 // TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504 - + // XXXManishearth http_loader has block_cookies: support content blocking here too + set_request_cookies(¤t_url, + &mut *http_request.headers.borrow_mut(), + &context.state.cookie_jar); // Substep 2 if !http_request.headers.borrow().has::<Authorization<String>>() { // Substep 3 let mut authorization_value = None; // Substep 4 - // TODO be able to retrieve https://fetch.spec.whatwg.org/#authentication-entry + if let Some(basic) = auth_from_cache(&context.state.auth_cache, ¤t_url) { + if !http_request.use_url_credentials || !has_credentials(¤t_url) { + authorization_value = Some(basic); + } + } // Substep 5 - if authentication_fetch_flag { - let current_url = http_request.current_url(); - - authorization_value = if has_credentials(¤t_url) { - Some(Basic { + if authentication_fetch_flag && authorization_value.is_none() { + if has_credentials(¤t_url) { + authorization_value = Some(Basic { username: current_url.username().to_owned(), password: current_url.password().map(str::to_owned) }) - } else { - None } } @@ -753,7 +888,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>, // Step 18 if response.is_none() { - response = Some(http_network_fetch(request.clone(), http_request.clone(), credentials_flag)); + response = Some(http_network_fetch(http_request.clone(), credentials_flag, done_chan)); } let response = response.unwrap(); @@ -788,8 +923,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>, /// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch) fn http_network_fetch(request: Rc<Request>, - _http_request: Rc<Request>, - _credentials_flag: bool) -> Response { + _credentials_flag: bool, + done_chan: &mut DoneChannel) -> Response { // TODO: Implement HTTP network fetch spec // Step 1 @@ -811,42 +946,70 @@ fn http_network_fetch(request: Rc<Request>, let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(), &request.headers.borrow(), - &cancellation_listener, &None, &request.method.borrow(), - &None, request.redirect_count.get(), &None, ""); + &cancellation_listener, &request.body.borrow(), &request.method.borrow(), + &None, request.redirect_count.get() + 1, &None, ""); let mut response = Response::new(); match wrapped_response { - Ok((mut res, _)) => { - response.url = Some(res.response.url.clone()); + Ok((res, _)) => { + response.url = Some(url.clone()); response.status = Some(res.response.status); + response.raw_status = Some(res.response.status_raw().clone()); response.headers = res.response.headers.clone(); let res_body = response.body.clone(); - thread::spawn(move || { - *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); - - loop { - match read_block(&mut res.response) { - Ok(ReadResult::Payload(ref mut chunk)) => { - if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { - body.append(chunk); + + // We're about to spawn a thread to be waited on here + *done_chan = Some(channel()); + let meta = response.metadata().expect("Response metadata should exist at this stage"); + let done_sender = done_chan.as_ref().map(|ch| ch.0.clone()); + spawn_named(format!("fetch worker thread"), move || { + match StreamedResponse::from_http_response(box res, meta) { + Ok(mut res) => { + *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); + loop { + match read_block(&mut res) { + Ok(ReadResult::Payload(chunk)) => { + if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { + body.extend_from_slice(&chunk); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Payload(chunk)); + } + } + }, + Ok(ReadResult::EOF) | Err(_) => { + let mut empty_vec = Vec::new(); + let completed_body = match *res_body.lock().unwrap() { + ResponseBody::Receiving(ref mut body) => { + // avoid cloning the body + swap(body, &mut empty_vec); + empty_vec + }, + _ => empty_vec, + }; + *res_body.lock().unwrap() = ResponseBody::Done(completed_body); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Done); + } + break; + } } - }, - Ok(ReadResult::EOF) | Err(_) => { - let completed_body = match *res_body.lock().unwrap() { - ResponseBody::Receiving(ref body) => (*body).clone(), - _ => vec![] - }; - *res_body.lock().unwrap() = ResponseBody::Done(completed_body); - break; + + } + } + Err(_) => { + // XXXManishearth we should propagate this error somehow + *res_body.lock().unwrap() = ResponseBody::Done(vec![]); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Done); } } - } }); }, - Err(_) => - response.termination_reason = Some(TerminationReason::Fatal) + Err(_) => { + response.termination_reason = Some(TerminationReason::Fatal); + } }; // TODO these substeps aren't possible yet @@ -860,7 +1023,10 @@ fn http_network_fetch(request: Rc<Request>, // TODO Read request - // Step 5 + // Step 5-9 + // (needs stream bodies) + + // Step 10 // TODO when https://bugzilla.mozilla.org/show_bug.cgi?id=1030660 // is resolved, this step will become uneccesary // TODO this step @@ -872,22 +1038,19 @@ fn http_network_fetch(request: Rc<Request>, } }; - // Step 6 - *response.url_list.borrow_mut() = request.url_list.borrow().clone(); - - // Step 7 - // TODO this step isn't possible yet + // Step 11 + // TODO this step isn't possible yet (CSP) - // Step 8 + // Step 12 if response.is_network_error() && request.cache_mode.get() == CacheMode::NoStore { // TODO update response in the HTTP cache for request } // TODO this step isn't possible yet - // Step 9 + // Step 13 // TODO these steps - // Step 10 + // Step 14 // Substep 1 // Substep 2 // Sub-substep 1 @@ -896,19 +1059,20 @@ fn http_network_fetch(request: Rc<Request>, // Sub-substep 4 // Substep 3 - // Step 11 + // Step 15 response } /// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch) -fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response { +fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, context: &FetchContext) -> Response { // Step 1 let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()), false); *preflight.method.borrow_mut() = Method::Options; preflight.initiator = request.initiator.clone(); preflight.type_ = request.type_.clone(); preflight.destination = request.destination.clone(); - preflight.referer = request.referer.clone(); + *preflight.referer.borrow_mut() = request.referer.borrow().clone(); + preflight.referrer_policy.set(preflight.referrer_policy.get()); // Step 2 preflight.headers.borrow_mut().set::<AccessControlRequestMethod>( @@ -929,7 +1093,7 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response // Step 6 let preflight = Rc::new(preflight); - let response = http_network_or_cache_fetch(preflight.clone(), false, false); + let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, context); // Step 7 if cors_check(request.clone(), &response).is_ok() && @@ -962,12 +1126,16 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response } // Substep 5 + debug!("CORS check: Allowed methods: {:?}, current method: {:?}", + methods, request.method.borrow()); if methods.iter().all(|method| *method != *request.method.borrow()) && !is_simple_method(&*request.method.borrow()) { return Response::network_error(); } // Substep 6 + debug!("CORS check: Allowed headers: {:?}, current headers: {:?}", + header_names, request.headers.borrow()); let set: HashSet<&UniCase<String>> = HashSet::from_iter(header_names.iter()); if request.headers.borrow().iter().any(|ref hv| !set.contains(&UniCase(hv.name().to_owned())) && !is_simple_header(hv)) { @@ -1040,12 +1208,6 @@ fn cors_check(request: Rc<Request>, response: &Response) -> Result<(), ()> { Err(()) } -fn global_user_agent() -> String { - // TODO have a better useragent string - const USER_AGENT_STRING: &'static str = "Servo"; - USER_AGENT_STRING.to_owned() -} - fn has_credentials(url: &Url) -> bool { !url.username().is_empty() || url.password().is_some() } @@ -1056,6 +1218,7 @@ fn is_no_store_cache(headers: &Headers) -> bool { headers.has::<IfRange>() } +/// https://fetch.spec.whatwg.org/#cors-safelisted-request-header fn is_simple_header(h: &HeaderView) -> bool { if h.is::<ContentType>() { match h.value() { diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index f94b6ffce32..2855e38bf45 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -370,7 +370,7 @@ impl Error for LoadErrorType { } } -fn set_default_accept_encoding(headers: &mut Headers) { +pub fn set_default_accept_encoding(headers: &mut Headers) { if headers.has::<AcceptEncoding>() { return } @@ -434,10 +434,10 @@ fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option<Url> { } /// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer -fn determine_request_referrer(headers: &mut Headers, - referrer_policy: Option<ReferrerPolicy>, - referrer_url: Option<Url>, - url: Url) -> Option<Url> { +pub fn determine_request_referrer(headers: &mut Headers, + referrer_policy: Option<ReferrerPolicy>, + referrer_url: Option<Url>, + url: Url) -> Option<Url> { //TODO - algorithm step 2 not addressed assert!(!headers.has::<Referer>()); if let Some(ref_url) = referrer_url { @@ -453,9 +453,9 @@ fn determine_request_referrer(headers: &mut Headers, return None; } -pub fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) { +pub fn set_request_cookies(url: &Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) { let mut cookie_jar = cookie_jar.write().unwrap(); - if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) { + if let Some(cookie_list) = cookie_jar.cookies_for_url(url, CookieSource::HTTP) { let mut v = Vec::new(); v.push(cookie_list.into_bytes()); headers.set_raw("Cookie".to_owned(), v); @@ -540,7 +540,7 @@ impl StreamedResponse { StreamedResponse { metadata: m, decoder: d } } - fn from_http_response(response: Box<HttpResponse>, m: Metadata) -> Result<StreamedResponse, LoadError> { + pub fn from_http_response(response: Box<HttpResponse>, m: Metadata) -> Result<StreamedResponse, LoadError> { let decoder = match response.content_encoding() { Some(Encoding::Gzip) => { let result = GzDecoder::new(response); @@ -627,10 +627,7 @@ fn request_must_be_secured(url: &Url, hsts_list: &Arc<RwLock<HstsList>>) -> bool pub fn modify_request_headers(headers: &mut Headers, url: &Url, user_agent: &str, - cookie_jar: &Arc<RwLock<CookieStorage>>, - auth_cache: &Arc<RwLock<AuthCache>>, - load_data: &LoadData, - block_cookies: bool, + referrer_policy: Option<ReferrerPolicy>, referrer_url: &mut Option<Url>) { // Ensure that the host header is set from the original url let host = Host { @@ -654,23 +651,13 @@ pub fn modify_request_headers(headers: &mut Headers, set_default_accept_encoding(headers); *referrer_url = determine_request_referrer(headers, - load_data.referrer_policy.clone(), + referrer_policy.clone(), referrer_url.clone(), url.clone()); if let Some(referer_val) = referrer_url.clone() { headers.set(Referer(referer_val.into_string())); } - - // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11 - if load_data.credentials_flag { - if !block_cookies { - set_request_cookies(url.clone(), headers, cookie_jar); - } - - // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12 - set_auth_header(headers, url, auth_cache); - } } fn set_auth_header(headers: &mut Headers, @@ -680,18 +667,21 @@ fn set_auth_header(headers: &mut Headers, if let Some(auth) = auth_from_url(url) { headers.set(auth); } else { - if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) { - auth_from_entry(&auth_entry, headers); + if let Some(basic) = auth_from_cache(auth_cache, url) { + headers.set(Authorization(basic)); } } } } -fn auth_from_entry(auth_entry: &AuthCacheEntry, headers: &mut Headers) { - let user_name = auth_entry.user_name.clone(); - let password = Some(auth_entry.password.clone()); - - headers.set(Authorization(Basic { username: user_name, password: password })); +pub fn auth_from_cache(auth_cache: &Arc<RwLock<AuthCache>>, url: &Url) -> Option<Basic> { + if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) { + let user_name = auth_entry.user_name.clone(); + let password = Some(auth_entry.password.clone()); + Some(Basic { username: user_name, password: password }) + } else { + None + } } fn auth_from_url(doc_url: &Url) -> Option<Authorization<Basic>> { @@ -956,9 +946,18 @@ pub fn load<A, B>(load_data: &LoadData, let request_id = uuid::Uuid::new_v4().simple().to_string(); modify_request_headers(&mut request_headers, &doc_url, - &user_agent, &http_state.cookie_jar, - &http_state.auth_cache, &load_data, - block_cookies, &mut referrer_url); + &user_agent, load_data.referrer_policy, + &mut referrer_url); + + // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11 + if load_data.credentials_flag { + if !block_cookies { + set_request_cookies(&doc_url, &mut request_headers, &http_state.cookie_jar); + } + + // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12 + set_auth_header(&mut request_headers, &doc_url, &http_state.auth_cache); + } //if there is a new auth header then set the request headers with it if let Some(ref auth_header) = new_auth_header { diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index fecbff3e077..6e94fd1cf03 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -11,6 +11,7 @@ use cookie; use cookie_storage::CookieStorage; use data_loader; use devtools_traits::DevtoolsControlMsg; +use fetch::methods::{fetch, FetchContext}; use file_loader; use filemanager_thread::FileManagerThreadFactory; use hsts::HstsList; @@ -22,9 +23,11 @@ use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag}; use net_traits::LoadContext; use net_traits::ProgressMsg::Done; +use net_traits::request::{Request, RequestInit}; use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, CoreResourceThread}; -use net_traits::{CoreResourceMsg, CookieSource, LoadConsumer, LoadData, LoadResponse, ResourceId}; -use net_traits::{NetworkError, WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; +use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, FetchTaskTarget, LoadConsumer}; +use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId}; +use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; use profile_traits::time::ProfilerChan; use rustc_serialize::json; use rustc_serialize::{Decodable, Encodable}; @@ -36,6 +39,7 @@ use std::error::Error; use std::fs::File; use std::io::prelude::*; use std::path::Path; +use std::rc::Rc; use std::sync::mpsc::{Receiver, Sender, channel}; use std::sync::{Arc, RwLock}; use storage_thread::StorageThreadFactory; @@ -193,6 +197,8 @@ impl ResourceChannelManager { match self.from_client.recv().unwrap() { CoreResourceMsg::Load(load_data, consumer, id_sender) => self.resource_manager.load(load_data, consumer, id_sender, control_sender.clone()), + CoreResourceMsg::Fetch(init, sender) => + self.resource_manager.fetch(init, sender), CoreResourceMsg::WebsocketConnect(connect, connect_data) => self.resource_manager.websocket_connect(connect, connect_data), CoreResourceMsg::SetCookiesForUrl(request, cookie_list, source) => @@ -480,6 +486,26 @@ impl CoreResourceManager { cancel_listener)); } + fn fetch(&self, init: RequestInit, sender: IpcSender<FetchResponseMsg>) { + let http_state = HttpState { + hsts_list: self.hsts_list.clone(), + cookie_jar: self.cookie_jar.clone(), + auth_cache: self.auth_cache.clone(), + blocked_content: BLOCKED_CONTENT_RULES.clone(), + }; + let ua = self.user_agent.clone(); + spawn_named(format!("fetch thread for {}", init.url), move || { + let request = Request::from_init(init); + // XXXManishearth: Check origin against pipeline id (also ensure that the mode is allowed) + // todo load context / mimesniff in fetch + // todo referrer policy? + // todo service worker stuff + let mut target = Some(Box::new(sender) as Box<FetchTaskTarget + Send + 'static>); + let context = FetchContext { state: http_state, user_agent: ua }; + fetch(Rc::new(request), &mut target, context); + }) + } + fn websocket_connect(&self, connect: WebSocketCommunicate, connect_data: WebSocketConnectData) { diff --git a/components/net/websocket_loader.rs b/components/net/websocket_loader.rs index 60d6c8932f6..d748e309d20 100644 --- a/components/net/websocket_loader.rs +++ b/components/net/websocket_loader.rs @@ -43,7 +43,7 @@ fn establish_a_websocket_connection(resource_url: &Url, net_url: (Host, String, request.headers.set(WebSocketProtocol(protocols.clone())); }; - http_loader::set_request_cookies(resource_url.clone(), &mut request.headers, &cookie_jar); + http_loader::set_request_cookies(&resource_url, &mut request.headers, &cookie_jar); let response = try!(request.send()); try!(response.validate()); diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 4af820b0ebd..bb8638d7c3c 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -36,6 +36,8 @@ use hyper::method::Method; use hyper::mime::{Attr, Mime}; use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use msg::constellation_msg::{PipelineId, ReferrerPolicy}; +use request::{Request, RequestInit}; +use response::{HttpsState, Response}; use std::io::Error as IOError; use std::sync::mpsc::Sender; use std::thread; @@ -112,6 +114,7 @@ pub struct LoadData { pub headers: Headers, #[ignore_heap_size_of = "Defined in hyper"] /// Headers that will apply to the initial request and any redirects + /// Unused in fetch pub preserved_headers: Headers, pub data: Option<Vec<u8>>, pub cors: Option<ResourceCORSData>, @@ -154,9 +157,81 @@ pub trait LoadOrigin { fn pipeline_id(&self) -> Option<PipelineId>; } -/// Interface for observing the final response for an asynchronous fetch operation. -pub trait AsyncFetchListener { - fn response_available(&self, response: response::Response); +#[derive(Deserialize, Serialize)] +pub enum FetchResponseMsg { + // todo: should have fields for transmitted/total bytes + ProcessRequestBody, + ProcessRequestEOF, + // todo: send more info about the response (or perhaps the entire Response) + ProcessResponse(Result<Metadata, NetworkError>), + ProcessResponseChunk(Vec<u8>), + ProcessResponseEOF(Result<(), NetworkError>), +} + +pub trait FetchTaskTarget { + /// https://fetch.spec.whatwg.org/#process-request-body + /// + /// Fired when a chunk of the request body is transmitted + fn process_request_body(&mut self, request: &Request); + + /// https://fetch.spec.whatwg.org/#process-request-end-of-file + /// + /// Fired when the entire request finishes being transmitted + fn process_request_eof(&mut self, request: &Request); + + /// https://fetch.spec.whatwg.org/#process-response + /// + /// Fired when headers are received + fn process_response(&mut self, response: &Response); + + /// Fired when a chunk of response content is received + fn process_response_chunk(&mut self, chunk: Vec<u8>); + + /// https://fetch.spec.whatwg.org/#process-response-end-of-file + /// + /// Fired when the response is fully fetched + fn process_response_eof(&mut self, response: &Response); +} + +pub trait FetchResponseListener { + fn process_request_body(&mut self); + fn process_request_eof(&mut self); + fn process_response(&mut self, metadata: Result<Metadata, NetworkError>); + fn process_response_chunk(&mut self, chunk: Vec<u8>); + fn process_response_eof(&mut self, response: Result<(), NetworkError>); +} + +impl FetchTaskTarget for IpcSender<FetchResponseMsg> { + fn process_request_body(&mut self, _: &Request) { + let _ = self.send(FetchResponseMsg::ProcessRequestBody); + } + + fn process_request_eof(&mut self, _: &Request) { + let _ = self.send(FetchResponseMsg::ProcessRequestEOF); + } + + fn process_response(&mut self, response: &Response) { + let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata())); + } + + fn process_response_chunk(&mut self, chunk: Vec<u8>) { + let _ = self.send(FetchResponseMsg::ProcessResponseChunk(chunk)); + } + + fn process_response_eof(&mut self, response: &Response) { + if response.is_network_error() { + // todo: finer grained errors + let _ = self.send(FetchResponseMsg::ProcessResponseEOF( + Err(NetworkError::Internal("Network error".into())))); + } else { + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(()))); + } + } +} + + +pub trait Action<Listener> { + fn process(self, listener: &mut Listener); } /// A listener for asynchronous network events. Cancelling the underlying request is unsupported. @@ -183,9 +258,9 @@ pub enum ResponseAction { ResponseComplete(Result<(), NetworkError>) } -impl ResponseAction { +impl<T: AsyncResponseListener> Action<T> for ResponseAction { /// Execute the default action on a provided listener. - pub fn process(self, listener: &mut AsyncResponseListener) { + fn process(self, listener: &mut T) { match self { ResponseAction::HeadersAvailable(m) => listener.headers_available(m), ResponseAction::DataAvailable(d) => listener.data_available(d), @@ -194,6 +269,19 @@ impl ResponseAction { } } +impl<T: FetchResponseListener> Action<T> for FetchResponseMsg { + /// Execute the default action on a provided listener. + fn process(self, listener: &mut T) { + match self { + FetchResponseMsg::ProcessRequestBody => listener.process_request_body(), + FetchResponseMsg::ProcessRequestEOF => listener.process_request_eof(), + FetchResponseMsg::ProcessResponse(meta) => listener.process_response(meta), + FetchResponseMsg::ProcessResponseChunk(data) => listener.process_response_chunk(data), + FetchResponseMsg::ProcessResponseEOF(data) => listener.process_response_eof(data), + } + } +} + /// A target for async networking events. Commonly used to dispatch a runnable event to another /// thread storing the wrapped closure for later execution. #[derive(Deserialize, Serialize)] @@ -331,6 +419,7 @@ pub struct WebSocketConnectData { pub enum CoreResourceMsg { /// Request the data associated with a particular URL Load(LoadData, LoadConsumer, Option<IpcSender<ResourceId>>), + Fetch(RequestInit, IpcSender<FetchResponseMsg>), /// Try to make a websocket connection to a URL. WebsocketConnect(WebSocketCommunicate, WebSocketConnectData), /// Store a set of cookies for a given originating URL @@ -469,7 +558,7 @@ pub struct Metadata { pub status: Option<RawStatus>, /// Is successful HTTPS connection - pub https_state: response::HttpsState, + pub https_state: HttpsState, } impl Metadata { @@ -482,7 +571,7 @@ impl Metadata { headers: None, // https://fetch.spec.whatwg.org/#concept-response-status-message status: Some(RawStatus(200, "OK".into())), - https_state: response::HttpsState::None, + https_state: HttpsState::None, } } diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index 675a78fd788..1fbcdabe868 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -4,7 +4,9 @@ use hyper::header::Headers; use hyper::method::Method; +use msg::constellation_msg::ReferrerPolicy; use std::cell::{Cell, RefCell}; +use std::mem::swap; use url::{Origin as UrlOrigin, Url}; /// An [initiator](https://fetch.spec.whatwg.org/#concept-request-initiator) @@ -25,7 +27,7 @@ pub enum Type { } /// A request [destination](https://fetch.spec.whatwg.org/#concept-request-destination) -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum Destination { None, Document, Embed, Font, Image, Manifest, Media, Object, Report, Script, ServiceWorker, @@ -43,12 +45,13 @@ pub enum Origin { #[derive(Clone, PartialEq)] pub enum Referer { NoReferer, + /// Default referer if nothing is specified Client, RefererUrl(Url) } /// A [request mode](https://fetch.spec.whatwg.org/#concept-request-mode) -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum RequestMode { Navigate, SameOrigin, @@ -57,7 +60,7 @@ pub enum RequestMode { } /// Request [credentials mode](https://fetch.spec.whatwg.org/#concept-request-credentials-mode) -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum CredentialsMode { Omit, CredentialsSameOrigin, @@ -106,6 +109,29 @@ pub enum CORSSettings { UseCredentials } +#[derive(Serialize, Deserialize, Clone)] +pub struct RequestInit { + pub method: Method, + pub url: Url, + pub headers: Headers, + pub unsafe_request: bool, + pub same_origin_data: bool, + pub body: Option<Vec<u8>>, + // TODO: client object + pub destination: Destination, + pub synchronous: bool, + pub mode: RequestMode, + pub use_cors_preflight: bool, + pub credentials_mode: CredentialsMode, + pub use_url_credentials: bool, + // this should actually be set by fetch, but fetch + // doesn't have info about the client right now + pub origin: Url, + // XXXManishearth these should be part of the client object + pub referer_url: Option<Url>, + pub referrer_policy: Option<ReferrerPolicy>, +} + /// A [Request](https://fetch.spec.whatwg.org/#requests) as defined by the Fetch spec #[derive(Clone)] pub struct Request { @@ -130,8 +156,9 @@ pub struct Request { pub origin: RefCell<Origin>, pub omit_origin_header: Cell<bool>, pub same_origin_data: Cell<bool>, - pub referer: Referer, - // TODO: referrer policy + /// https://fetch.spec.whatwg.org/#concept-request-referrer + pub referer: RefCell<Referer>, + pub referrer_policy: Cell<Option<ReferrerPolicy>>, pub synchronous: bool, pub mode: RequestMode, pub use_cors_preflight: bool, @@ -145,7 +172,7 @@ pub struct Request { pub url_list: RefCell<Vec<Url>>, pub redirect_count: Cell<u32>, pub response_tainting: Cell<ResponseTainting>, - pub done: Cell<bool> + pub done: Cell<bool>, } impl Request { @@ -169,7 +196,8 @@ impl Request { origin: RefCell::new(origin.unwrap_or(Origin::Client)), omit_origin_header: Cell::new(false), same_origin_data: Cell::new(false), - referer: Referer::Client, + referer: RefCell::new(Referer::Client), + referrer_policy: Cell::new(None), synchronous: false, mode: RequestMode::NoCORS, use_cors_preflight: false, @@ -185,6 +213,30 @@ impl Request { } } + pub fn from_init(init: RequestInit) -> Request { + let mut req = Request::new(init.url, + Some(Origin::Origin(init.origin.origin())), + false); + *req.method.borrow_mut() = init.method; + *req.headers.borrow_mut() = init.headers; + req.unsafe_request = init.unsafe_request; + req.same_origin_data.set(init.same_origin_data); + *req.body.borrow_mut() = init.body; + req.destination = init.destination; + req.synchronous = init.synchronous; + req.mode = init.mode; + req.use_cors_preflight = init.use_cors_preflight; + req.credentials_mode = init.credentials_mode; + req.use_url_credentials = init.use_url_credentials; + *req.referer.borrow_mut() = if let Some(url) = init.referer_url { + Referer::RefererUrl(url) + } else { + Referer::NoReferer + }; + req.referrer_policy.set(init.referrer_policy); + req + } + /// https://html.spec.whatwg.org/multipage/#create-a-potential-cors-request pub fn potential_cors_request(url: Url, cors_attribute_state: Option<CORSSettings>, @@ -207,7 +259,8 @@ impl Request { origin: RefCell::new(Origin::Client), omit_origin_header: Cell::new(false), same_origin_data: Cell::new(false), - referer: Referer::Client, + referer: RefCell::new(Referer::Client), + referrer_policy: Cell::new(None), synchronous: false, // Step 1-2 mode: match cors_attribute_state { @@ -258,3 +311,27 @@ impl Request { } } } + +impl Referer { + pub fn to_url(&self) -> Option<&Url> { + match *self { + Referer::NoReferer | Referer::Client => None, + Referer::RefererUrl(ref url) => Some(url) + } + } + pub fn from_url(url: Option<Url>) -> Self { + if let Some(url) = url { + Referer::RefererUrl(url) + } else { + Referer::NoReferer + } + } + pub fn take(&mut self) -> Option<Url> { + let mut new = Referer::Client; + swap(self, &mut new); + match new { + Referer::NoReferer | Referer::Client => None, + Referer::RefererUrl(url) => Some(url) + } + } +} diff --git a/components/net_traits/response.rs b/components/net_traits/response.rs index d107e219988..ab1d5b1c8f5 100644 --- a/components/net_traits/response.rs +++ b/components/net_traits/response.rs @@ -4,15 +4,17 @@ //! The [Response](https://fetch.spec.whatwg.org/#responses) object //! resulting from a [fetch operation](https://fetch.spec.whatwg.org/#concept-fetch) -use hyper::header::{AccessControlExposeHeaders, Headers}; +use hyper::header::{AccessControlExposeHeaders, ContentType, Headers}; +use hyper::http::RawStatus; use hyper::status::StatusCode; use std::ascii::AsciiExt; use std::cell::{Cell, RefCell}; use std::sync::{Arc, Mutex}; use url::Url; +use {Metadata, NetworkError}; /// [Response type](https://fetch.spec.whatwg.org/#concept-response-type) -#[derive(Clone, PartialEq, Copy, Debug)] +#[derive(Clone, PartialEq, Copy, Debug, Deserialize, Serialize)] pub enum ResponseType { Basic, CORS, @@ -23,7 +25,7 @@ pub enum ResponseType { } /// [Response termination reason](https://fetch.spec.whatwg.org/#concept-response-termination-reason) -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Deserialize, Serialize)] pub enum TerminationReason { EndUserAbort, Fatal, @@ -50,7 +52,7 @@ impl ResponseBody { /// [Cache state](https://fetch.spec.whatwg.org/#concept-response-cache-state) -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum CacheState { None, Local, @@ -81,6 +83,7 @@ pub struct Response { pub url_list: RefCell<Vec<Url>>, /// `None` can be considered a StatusCode of `0`. pub status: Option<StatusCode>, + pub raw_status: Option<RawStatus>, pub headers: Headers, pub body: Arc<Mutex<ResponseBody>>, pub cache_state: CacheState, @@ -100,6 +103,7 @@ impl Response { url: None, url_list: RefCell::new(Vec::new()), status: Some(StatusCode::Ok), + raw_status: Some(RawStatus(200, "OK".into())), headers: Headers::new(), body: Arc::new(Mutex::new(ResponseBody::Empty)), cache_state: CacheState::None, @@ -116,6 +120,7 @@ impl Response { url: None, url_list: RefCell::new(vec![]), status: None, + raw_status: None, headers: Headers::new(), body: Arc::new(Mutex::new(ResponseBody::Empty)), cache_state: CacheState::None, @@ -132,18 +137,6 @@ impl Response { } } - pub fn wait_until_done(&self) { - match self.response_type { - // since these response types can't hold a body, they should be considered done - ResponseType::Error | ResponseType::Opaque | ResponseType::OpaqueRedirect => {}, - _ => { - while !self.body.lock().unwrap().is_done() && !self.is_network_error() { - // loop until done - } - } - } - } - pub fn actual_response(&self) -> &Response { if self.return_internal.get() && self.internal_response.is_some() { &**self.internal_response.as_ref().unwrap() @@ -228,4 +221,25 @@ impl Response { response } + + pub fn metadata(&self) -> Result<Metadata, NetworkError> { + let mut metadata = if let Some(ref url) = self.url { + Metadata::default(url.clone()) + } else { + return Err(NetworkError::Internal("No url found in response".to_string())); + }; + + if self.is_network_error() { + return Err(NetworkError::Internal("Cannot extract metadata from network error".to_string())); + } + + metadata.set_content_type(match self.headers.get() { + Some(&ContentType(ref mime)) => Some(mime), + None => None + }); + metadata.headers = Some(self.headers.clone()); + metadata.status = self.raw_status.clone(); + metadata.https_state = self.https_state; + return Ok(metadata); + } } diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml index 056bb6f72bb..95f1044fd36 100644 --- a/components/script/Cargo.toml +++ b/components/script/Cargo.toml @@ -60,7 +60,6 @@ smallvec = "0.1" string_cache = {version = "0.2.18", features = ["heap_size", "unstable"]} style = {path = "../style"} time = "0.1.12" -unicase = "1.0" url = {version = "1.0.0", features = ["heap_size", "query_encoding"]} util = {path = "../util"} uuid = {version = "0.2", features = ["v4"]} diff --git a/components/script/cors.rs b/components/script/cors.rs deleted file mode 100644 index f9d0bbbd264..00000000000 --- a/components/script/cors.rs +++ /dev/null @@ -1,490 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! A partial implementation of CORS -//! For now this library is XHR-specific. -//! For stuff involving `<img>`, `<iframe>`, `<form>`, etc please check what -//! the request mode should be and compare with the fetch spec -//! This library will eventually become the core of the Fetch crate -//! with CORSRequest being expanded into FetchRequest (etc) - -use hyper::client::Request; -use hyper::header::{AccessControlAllowHeaders, AccessControlRequestHeaders}; -use hyper::header::{AccessControlAllowMethods, AccessControlRequestMethod}; -use hyper::header::{AccessControlAllowOrigin, AccessControlMaxAge}; -use hyper::header::{ContentType, Host}; -use hyper::header::{HeaderView, Headers}; -use hyper::method::Method; -use hyper::mime::{Mime, SubLevel, TopLevel}; -use hyper::status::StatusClass::Success; -use net_traits::{AsyncResponseListener, Metadata, NetworkError, ResponseAction}; -use network_listener::{NetworkListener, PreInvoke}; -use script_runtime::ScriptChan; -use std::ascii::AsciiExt; -use std::borrow::ToOwned; -use std::sync::{Arc, Mutex}; -use time::{self, Timespec, now}; -use unicase::UniCase; -use url::Url; -use util::thread::spawn_named; - -/// Interface for network listeners concerned with CORS checks. Proper network requests -/// should be initiated from this method, based on the response provided. -pub trait AsyncCORSResponseListener { - fn response_available(&self, response: CORSResponse); -} - -#[derive(Clone, HeapSizeOf)] -pub struct CORSRequest { - pub origin: Url, - pub destination: Url, - pub mode: RequestMode, - #[ignore_heap_size_of = "Defined in hyper"] - pub method: Method, - #[ignore_heap_size_of = "Defined in hyper"] - pub headers: Headers, - /// CORS preflight flag (https://fetch.spec.whatwg.org/#concept-http-fetch) - /// Indicates that a CORS preflight request and/or cache check is to be performed - pub preflight_flag: bool, -} - -/// https://fetch.spec.whatwg.org/#concept-request-mode -/// This only covers some of the request modes. The -/// `same-origin` and `no CORS` modes are unnecessary for XHR. -#[derive(PartialEq, Copy, Clone, HeapSizeOf)] -pub enum RequestMode { - CORS, // CORS - ForcedPreflight, // CORS-with-forced-preflight -} - -impl CORSRequest { - /// Creates a CORS request if necessary. Will return an error when fetching is forbidden - pub fn maybe_new(referer: Url, - destination: Url, - mode: RequestMode, - method: Method, - headers: Headers, - same_origin_data_url_flag: bool) - -> Result<Option<CORSRequest>, ()> { - if referer.origin() == destination.origin() { - return Ok(None); // Not cross-origin, proceed with a normal fetch - } - match destination.scheme() { - // As per (https://fetch.spec.whatwg.org/#main-fetch 5.1.9), about URLs can be fetched - // the same as a basic request. - "about" if destination.path() == "blank" => Ok(None), - // As per (https://fetch.spec.whatwg.org/#main-fetch 5.1.9), data URLs can be fetched - // the same as a basic request if the request's method is GET and the - // same-origin data-URL flag is set. - "data" if same_origin_data_url_flag && method == Method::Get => Ok(None), - "http" | "https" => { - let mut req = CORSRequest::new(referer, destination, mode, method, headers); - req.preflight_flag = !is_simple_method(&req.method) || - mode == RequestMode::ForcedPreflight; - if req.headers.iter().any(|h| !is_simple_header(&h)) { - req.preflight_flag = true; - } - Ok(Some(req)) - }, - _ => Err(()), - } - } - - fn new(mut referer: Url, - destination: Url, - mode: RequestMode, - method: Method, - headers: Headers) - -> CORSRequest { - referer.set_fragment(None); - referer.set_query(None); - referer.set_path(""); - CORSRequest { - origin: referer, - destination: destination, - mode: mode, - method: method, - headers: headers, - preflight_flag: false, - } - } - - pub fn http_fetch_async(&self, - listener: Box<AsyncCORSResponseListener + Send>, - script_chan: Box<ScriptChan + Send>) { - struct CORSContext { - listener: Box<AsyncCORSResponseListener + Send>, - response: Option<CORSResponse>, - } - - // This is shoe-horning the CORSReponse stuff into the rest of the async network - // framework right now. It would be worth redesigning http_fetch to do this properly. - impl AsyncResponseListener for CORSContext { - fn headers_available(&mut self, _metadata: Result<Metadata, NetworkError>) { - } - - fn data_available(&mut self, _payload: Vec<u8>) { - } - - fn response_complete(&mut self, _status: Result<(), NetworkError>) { - let response = self.response.take().unwrap(); - self.listener.response_available(response); - } - } - impl PreInvoke for CORSContext {} - - let context = CORSContext { - listener: listener, - response: None, - }; - let listener = NetworkListener { - context: Arc::new(Mutex::new(context)), - script_chan: script_chan, - }; - - // TODO: this exists only to make preflight check non-blocking - // perhaps should be handled by the resource thread? - let req = self.clone(); - spawn_named("cors".to_owned(), move || { - let response = req.http_fetch(); - let mut context = listener.context.lock(); - let context = context.as_mut().unwrap(); - context.response = Some(response); - listener.notify(ResponseAction::ResponseComplete(Ok(()))); - }); - } - - /// http://fetch.spec.whatwg.org/#concept-http-fetch - /// This method assumes that the CORS flag is set - /// This does not perform the full HTTP fetch, rather it handles part of the CORS filtering - /// if self.mode is ForcedPreflight, then the CORS-with-forced-preflight - /// fetch flag is set as well - pub fn http_fetch(&self) -> CORSResponse { - let response = CORSResponse::new(); - // Step 2: Handle service workers (unimplemented) - // Step 3 - // Substep 1: Service workers (unimplemented ) - // Substep 2 - let cache = &mut CORSCache(vec!()); // XXXManishearth Should come from user agent - if self.preflight_flag && - !cache.match_method(self, &self.method) && - !self.headers.iter().all(|h| is_simple_header(&h) && cache.match_header(self, h.name())) && - (!is_simple_method(&self.method) || self.mode == RequestMode::ForcedPreflight) { - return self.preflight_fetch(); - // Everything after this is part of XHR::fetch() - // Expect the organization of code to improve once we have a fetch crate - } - response - } - - /// https://fetch.spec.whatwg.org/#cors-preflight-fetch - fn preflight_fetch(&self) -> CORSResponse { - let error = CORSResponse::new_error(); - let mut cors_response = CORSResponse::new(); - - // Step 1 - let mut preflight = self.clone(); - preflight.method = Method::Options; - preflight.headers = Headers::new(); - // Step 2 - preflight.headers.set(AccessControlRequestMethod(self.method.clone())); - - // Steps 3-5 - let mut header_names = vec![]; - for header in self.headers.iter() { - header_names.push(header.name().to_owned()); - } - header_names.sort(); - preflight.headers - .set(AccessControlRequestHeaders(header_names.into_iter().map(UniCase).collect())); - - let preflight_request = Request::new(preflight.method, preflight.destination); - let mut req = match preflight_request { - Ok(req) => req, - Err(_) => return error, - }; - - let host = req.headers().get::<Host>().unwrap().clone(); - *req.headers_mut() = preflight.headers.clone(); - req.headers_mut().set(host); - let stream = match req.start() { - Ok(s) => s, - Err(_) => return error, - }; - // Step 6 - let response = match stream.send() { - Ok(r) => r, - Err(_) => return error, - }; - - // Step 7: We don't perform a CORS check here - // FYI, fn allow_cross_origin_request() performs the CORS check - match response.status.class() { - Success => {} - _ => return error, - } - cors_response.headers = response.headers.clone(); - // Substeps 1-3 (parsing rules: https://fetch.spec.whatwg.org/#http-new-header-syntax) - let methods_substep4 = [self.method.clone()]; - let mut methods = match response.headers.get() { - Some(&AccessControlAllowMethods(ref v)) => &**v, - _ => return error, - }; - let headers = match response.headers.get() { - Some(&AccessControlAllowHeaders(ref h)) => h, - _ => return error, - }; - // Substep 4 - if methods.is_empty() && preflight.mode == RequestMode::ForcedPreflight { - methods = &methods_substep4; - } - // Substep 5 - if !is_simple_method(&self.method) && !methods.iter().any(|m| m == &self.method) { - return error; - } - // Substep 6 - for h in self.headers.iter() { - if is_simple_header(&h) { - continue; - } - if !headers.iter().any(|ref h2| h.name().eq_ignore_ascii_case(h2)) { - return error; - } - } - // Substeps 7-8 - let max_age = match response.headers.get() { - Some(&AccessControlMaxAge(num)) => num, - None => 0, - }; - // Substep 9: Impose restrictions on max-age, if any (unimplemented) - // Substeps 10-12: Add a cache (partially implemented, XXXManishearth) - // This cache should come from the user agent, creating a new one here to check - // for compile time errors - let cache = &mut CORSCache(vec![]); - for m in methods { - let cache_match = cache.match_method_and_update(self, m, max_age); - if !cache_match { - cache.insert(CORSCacheEntry::new(self.origin.clone(), - self.destination.clone(), - max_age, - false, - HeaderOrMethod::MethodData(m.clone()))); - } - } - // Substeps 13-14 - for h in response.headers.iter() { - let cache_match = cache.match_header_and_update(self, h.name(), max_age); - if !cache_match { - cache.insert(CORSCacheEntry::new(self.origin.clone(), - self.destination.clone(), - max_age, - false, - HeaderOrMethod::HeaderData(h.to_string()))); - } - } - // Substep 15 - cors_response - } -} - - -pub struct CORSResponse { - pub network_error: bool, - pub headers: Headers, -} - -impl CORSResponse { - fn new() -> CORSResponse { - CORSResponse { - network_error: false, - headers: Headers::new(), - } - } - - fn new_error() -> CORSResponse { - CORSResponse { - network_error: true, - headers: Headers::new(), - } - } -} - -// CORS Cache stuff - -/// A CORS cache object. Anchor it somewhere to the user agent. -#[derive(Clone)] -pub struct CORSCache(Vec<CORSCacheEntry>); - -/// Union type for CORS cache entries -/// Each entry might pertain to a header or method -#[derive(Clone)] -pub enum HeaderOrMethod { - HeaderData(String), - MethodData(Method), -} - -impl HeaderOrMethod { - fn match_header(&self, header_name: &str) -> bool { - match *self { - HeaderOrMethod::HeaderData(ref s) => (&**s).eq_ignore_ascii_case(header_name), - _ => false, - } - } - - fn match_method(&self, method: &Method) -> bool { - match *self { - HeaderOrMethod::MethodData(ref m) => m == method, - _ => false, - } - } -} - -// An entry in the CORS cache -#[derive(Clone)] -pub struct CORSCacheEntry { - pub origin: Url, - pub url: Url, - pub max_age: u32, - pub credentials: bool, - pub header_or_method: HeaderOrMethod, - created: Timespec, -} - -impl CORSCacheEntry { - fn new(origin: Url, - url: Url, - max_age: u32, - credentials: bool, - header_or_method: HeaderOrMethod) - -> CORSCacheEntry { - CORSCacheEntry { - origin: origin, - url: url, - max_age: max_age, - credentials: credentials, - header_or_method: header_or_method, - created: time::now().to_timespec(), - } - } -} - -impl CORSCache { - /// https://fetch.spec.whatwg.org/#concept-cache-clear - #[allow(dead_code)] - fn clear(&mut self, request: &CORSRequest) { - let CORSCache(buf) = self.clone(); - let new_buf: Vec<CORSCacheEntry> = - buf.into_iter() - .filter(|e| e.origin == request.origin && request.destination == e.url) - .collect(); - *self = CORSCache(new_buf); - } - - // Remove old entries - fn cleanup(&mut self) { - let CORSCache(buf) = self.clone(); - let now = time::now().to_timespec(); - let new_buf: Vec<CORSCacheEntry> = buf.into_iter() - .filter(|e| now.sec > e.created.sec + e.max_age as i64) - .collect(); - *self = CORSCache(new_buf); - } - - /// https://fetch.spec.whatwg.org/#concept-cache-match-header - fn find_entry_by_header<'a>(&'a mut self, - request: &CORSRequest, - header_name: &str) - -> Option<&'a mut CORSCacheEntry> { - self.cleanup(); - // Credentials are not yet implemented here - self.0.iter_mut().find(|e| { - e.origin.scheme() == request.origin.scheme() && - e.origin.host_str() == request.origin.host_str() && - e.origin.port() == request.origin.port() && - e.url == request.destination && - e.header_or_method.match_header(header_name) - }) - } - - fn match_header(&mut self, request: &CORSRequest, header_name: &str) -> bool { - self.find_entry_by_header(request, header_name).is_some() - } - - fn match_header_and_update(&mut self, - request: &CORSRequest, - header_name: &str, - new_max_age: u32) - -> bool { - self.find_entry_by_header(request, header_name).map(|e| e.max_age = new_max_age).is_some() - } - - fn find_entry_by_method<'a>(&'a mut self, - request: &CORSRequest, - method: &Method) - -> Option<&'a mut CORSCacheEntry> { - // we can take the method from CORSRequest itself - self.cleanup(); - // Credentials are not yet implemented here - self.0.iter_mut().find(|e| { - e.origin.scheme() == request.origin.scheme() && - e.origin.host_str() == request.origin.host_str() && - e.origin.port() == request.origin.port() && - e.url == request.destination && - e.header_or_method.match_method(method) - }) - } - - /// https://fetch.spec.whatwg.org/#concept-cache-match-method - fn match_method(&mut self, request: &CORSRequest, method: &Method) -> bool { - self.find_entry_by_method(request, method).is_some() - } - - fn match_method_and_update(&mut self, - request: &CORSRequest, - method: &Method, - new_max_age: u32) - -> bool { - self.find_entry_by_method(request, method).map(|e| e.max_age = new_max_age).is_some() - } - - fn insert(&mut self, entry: CORSCacheEntry) { - self.cleanup(); - self.0.push(entry); - } -} - -fn is_simple_header(h: &HeaderView) -> bool { - // FIXME: use h.is::<HeaderType>() when AcceptLanguage and - // ContentLanguage headers exist - match &*h.name().to_ascii_lowercase() { - "accept" | "accept-language" | "content-language" => true, - "content-type" => match h.value() { - Some(&ContentType(Mime(TopLevel::Text, SubLevel::Plain, _))) | - Some(&ContentType(Mime(TopLevel::Application, SubLevel::WwwFormUrlEncoded, _))) | - Some(&ContentType(Mime(TopLevel::Multipart, SubLevel::FormData, _))) => true, - - _ => false, - - }, - _ => false, - } -} - -fn is_simple_method(m: &Method) -> bool { - match *m { - Method::Get | Method::Head | Method::Post => true, - _ => false, - } -} - -/// Perform a CORS check on a header list and CORS request -/// https://fetch.spec.whatwg.org/#cors-check -pub fn allow_cross_origin_request(req: &CORSRequest, headers: &Headers) -> bool { - match headers.get::<AccessControlAllowOrigin>() { - Some(&AccessControlAllowOrigin::Any) => true, // Not always true, depends on credentials mode - Some(&AccessControlAllowOrigin::Value(ref url)) => req.origin.as_str() == *url, - Some(&AccessControlAllowOrigin::Null) | - None => false, - } -} diff --git a/components/script/dom/htmllinkelement.rs b/components/script/dom/htmllinkelement.rs index 2b0056a8d65..79b6ed32f0b 100644 --- a/components/script/dom/htmllinkelement.rs +++ b/components/script/dom/htmllinkelement.rs @@ -227,7 +227,7 @@ impl HTMLLinkElement { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); if self.parser_inserted.get() { diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 7256ae1b131..70563ec95c7 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -484,7 +484,7 @@ impl HTMLMediaElement { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); // FIXME: we're supposed to block the load event much earlier than now diff --git a/components/script/dom/htmlscriptelement.rs b/components/script/dom/htmlscriptelement.rs index 1f9f49516dd..f280f670e4b 100644 --- a/components/script/dom/htmlscriptelement.rs +++ b/components/script/dom/htmlscriptelement.rs @@ -314,7 +314,7 @@ impl HTMLScriptElement { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); doc.load_async(LoadType::Script(url), response_target); diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 318d3bbc1ec..785a5928c9f 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -2,8 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use cors::CORSResponse; -use cors::{AsyncCORSResponseListener, CORSRequest, RequestMode, allow_cross_origin_request}; use document_loader::DocumentLoader; use dom::bindings::cell::DOMRefCell; use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods; @@ -35,27 +33,28 @@ use encoding::label::encoding_from_whatwg_label; use encoding::types::{DecoderTrap, EncoderTrap, Encoding, EncodingRef}; use euclid::length::Length; use hyper::header::Headers; -use hyper::header::{Accept, ContentLength, ContentType, qitem}; +use hyper::header::{ContentLength, ContentType}; use hyper::http::RawStatus; use hyper::method::Method; -use hyper::mime::{self, Mime}; +use hyper::mime::{self, Mime, Attr as MimeAttr, Value as MimeValue}; use ipc_channel::ipc; use ipc_channel::router::ROUTER; use js::jsapi::JS_ClearPendingException; use js::jsapi::{JSContext, JS_ParseJSON, RootedValue}; use js::jsval::{JSVal, NullValue, UndefinedValue}; use msg::constellation_msg::{PipelineId, ReferrerPolicy}; -use net_traits::CoreResourceMsg::Load; +use net_traits::CoreResourceMsg::Fetch; +use net_traits::request::{CredentialsMode, Destination, RequestInit, RequestMode}; use net_traits::trim_http_whitespace; -use net_traits::{AsyncResponseListener, AsyncResponseTarget, Metadata, NetworkError, RequestSource}; -use net_traits::{LoadConsumer, LoadContext, LoadData, ResourceCORSData, CoreResourceThread, LoadOrigin}; +use net_traits::{CoreResourceThread, LoadOrigin}; +use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource}; use network_listener::{NetworkListener, PreInvoke}; use parse::html::{ParseContext, parse_html}; use parse::xml::{self, parse_xml}; use script_runtime::ScriptChan; use std::ascii::AsciiExt; use std::borrow::ToOwned; -use std::cell::{Cell, RefCell}; +use std::cell::Cell; use std::default::Default; use std::str; use std::sync::{Arc, Mutex}; @@ -82,7 +81,6 @@ pub struct GenerationId(u32); struct XHRContext { xhr: TrustedXHRAddress, gen_id: GenerationId, - cors_request: Option<CORSRequest>, buf: DOMRefCell<Vec<u8>>, sync_status: DOMRefCell<Option<ErrorResult>>, } @@ -142,7 +140,6 @@ pub struct XMLHttpRequest { request_body_len: Cell<usize>, sync: Cell<bool>, upload_complete: Cell<bool>, - upload_events: Cell<bool>, send_flag: Cell<bool>, timeout_cancel: DOMRefCell<Option<OneshotTimerHandle>>, @@ -187,7 +184,6 @@ impl XMLHttpRequest { request_body_len: Cell::new(0), sync: Cell::new(false), upload_complete: Cell::new(false), - upload_events: Cell::new(false), send_flag: Cell::new(false), timeout_cancel: DOMRefCell::new(None), @@ -216,75 +212,40 @@ impl XMLHttpRequest { } } - fn check_cors(context: Arc<Mutex<XHRContext>>, - load_data: LoadData, - req: CORSRequest, - script_chan: Box<ScriptChan + Send>, - core_resource_thread: CoreResourceThread) { - struct CORSContext { - xhr: Arc<Mutex<XHRContext>>, - load_data: RefCell<Option<LoadData>>, - req: CORSRequest, - script_chan: Box<ScriptChan + Send>, - core_resource_thread: CoreResourceThread, - } - - impl AsyncCORSResponseListener for CORSContext { - fn response_available(&self, response: CORSResponse) { - if response.network_error { - let mut context = self.xhr.lock().unwrap(); - let xhr = context.xhr.root(); - xhr.process_partial_response(XHRProgress::Errored(context.gen_id, Error::Network)); - *context.sync_status.borrow_mut() = Some(Err(Error::Network)); - return; - } - - let mut load_data = self.load_data.borrow_mut().take().unwrap(); - load_data.cors = Some(ResourceCORSData { - preflight: self.req.preflight_flag, - origin: self.req.origin.clone() - }); - - XMLHttpRequest::initiate_async_xhr(self.xhr.clone(), self.script_chan.clone(), - self.core_resource_thread.clone(), load_data); - } - } - - let cors_context = CORSContext { - xhr: context, - load_data: RefCell::new(Some(load_data)), - req: req.clone(), - script_chan: script_chan.clone(), - core_resource_thread: core_resource_thread, - }; - - req.http_fetch_async(box cors_context, script_chan); - } - fn initiate_async_xhr(context: Arc<Mutex<XHRContext>>, script_chan: Box<ScriptChan + Send>, core_resource_thread: CoreResourceThread, - load_data: LoadData) { - impl AsyncResponseListener for XHRContext { - fn headers_available(&mut self, metadata: Result<Metadata, NetworkError>) { - let xhr = self.xhr.root(); - let rv = xhr.process_headers_available(self.cors_request.clone(), - self.gen_id, - metadata); - if rv.is_err() { + init: RequestInit) { + impl FetchResponseListener for XHRContext { + fn process_request_body(&mut self) { + // todo + } + fn process_request_eof(&mut self) { + // todo + } + fn process_response(&mut self, metadata: Result<Metadata, NetworkError>) { + let xhr = self.xhr.root(); + let rv = xhr.process_headers_available(self.gen_id, + metadata); + if rv.is_err() { + *self.sync_status.borrow_mut() = Some(rv); + } + } + fn process_response_chunk(&mut self, mut chunk: Vec<u8>) { + self.buf.borrow_mut().append(&mut chunk); + self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); + } + fn process_response_eof(&mut self, response: Result<(), NetworkError>) { + let rv = match response { + Ok(()) => { + self.xhr.root().process_response_complete(self.gen_id, Ok(())) + } + Err(e) => { + self.xhr.root().process_response_complete(self.gen_id, Err(e)) + } + }; *self.sync_status.borrow_mut() = Some(rv); } - } - - fn data_available(&mut self, payload: Vec<u8>) { - self.buf.borrow_mut().extend_from_slice(&payload); - self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); - } - - fn response_complete(&mut self, status: Result<(), NetworkError>) { - let rv = self.xhr.root().process_response_complete(self.gen_id, status); - *self.sync_status.borrow_mut() = Some(rv); - } } impl PreInvoke for XHRContext { @@ -298,13 +259,10 @@ impl XMLHttpRequest { context: context, script_chan: script_chan, }; - let response_target = AsyncResponseTarget { - sender: action_sender, - }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_fetch(message.to().unwrap()); }); - core_resource_thread.send(Load(load_data, LoadConsumer::Listener(response_target), None)).unwrap(); + core_resource_thread.send(Fetch(init, action_sender)).unwrap(); } } @@ -563,12 +521,15 @@ impl XMLHttpRequestMethods for XMLHttpRequest { Method::Get | Method::Head => None, _ => data }; - // Step 4 + // Step 4 (first half) let extracted = data.as_ref().map(|d| d.extract()); + self.request_body_len.set(extracted.as_ref().map_or(0, |e| e.0.len())); + // todo preserved headers? + // Step 6 - self.upload_events.set(false); + self.upload_complete.set(false); // Step 7 self.upload_complete.set(match extracted { None => true, @@ -580,11 +541,6 @@ impl XMLHttpRequestMethods for XMLHttpRequest { // Step 9 if !self.sync.get() { - let event_target = self.upload.upcast::<EventTarget>(); - if event_target.has_handlers() { - self.upload_events.set(true); - } - // If one of the event handlers below aborts the fetch by calling // abort or open we will need the current generation id to detect it. // Substep 1 @@ -604,67 +560,108 @@ impl XMLHttpRequestMethods for XMLHttpRequest { } // Step 5 - let global = self.global(); + //TODO - set referrer_policy/referrer_url in request + let has_handlers = self.upload.upcast::<EventTarget>().has_handlers(); + let credentials_mode = if self.with_credentials.get() { + CredentialsMode::Include + } else { + CredentialsMode::CredentialsSameOrigin + }; + let use_url_credentials = if let Some(ref url) = *self.request_url.borrow() { + !url.username().is_empty() || url.password().is_some() + } else { + unreachable!() + }; - let mut load_data = - LoadData::new(LoadContext::Browsing, - self.request_url.borrow().clone().unwrap(), - self); + let bypass_cross_origin_check = { + // We want to be able to do cross-origin requests in browser.html. + // If the XHR happens in a top level window and the mozbrowser + // preference is enabled, we allow bypassing the CORS check. + // This is a temporary measure until we figure out Servo privilege + // story. See https://github.com/servo/servo/issues/9582 + if let GlobalRoot::Window(win) = self.global() { + let is_root_pipeline = win.parent_info().is_none(); + let is_mozbrowser_enabled = mozbrowser_enabled(); + is_root_pipeline && is_mozbrowser_enabled + } else { + false + } + }; - if load_data.url.origin().ne(&global.r().get_url().origin()) { - load_data.credentials_flag = self.WithCredentials(); - } - load_data.data = extracted.as_ref().map(|e| e.0.clone()); + let mut request = RequestInit { + method: self.request_method.borrow().clone(), + url: self.request_url.borrow().clone().unwrap(), + headers: (*self.request_headers.borrow()).clone(), + unsafe_request: true, + same_origin_data: true, + // XXXManishearth figure out how to avoid this clone + body: extracted.as_ref().map(|e| e.0.clone()), + // XXXManishearth actually "subresource", but it doesn't exist + // https://github.com/whatwg/xhr/issues/71 + destination: Destination::None, + synchronous: self.sync.get(), + mode: RequestMode::CORSMode, + use_cors_preflight: has_handlers, + credentials_mode: credentials_mode, + use_url_credentials: use_url_credentials, + origin: self.global().r().get_url(), + referer_url: self.referrer_url.clone(), + referrer_policy: self.referrer_policy.clone(), + }; - // XHR spec differs from http, and says UTF-8 should be in capitals, - // instead of "utf-8", which is what Hyper defaults to. So not - // using content types provided by Hyper. - let n = "content-type"; - match extracted { - Some((_, Some(ref content_type))) => - load_data.headers.set_raw(n.to_owned(), vec![content_type.bytes().collect()]), - _ => (), + if bypass_cross_origin_check { + request.mode = RequestMode::Navigate; } - load_data.preserved_headers = (*self.request_headers.borrow()).clone(); + // step 4 (second half) + match extracted { + Some((_, ref content_type)) => { + // this should handle Document bodies too, not just BodyInit + let encoding = if let Some(BodyInit::String(_)) = data { + // XHR spec differs from http, and says UTF-8 should be in capitals, + // instead of "utf-8", which is what Hyper defaults to. So not + // using content types provided by Hyper. + Some(MimeValue::Ext("UTF-8".to_string())) + } else { + None + }; - if !load_data.preserved_headers.has::<Accept>() { - let mime = Mime(mime::TopLevel::Star, mime::SubLevel::Star, vec![]); - load_data.preserved_headers.set(Accept(vec![qitem(mime)])); - } + let mut content_type_set = false; + if let Some(ref ct) = *content_type { + if !request.headers.has::<ContentType>() { + request.headers.set_raw("content-type", vec![ct.bytes().collect()]); + content_type_set = true; + } + } - load_data.method = (*self.request_method.borrow()).clone(); + if !content_type_set { + let ct = request.headers.get::<ContentType>().map(|x| x.clone()); + if let Some(mut ct) = ct { + if let Some(encoding) = encoding { + for param in &mut (ct.0).2 { + if param.0 == MimeAttr::Charset { + if !param.0.as_str().eq_ignore_ascii_case(encoding.as_str()) { + *param = (MimeAttr::Charset, encoding.clone()); + } + } + } + } + // remove instead of mutate in place + // https://github.com/hyperium/hyper/issues/821 + request.headers.remove_raw("content-type"); + request.headers.set(ct); + } + } - // CORS stuff - let global = self.global(); - let referer_url = self.global().r().get_url(); - let mode = if self.upload_events.get() { - RequestMode::ForcedPreflight - } else { - RequestMode::CORS - }; - let mut combined_headers = load_data.headers.clone(); - combined_headers.extend(load_data.preserved_headers.iter()); - let cors_request = CORSRequest::maybe_new(referer_url.clone(), - load_data.url.clone(), - mode, - load_data.method.clone(), - combined_headers, - true); - match cors_request { - Ok(None) => { - let bytes = referer_url[..Position::AfterPath].as_bytes().to_vec(); - self.request_headers.borrow_mut().set_raw("Referer".to_owned(), vec![bytes]); - }, - Ok(Some(ref req)) => self.insert_trusted_header("origin".to_owned(), - req.origin.to_string()), - _ => {} + } + _ => (), } - debug!("request_headers = {:?}", *self.request_headers.borrow()); + debug!("request.headers = {:?}", request.headers); self.fetch_time.set(time::now().to_timespec().sec); - let rv = self.fetch(load_data, cors_request, global.r()); + + let rv = self.fetch(request, self.global().r()); // Step 10 if self.sync.get() { return rv; @@ -879,7 +876,7 @@ impl XMLHttpRequest { event.fire(self.upcast()); } - fn process_headers_available(&self, cors_request: Option<CORSRequest>, + fn process_headers_available(&self, gen_id: GenerationId, metadata: Result<Metadata, NetworkError>) -> Result<(), Error> { let metadata = match metadata { @@ -890,35 +887,6 @@ impl XMLHttpRequest { }, }; - let bypass_cross_origin_check = { - // We want to be able to do cross-origin requests in browser.html. - // If the XHR happens in a top level window and the mozbrowser - // preference is enabled, we allow bypassing the CORS check. - // This is a temporary measure until we figure out Servo privilege - // story. See https://github.com/servo/servo/issues/9582 - if let GlobalRoot::Window(win) = self.global() { - let is_root_pipeline = win.parent_info().is_none(); - let is_mozbrowser_enabled = mozbrowser_enabled(); - is_root_pipeline && is_mozbrowser_enabled - } else { - false - } - }; - - if !bypass_cross_origin_check { - if let Some(ref req) = cors_request { - match metadata.headers { - Some(ref h) if allow_cross_origin_request(req, h) => {}, - _ => { - self.process_partial_response(XHRProgress::Errored(gen_id, Error::Network)); - return Err(Error::Network); - } - } - } - } else { - debug!("Bypassing cross origin check"); - } - *self.response_url.borrow_mut() = metadata.final_url[..Position::AfterQuery].to_owned(); // XXXManishearth Clear cache entries in case of a network error @@ -1318,25 +1286,12 @@ impl XMLHttpRequest { } fn fetch(&self, - load_data: LoadData, - cors_request: Result<Option<CORSRequest>, ()>, + init: RequestInit, global: GlobalRef) -> ErrorResult { - let cors_request = match cors_request { - Err(_) => { - // Happens in case of unsupported cross-origin URI schemes. - // Supported schemes are http, https, data, and about. - self.process_partial_response(XHRProgress::Errored( - self.generation_id.get(), Error::Network)); - return Err(Error::Network); - } - Ok(req) => req, - }; - let xhr = Trusted::new(self); let context = Arc::new(Mutex::new(XHRContext { xhr: xhr, - cors_request: cors_request.clone(), gen_id: self.generation_id.get(), buf: DOMRefCell::new(vec!()), sync_status: DOMRefCell::new(None), @@ -1350,13 +1305,8 @@ impl XMLHttpRequest { }; let core_resource_thread = global.core_resource_thread(); - if let Some(req) = cors_request { - XMLHttpRequest::check_cors(context.clone(), load_data, req.clone(), - script_chan.clone(), core_resource_thread); - } else { - XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan, - core_resource_thread, load_data); - } + XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan, + core_resource_thread, init); if let Some(script_port) = script_port { loop { diff --git a/components/script/lib.rs b/components/script/lib.rs index 71e53141d9a..2f76a112c0a 100644 --- a/components/script/lib.rs +++ b/components/script/lib.rs @@ -79,7 +79,6 @@ extern crate style; extern crate time; #[cfg(any(target_os = "macos", target_os = "linux"))] extern crate tinyfiledialogs; -extern crate unicase; extern crate url; #[macro_use] extern crate util; @@ -91,7 +90,6 @@ extern crate xml5ever; mod blob_url_store; pub mod bluetooth_blacklist; pub mod clipboard_provider; -pub mod cors; mod devtools; pub mod document_loader; #[macro_use] diff --git a/components/script/network_listener.rs b/components/script/network_listener.rs index cff0723d277..13aa26c25e4 100644 --- a/components/script/network_listener.rs +++ b/components/script/network_listener.rs @@ -2,7 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use net_traits::{AsyncResponseListener, ResponseAction}; +use net_traits::{Action, AsyncResponseListener, FetchResponseListener}; +use net_traits::{FetchResponseMsg, ResponseAction}; use script_runtime::ScriptThreadEventCategory::NetworkEvent; use script_runtime::{CommonScriptMsg, ScriptChan}; use script_thread::Runnable; @@ -10,13 +11,13 @@ use std::sync::{Arc, Mutex}; /// An off-thread sink for async network event runnables. All such events are forwarded to /// a target thread, where they are invoked on the provided context object. -pub struct NetworkListener<T: AsyncResponseListener + PreInvoke + Send + 'static> { - pub context: Arc<Mutex<T>>, +pub struct NetworkListener<Listener: PreInvoke + Send + 'static> { + pub context: Arc<Mutex<Listener>>, pub script_chan: Box<ScriptChan + Send>, } -impl<T: AsyncResponseListener + PreInvoke + Send + 'static> NetworkListener<T> { - pub fn notify(&self, action: ResponseAction) { +impl<Listener: PreInvoke + Send + 'static> NetworkListener<Listener> { + pub fn notify<A: Action<Listener> + Send + 'static>(&self, action: A) { if let Err(err) = self.script_chan.send(CommonScriptMsg::RunnableMsg(NetworkEvent, box ListenerRunnable { context: self.context.clone(), action: action, @@ -26,6 +27,20 @@ impl<T: AsyncResponseListener + PreInvoke + Send + 'static> NetworkListener<T> { } } +// helps type inference +impl<Listener: AsyncResponseListener + PreInvoke + Send + 'static> NetworkListener<Listener> { + pub fn notify_action(&self, action: ResponseAction) { + self.notify(action); + } +} + +// helps type inference +impl<Listener: FetchResponseListener + PreInvoke + Send + 'static> NetworkListener<Listener> { + pub fn notify_fetch(&self, action: FetchResponseMsg) { + self.notify(action); + } +} + /// A gating mechanism that runs before invoking the runnable on the target thread. /// If the `should_invoke` method returns false, the runnable is discarded without /// being invoked. @@ -36,13 +51,13 @@ pub trait PreInvoke { } /// A runnable for moving the async network events between threads. -struct ListenerRunnable<T: AsyncResponseListener + PreInvoke + Send> { - context: Arc<Mutex<T>>, - action: ResponseAction, +struct ListenerRunnable<A: Action<Listener> + Send + 'static, Listener: PreInvoke + Send> { + context: Arc<Mutex<Listener>>, + action: A, } -impl<T: AsyncResponseListener + PreInvoke + Send> Runnable for ListenerRunnable<T> { - fn handler(self: Box<ListenerRunnable<T>>) { +impl<A: Action<Listener> + Send + 'static, Listener: PreInvoke + Send> Runnable for ListenerRunnable<A, Listener> { + fn handler(self: Box<ListenerRunnable<A, Listener>>) { let this = *self; let mut context = this.context.lock().unwrap(); if context.should_invoke() { diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 7684eb9e336..8cf97710bc7 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -1962,7 +1962,7 @@ impl ScriptThread { script_chan: self.chan.clone(), }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); let response_target = AsyncResponseTarget { sender: action_sender, diff --git a/components/servo/Cargo.lock b/components/servo/Cargo.lock index 00399f7b82f..91fa8562c38 100644 --- a/components/servo/Cargo.lock +++ b/components/servo/Cargo.lock @@ -1916,7 +1916,6 @@ dependencies = [ "style 0.0.1", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "tinyfiledialogs 0.1.0 (git+https://github.com/jdm/tinyfiledialogs)", - "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "util 0.0.1", "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ports/cef/Cargo.lock b/ports/cef/Cargo.lock index 04dfbcb9acd..d2e0cdc5ac1 100644 --- a/ports/cef/Cargo.lock +++ b/ports/cef/Cargo.lock @@ -1774,7 +1774,6 @@ dependencies = [ "style 0.0.1", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "tinyfiledialogs 0.1.0 (git+https://github.com/jdm/tinyfiledialogs)", - "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "util 0.0.1", "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/tests/unit/net/fetch.rs b/tests/unit/net/fetch.rs index 4803eee5178..3514f69d6e0 100644 --- a/tests/unit/net/fetch.rs +++ b/tests/unit/net/fetch.rs @@ -14,8 +14,9 @@ use hyper::server::{Request as HyperRequest, Response as HyperResponse}; use hyper::status::StatusCode; use hyper::uri::RequestUri; use net::fetch::cors_cache::CORSCache; -use net::fetch::methods::{fetch, fetch_async, fetch_with_cors_cache}; -use net_traits::AsyncFetchListener; +use net::fetch::methods::{FetchContext, fetch, fetch_with_cors_cache}; +use net::http_loader::HttpState; +use net_traits::FetchTaskTarget; use net_traits::request::{Origin, RedirectMode, Referer, Request, RequestMode}; use net_traits::response::{CacheState, Response, ResponseBody, ResponseType}; use std::fs::File; @@ -24,6 +25,7 @@ use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Sender, channel}; use std::sync::{Arc, Mutex}; +use std::thread; use time::{self, Duration}; use unicase::UniCase; use url::{Origin as UrlOrigin, Url}; @@ -35,11 +37,32 @@ struct FetchResponseCollector { sender: Sender<Response>, } -impl AsyncFetchListener for FetchResponseCollector { - fn response_available(&self, response: Response) { - let _ = self.sender.send(response); +fn new_fetch_context() -> FetchContext { + FetchContext { + state: HttpState::new(), + user_agent: "Such Browser. Very Layout. Wow.".into(), } } +impl FetchTaskTarget for FetchResponseCollector { + fn process_request_body(&mut self, _: &Request) {} + fn process_request_eof(&mut self, _: &Request) {} + fn process_response(&mut self, _: &Response) {} + fn process_response_chunk(&mut self, _: Vec<u8>) {} + /// Fired when the response is fully fetched + fn process_response_eof(&mut self, response: &Response) { + self.sender.send(response.clone()); + } +} + +fn fetch_async(request: Request, target: Box<FetchTaskTarget + Send>) { + thread::spawn(move || { + fetch(Rc::new(request), &mut Some(target), new_fetch_context()); + }); +} + +fn fetch_sync(request: Request) -> Response { + fetch(Rc::new(request), &mut None, new_fetch_context()) +} fn make_server<H: Handler + 'static>(handler: H) -> (Listening, Url) { // this is a Listening server because of handle_threads() @@ -61,10 +84,8 @@ fn test_fetch_response_is_not_network_error() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + *request.referer.borrow_mut() = Referer::NoReferer; + let fetch_response = fetch_sync(request); let _ = server.close(); if fetch_response.is_network_error() { @@ -82,10 +103,8 @@ fn test_fetch_response_body_matches_const_message() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + *request.referer.borrow_mut() = Referer::NoReferer; + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -104,10 +123,8 @@ fn test_fetch_aboutblank() { let url = Url::parse("about:blank").unwrap(); let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + *request.referer.borrow_mut() = Referer::NoReferer; + let fetch_response = fetch_sync(request); assert!(!fetch_response.is_network_error()); assert!(*fetch_response.body.lock().unwrap() == ResponseBody::Done(vec![])); } @@ -119,7 +136,7 @@ fn test_fetch_data() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); let expected_resp_body = "<p>Servo</p>".to_owned(); - let fetch_response = fetch(Rc::new(request)); + let fetch_response = fetch_sync(request); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); @@ -148,7 +165,7 @@ fn test_fetch_file() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); - let fetch_response = fetch(Rc::new(request)); + let fetch_response = fetch_sync(request); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); let content_type: &ContentType = fetch_response.headers.get().unwrap(); @@ -187,12 +204,10 @@ fn test_cors_preflight_fetch() { let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.use_cors_preflight = true; request.mode = RequestMode::CORSMode; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -226,14 +241,14 @@ fn test_cors_preflight_cache_fetch() { let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url.clone(), Some(origin.clone()), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.use_cors_preflight = true; request.mode = RequestMode::CORSMode; let wrapped_request0 = Rc::new(request.clone()); let wrapped_request1 = Rc::new(request); - let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache); - let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache); + let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, &mut None, new_fetch_context()); + let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response0.is_network_error() && !fetch_response1.is_network_error()); @@ -276,12 +291,10 @@ fn test_cors_preflight_fetch_network_error() { let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); *request.method.borrow_mut() = Method::Extension("CHICKEN".to_owned()); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.use_cors_preflight = true; request.mode = RequestMode::CORSMode; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(fetch_response.is_network_error()); @@ -301,10 +314,8 @@ fn test_fetch_response_is_basic_filtered() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + *request.referer.borrow_mut() = Referer::NoReferer; + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -348,11 +359,9 @@ fn test_fetch_response_is_cors_filtered() { // an origin mis-match will stop it from defaulting to a basic filtered response let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.mode = RequestMode::CORSMode; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -382,10 +391,8 @@ fn test_fetch_response_is_opaque_filtered() { // an origin mis-match will fall through to an Opaque filtered response let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + *request.referer.borrow_mut() = Referer::NoReferer; + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -431,11 +438,9 @@ fn test_fetch_response_is_opaque_redirect_filtered() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.redirect_mode.set(RedirectMode::Manual); - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch_sync(request); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -467,13 +472,12 @@ fn test_fetch_with_local_urls_only() { let do_fetch = |url: Url| { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; // Set the flag. request.local_urls_only = true; - let wrapped_request = Rc::new(request); - fetch(wrapped_request) + fetch_sync(request) }; let local_url = Url::parse("about:blank").unwrap(); @@ -509,10 +513,8 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; - let wrapped_request = Rc::new(request); - - let fetch_response = fetch(wrapped_request); + *request.referer.borrow_mut() = Referer::NoReferer; + let fetch_response = fetch_sync(request); let _ = server.close(); fetch_response } @@ -594,11 +596,10 @@ fn test_fetch_redirect_updates_method_runner(tx: Sender<bool>, status_code: Stat let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; *request.method.borrow_mut() = method; - let wrapped_request = Rc::new(request); - let _ = fetch(wrapped_request); + let _ = fetch_sync(request); let _ = server.close(); } @@ -670,7 +671,7 @@ fn test_fetch_async_returns_complete_response() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let (tx, rx) = channel(); let listener = Box::new(FetchResponseCollector { @@ -695,7 +696,7 @@ fn test_opaque_filtered_fetch_async_returns_complete_response() { // an origin mis-match will fall through to an Opaque filtered response let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let (tx, rx) = channel(); let listener = Box::new(FetchResponseCollector { @@ -735,7 +736,7 @@ fn test_opaque_redirect_filtered_fetch_async_returns_complete_response() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.redirect_mode.set(RedirectMode::Manual); let (tx, rx) = channel(); diff --git a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini deleted file mode 100644 index 61fdcbeb2f1..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini +++ /dev/null @@ -1,5 +0,0 @@ -[send-authentication-basic-setrequestheader-existing-session.htm] - type: testharness - [XMLHttpRequest: send() - "Basic" authenticated request using setRequestHeader() when there is an existing session] - expected: FAIL - diff --git a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini deleted file mode 100644 index 4aa67f4e1f8..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini +++ /dev/null @@ -1,5 +0,0 @@ -[send-authentication-basic.htm] - type: testharness - [XMLHttpRequest: send() - "Basic" authenticated requests with user name and password passed to open()] - expected: FAIL - diff --git a/tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini new file mode 100644 index 00000000000..98f1333fabb --- /dev/null +++ b/tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini @@ -0,0 +1,8 @@ +[send-conditional.htm] + type: testharness + [XMLHttpRequest: send() - conditional requests (tag)] + expected: FAIL + + [XMLHttpRequest: send() - conditional requests (date)] + expected: FAIL + diff --git a/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini index b3144791ecf..9c6e15cc513 100644 --- a/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini +++ b/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini @@ -15,9 +15,12 @@ [charset given but wrong, fix it (known MIME, bogus charset)] expected: FAIL - [charset given but wrong, fix it (known MIME, actual charset)] + [If multiple charset parameters are given, all should be rewritten] expected: FAIL - [If multiple charset parameters are given, all should be rewritten] + [Correct text/plain MIME with charset] + expected: FAIL + + [charset given but wrong, fix it (known MIME, actual charset)] expected: FAIL diff --git a/tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini deleted file mode 100644 index c1ab904b4bb..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini +++ /dev/null @@ -1,5 +0,0 @@ -[send-entity-body-get-head.htm] - type: testharness - [XMLHttpRequest: send() - non-empty data argument and GET/HEAD (HEAD)] - expected: FAIL - diff --git a/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini new file mode 100644 index 00000000000..8a2f5593ca5 --- /dev/null +++ b/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini @@ -0,0 +1,14 @@ +[send-redirect-to-cors.htm] + type: testharness + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (301)] + expected: FAIL + + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (302)] + expected: FAIL + + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (303)] + expected: FAIL + + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (307)] + expected: FAIL + diff --git a/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini deleted file mode 100644 index eabe809eb86..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini +++ /dev/null @@ -1,14 +0,0 @@ -[send-redirect-to-non-cors.htm] - type: testharness - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (301)] - expected: FAIL - - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (302)] - expected: FAIL - - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (303)] - expected: FAIL - - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (307)] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/allow-headers.htm.ini b/tests/wpt/metadata/cors/allow-headers.htm.ini index 07bd1905cc5..c46e502e53c 100644 --- a/tests/wpt/metadata/cors/allow-headers.htm.ini +++ b/tests/wpt/metadata/cors/allow-headers.htm.ini @@ -33,6 +33,3 @@ [Allow origin: [tab\]http://web-platform.test:8000] expected: FAIL - [Disallow origin: http://web-platform.test:8000/] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/origin.htm.ini b/tests/wpt/metadata/cors/origin.htm.ini index 84071a32cfa..6b627bf7b8c 100644 --- a/tests/wpt/metadata/cors/origin.htm.ini +++ b/tests/wpt/metadata/cors/origin.htm.ini @@ -33,12 +33,3 @@ [Allow origin: [tab\]http://web-platform.test:8000] expected: FAIL - [Disallow origin: http://web-platform.test:8000/] - expected: FAIL - - [Disallow multiple headers (, *)] - expected: FAIL - - [Disallow multiple headers (*, )] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/preflight-cache.htm.ini b/tests/wpt/metadata/cors/preflight-cache.htm.ini index c61b6a45bdc..951b025fc02 100644 --- a/tests/wpt/metadata/cors/preflight-cache.htm.ini +++ b/tests/wpt/metadata/cors/preflight-cache.htm.ini @@ -1,17 +1,8 @@ [preflight-cache.htm] type: testharness - [Test preflight] - expected: FAIL - [preflight for x-print should be cached] expected: FAIL - [age = 0, should not be cached] - expected: FAIL - - [age = -1, should not be cached] - expected: FAIL - [preflight first request, second from cache, wait, third should preflight again] expected: FAIL diff --git a/tests/wpt/metadata/cors/redirect-origin.htm.ini b/tests/wpt/metadata/cors/redirect-origin.htm.ini index 10780608fa9..ef3413cca5a 100644 --- a/tests/wpt/metadata/cors/redirect-origin.htm.ini +++ b/tests/wpt/metadata/cors/redirect-origin.htm.ini @@ -62,24 +62,12 @@ [local (*) to remote (http://web-platform.test:8000), expect origin=http://web-platform.test:8000] expected: FAIL - [local (*) to remote (null), expect to fail] - expected: FAIL - - [local (*) to remote (none), expect to fail] - expected: FAIL - [local (http://web-platform.test:8000) to remote (*), expect origin=http://web-platform.test:8000] expected: FAIL [local (http://web-platform.test:8000) to remote (http://web-platform.test:8000), expect origin=http://web-platform.test:8000] expected: FAIL - [local (http://web-platform.test:8000) to remote (null), expect to fail] - expected: FAIL - - [local (http://web-platform.test:8000) to remote (none), expect to fail] - expected: FAIL - [local (null) to remote (*), expect origin=http://web-platform.test:8000] expected: FAIL @@ -110,27 +98,3 @@ [remote (http://web-platform.test:8000) to remote2 (null), expect origin=null] expected: FAIL - [remote (http://www1.web-platform.test:8000) to remote (*), expect to fail] - expected: FAIL - - [remote (null) to remote2 (*), expect to fail] - expected: FAIL - - [remote (none) to remote2 (*), expect to fail] - expected: FAIL - - [remote (none) to remote2 (*), expect to fail] - expected: FAIL - - [remote (null) to remote (*), expect to fail] - expected: FAIL - - [remote (none) to remote (*), expect to fail] - expected: FAIL - - [remote (none) to local (*), expect to fail] - expected: FAIL - - [remote (null) to local (*), expect to fail] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/redirect-preflight-2.htm.ini b/tests/wpt/metadata/cors/redirect-preflight-2.htm.ini deleted file mode 100644 index 3db95f5aa7e..00000000000 --- a/tests/wpt/metadata/cors/redirect-preflight-2.htm.ini +++ /dev/null @@ -1,8 +0,0 @@ -[redirect-preflight-2.htm] - type: testharness - [Same-origin custom-header request, redirect to cross-origin succeeds after doing a preflight] - expected: FAIL - - [Same-origin custom-header request, redirect to cross-origin fails after doing a non-successful preflight] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/response-headers.htm.ini b/tests/wpt/metadata/cors/response-headers.htm.ini index 15d1c72e334..46c5d996971 100644 --- a/tests/wpt/metadata/cors/response-headers.htm.ini +++ b/tests/wpt/metadata/cors/response-headers.htm.ini @@ -9,6 +9,3 @@ [getResponse: don't expose x-nonexposed] expected: FAIL - [getAllResponseHeaders: don't expose x-nonexposed] - expected: FAIL - diff --git a/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js b/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js index 56e9e09640f..c00b2e0e0ca 100644 --- a/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js +++ b/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js @@ -1,3 +1,5 @@ +const NUM_TESTS = 128; + function encode(n) { if (n === 0x20) { return "\x2B"; @@ -13,27 +15,34 @@ function encode(n) { return "%" + (s.length === 2 ? s : '0' + s); } -function do_test(n) { - async_test(function() { - var x = new XMLHttpRequest(); - x.onload = this.step_func_done(function(e) { - assert_equals(x.response, "a=" + encode(n)) - }); - x.onerror = this.unreached_func(); - x.open("POST", "resources/content.py"); - var usp = new URLSearchParams(); - usp.append("a", String.fromCharCode(n)); - x.send(usp) - }, "XMLHttpRequest.send(URLSearchParams) (" + n + ")"); -} - function run_test() { - var i = 0; - add_result_callback(function() { - if (++i === 128) { - return; + var tests = []; + var overall_test = async_test("Overall fetch with URLSearchParams"); + for (var i = 0; i < NUM_TESTS; i++) { + // Multiple subtests so that failures can be fine-grained + tests[i] = async_test("XMLHttpRequest.send(URLSearchParams) (" + i + ")"); + } + + // We use a single XHR since this test tends to time out + // with 128 consecutive fetches when run in parallel + // with many other WPT tests. + var x = new XMLHttpRequest(); + x.onload = overall_test.step_func(function() { + var response_split = x.response.split("&"); + overall_test.done(); + for (var i = 0; i < NUM_TESTS; i++) { + tests[i].step(function() { + assert_equals(response_split[i], "a" + i + "="+encode(i)); + tests[i].done(); + }); } - do_test(i); }); - do_test(i); + x.onerror = overall_test.unreached_func(); + + x.open("POST", "resources/content.py"); + var usp = new URLSearchParams(); + for (var i = 0; i < NUM_TESTS; i++) { + usp.append("a" + i, String.fromCharCode(i)); + } + x.send(usp) } |