diff options
Diffstat (limited to 'components/net/http_loader.rs')
| -rw-r--r-- | components/net/http_loader.rs | 692 |
1 files changed, 311 insertions, 381 deletions
diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index fe85fc5a6f7..20aaaffc237 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -3,7 +3,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use brotli::Decompressor; -use connector::{Connector, create_http_connector}; +use bytes::Bytes; +use connector::{BUF_SIZE, Connector, create_http_client, WrappedBody}; use cookie; use cookie_storage::CookieStorage; use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, HttpRequest as DevtoolsHttpRequest}; @@ -12,60 +13,51 @@ use fetch::cors_cache::CorsCache; use fetch::methods::{Data, DoneChannel, FetchContext, Target}; use fetch::methods::{is_cors_safelisted_request_header, is_cors_safelisted_method, main_fetch}; use flate2::read::{DeflateDecoder, GzDecoder}; +use headers_core::HeaderMapExt; +use headers_ext::{AccessControlAllowCredentials, AccessControlAllowHeaders}; +use headers_ext::{AccessControlAllowMethods, AccessControlRequestHeaders, AccessControlRequestMethod, Authorization}; +use headers_ext::{AccessControlAllowOrigin, AccessControlMaxAge, Basic}; +use headers_ext::{CacheControl, ContentEncoding, ContentLength}; +use headers_ext::{Host, IfModifiedSince, LastModified, Origin as HyperOrigin, Pragma, Referer, UserAgent}; use hsts::HstsList; +use http::{HeaderMap, Request as HyperRequest}; +use http::header::{self, HeaderName, HeaderValue}; +use http::uri::Authority; use http_cache::HttpCache; -use hyper::Error as HttpError; -use hyper::LanguageTag; -use hyper::client::{Pool, Request as HyperRequest, Response as HyperResponse}; -use hyper::header::{Accept, AccessControlAllowCredentials, AccessControlAllowHeaders}; -use hyper::header::{AccessControlAllowMethods, AccessControlAllowOrigin}; -use hyper::header::{AccessControlMaxAge, AccessControlRequestHeaders}; -use hyper::header::{AccessControlRequestMethod, AcceptEncoding, AcceptLanguage}; -use hyper::header::{Authorization, Basic, CacheControl, CacheDirective}; -use hyper::header::{ContentEncoding, ContentLength, Encoding, Header, Headers}; -use hyper::header::{Host, HttpDate, Origin as HyperOrigin, IfMatch, IfRange}; -use hyper::header::{IfUnmodifiedSince, IfModifiedSince, IfNoneMatch, Location}; -use hyper::header::{Pragma, Quality, QualityItem, Referer, SetCookie}; -use hyper::header::{UserAgent, q, qitem}; -use hyper::method::Method; -use hyper::status::StatusCode; -use hyper_openssl::OpensslClient; +use hyper::{Body, Client, Method, StatusCode, Response as HyperResponse}; use hyper_serde::Serde; use log; +use mime; use msg::constellation_msg::{HistoryStateId, PipelineId}; use net_traits::{CookieSource, FetchMetadata, NetworkError, ReferrerPolicy}; +use net_traits::quality::{quality_to_value, Quality, QualityItem}; use net_traits::request::{CacheMode, CredentialsMode, Destination, Origin}; use net_traits::request::{RedirectMode, Referrer, Request, RequestMode}; use net_traits::request::{ResponseTainting, ServiceWorkersMode}; use net_traits::response::{HttpsState, Response, ResponseBody, ResponseType}; +use openssl::ssl::SslConnectorBuilder; use resource_thread::AuthCache; use servo_channel::{channel, Sender}; use servo_url::{ImmutableOrigin, ServoUrl}; use std::collections::{HashMap, HashSet}; use std::error::Error; -use std::io::{self, Read, Write}; +use std::io::Cursor; use std::iter::FromIterator; use std::mem; use std::ops::Deref; use std::str::FromStr; +use std::sync::Mutex; use std::sync::RwLock; -use std::thread; -use time; -use time::Tm; -use unicase::UniCase; +use std::time::{Duration, SystemTime}; +use time::{self, Tm}; +use tokio::prelude::{Future, future, Stream}; +use tokio::runtime::Runtime; use uuid; -fn read_block<R: Read>(reader: &mut R) -> Result<Data, ()> { - let mut buf = vec![0; 32768]; - - match reader.read(&mut buf) { - Ok(len) if len > 0 => { - buf.truncate(len); - Ok(Data::Payload(buf)) - } - Ok(_) => Ok(Data::Done), - Err(_) => Err(()), - } +lazy_static! { + pub static ref HANDLE: Mutex<Runtime> = { + Mutex::new(Runtime::new().unwrap()) + }; } pub struct HttpState { @@ -74,20 +66,18 @@ pub struct HttpState { pub http_cache: RwLock<HttpCache>, pub auth_cache: RwLock<AuthCache>, pub history_states: RwLock<HashMap<HistoryStateId, Vec<u8>>>, - pub ssl_client: OpensslClient, - pub connector: Pool<Connector>, + pub client: Client<Connector, WrappedBody>, } impl HttpState { - pub fn new(ssl_client: OpensslClient) -> HttpState { + pub fn new(ssl_connector_builder: SslConnectorBuilder) -> HttpState { HttpState { hsts_list: RwLock::new(HstsList::new()), cookie_jar: RwLock::new(CookieStorage::new(150)), auth_cache: RwLock::new(AuthCache::new()), history_states: RwLock::new(HashMap::new()), http_cache: RwLock::new(HttpCache::new()), - ssl_client: ssl_client.clone(), - connector: create_http_connector(ssl_client), + client: create_http_client(ssl_connector_builder, HANDLE.lock().unwrap().executor()), } } } @@ -97,72 +87,63 @@ fn precise_time_ms() -> u64 { } // Step 3 of https://fetch.spec.whatwg.org/#concept-fetch. -pub fn set_default_accept(destination: Destination, headers: &mut Headers) { - if headers.has::<Accept>() { +pub fn set_default_accept(destination: Destination, headers: &mut HeaderMap) { + if headers.contains_key(header::ACCEPT) { return; } let value = match destination { // Step 3.2. Destination::Document => { vec![ - qitem(mime!(Text / Html)), - qitem(mime!(Application / ("xhtml+xml"))), - QualityItem::new(mime!(Application / Xml), q(0.9)), - QualityItem::new(mime!(_ / _), q(0.8)), + QualityItem::new(mime::TEXT_HTML, Quality::from_u16(1000)), + QualityItem::new("application/xhtml+xml".parse().unwrap(), Quality::from_u16(1000)), + QualityItem::new("application/xml".parse().unwrap(), Quality::from_u16(900)), + QualityItem::new(mime::STAR_STAR, Quality::from_u16(800)) ] }, // Step 3.3. Destination::Image => { vec![ - qitem(mime!(Image / Png)), - qitem(mime!(Image / ("svg+xml") )), - QualityItem::new(mime!(Image / _), q(0.8)), - QualityItem::new(mime!(_ / _), q(0.5)), + QualityItem::new(mime::IMAGE_PNG, Quality::from_u16(1000)), + QualityItem::new(mime::IMAGE_SVG, Quality::from_u16(1000)), + QualityItem::new(mime::IMAGE_STAR, Quality::from_u16(800)), + QualityItem::new(mime::STAR_STAR, Quality::from_u16(500)) ] }, // Step 3.3. Destination::Style => { vec![ - qitem(mime!(Text / Css)), - QualityItem::new(mime!(_ / _), q(0.1)) + QualityItem::new(mime::TEXT_CSS, Quality::from_u16(1000)), + QualityItem::new(mime::STAR_STAR, Quality::from_u16(100)) ] }, // Step 3.1. _ => { - vec![qitem(mime!(_ / _))] + vec![QualityItem::new(mime::STAR_STAR, Quality::from_u16(1000))] }, }; // Step 3.4. - headers.set(Accept(value)); + // TODO(eijebong): Change this once typed headers are done + headers.insert(header::ACCEPT, quality_to_value(value)); } -fn set_default_accept_encoding(headers: &mut Headers) { - if headers.has::<AcceptEncoding>() { +fn set_default_accept_encoding(headers: &mut HeaderMap) { + if headers.contains_key(header::ACCEPT_ENCODING) { return } - headers.set(AcceptEncoding(vec![ - qitem(Encoding::Gzip), - qitem(Encoding::Deflate), - qitem(Encoding::EncodingExt("br".to_owned())) - ])); + // TODO(eijebong): Change this once typed headers are done + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip, deflate, br")); } -pub fn set_default_accept_language(headers: &mut Headers) { - if headers.has::<AcceptLanguage>() { +pub fn set_default_accept_language(headers: &mut HeaderMap) { + if headers.contains_key(header::ACCEPT_LANGUAGE) { return; } - let mut en_us: LanguageTag = Default::default(); - en_us.language = Some("en".to_owned()); - en_us.region = Some("US".to_owned()); - let mut en: LanguageTag = Default::default(); - en.language = Some("en".to_owned()); - headers.set(AcceptLanguage(vec![ - qitem(en_us), - QualityItem::new(en, Quality(500)), - ])); + // TODO(eijebong): Change this once typed headers are done + headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US, en; q=0.5")); } /// <https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-state-no-referrer-when-downgrade> @@ -210,12 +191,12 @@ fn strip_url(mut referrer_url: ServoUrl, origin_only: bool) -> Option<ServoUrl> /// <https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer> /// Steps 4-6. -pub fn determine_request_referrer(headers: &mut Headers, +pub fn determine_request_referrer(headers: &mut HeaderMap, referrer_policy: ReferrerPolicy, referrer_source: ServoUrl, current_url: ServoUrl) -> Option<ServoUrl> { - assert!(!headers.has::<Referer>()); + assert!(!headers.contains_key(header::REFERER)); // FIXME(#14505): this does not seem to be the correct way of checking for // same-origin requests. let cross_origin = referrer_source.origin() != current_url.origin(); @@ -233,94 +214,65 @@ pub fn determine_request_referrer(headers: &mut Headers, } } -pub fn set_request_cookies(url: &ServoUrl, headers: &mut Headers, cookie_jar: &RwLock<CookieStorage>) { +pub fn set_request_cookies(url: &ServoUrl, headers: &mut HeaderMap, cookie_jar: &RwLock<CookieStorage>) { let mut cookie_jar = cookie_jar.write().unwrap(); if let Some(cookie_list) = cookie_jar.cookies_for_url(url, CookieSource::HTTP) { - let mut v = Vec::new(); - v.push(cookie_list.into_bytes()); - headers.set_raw("Cookie".to_owned(), v); + headers.insert(header::COOKIE, HeaderValue::from_bytes(cookie_list.as_bytes()).unwrap()); } } fn set_cookie_for_url(cookie_jar: &RwLock<CookieStorage>, request: &ServoUrl, - cookie_val: String) { + cookie_val: &str) { let mut cookie_jar = cookie_jar.write().unwrap(); let source = CookieSource::HTTP; - let header = Header::parse_header(&[cookie_val.into_bytes()]); - if let Ok(SetCookie(cookies)) = header { - for cookie in cookies { - if let Some(cookie) = cookie::Cookie::from_cookie_string(cookie, request, source) { - cookie_jar.push(cookie, request, source); - } - } + if let Some(cookie) = cookie::Cookie::from_cookie_string(cookie_val.into(), request, source) { + cookie_jar.push(cookie, request, source); } } -fn set_cookies_from_headers(url: &ServoUrl, headers: &Headers, cookie_jar: &RwLock<CookieStorage>) { - if let Some(cookies) = headers.get_raw("set-cookie") { - for cookie in cookies.iter() { - if let Ok(cookie_value) = String::from_utf8(cookie.clone()) { - set_cookie_for_url(&cookie_jar, - &url, - cookie_value); - } +fn set_cookies_from_headers(url: &ServoUrl, headers: &HeaderMap, cookie_jar: &RwLock<CookieStorage>) { + for cookie in headers.get_all(header::SET_COOKIE) { + if let Ok(cookie_str) = cookie.to_str() { + set_cookie_for_url(&cookie_jar, + &url, + &cookie_str); } } } -struct StreamedResponse { - decoder: Decoder, -} - - -impl Read for StreamedResponse { - #[inline] - fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { - match self.decoder { - Decoder::Gzip(ref mut d) => d.read(buf), - Decoder::Deflate(ref mut d) => d.read(buf), - Decoder::Brotli(ref mut d) => d.read(buf), - Decoder::Plain(ref mut d) => d.read(buf) - } - } -} - -impl StreamedResponse { - fn from_http_response(response: HyperResponse) -> io::Result<StreamedResponse> { - let decoder = { - if let Some(ref encoding) = response.headers.get::<ContentEncoding>().cloned() { - if encoding.contains(&Encoding::Gzip) { - Decoder::Gzip(GzDecoder::new(response)) - } - else if encoding.contains(&Encoding::Deflate) { - Decoder::Deflate(DeflateDecoder::new(response)) - } - else if encoding.contains(&Encoding::EncodingExt("br".to_owned())) { - Decoder::Brotli(Decompressor::new(response, 1024)) - } else { - Decoder::Plain(response) - } +impl Decoder { + fn from_http_response(response: &HyperResponse<Body>) -> Decoder { + if let Some(encoding) = response.headers().typed_get::<ContentEncoding>() { + if encoding.contains("gzip") { + Decoder::Gzip(None) + } + else if encoding.contains("deflate") { + Decoder::Deflate(DeflateDecoder::new(Cursor::new(Bytes::new()))) + } + else if encoding.contains("br") { + Decoder::Brotli(Decompressor::new(Cursor::new(Bytes::new()), BUF_SIZE)) } else { - Decoder::Plain(response) + Decoder::Plain } - }; - Ok(StreamedResponse { decoder: decoder }) + } else { + Decoder::Plain + } } } -enum Decoder { - Gzip(GzDecoder<HyperResponse>), - Deflate(DeflateDecoder<HyperResponse>), - Brotli(Decompressor<HyperResponse>), - Plain(HyperResponse) +pub enum Decoder { + Gzip(Option<GzDecoder<Cursor<Bytes>>>), + Deflate(DeflateDecoder<Cursor<Bytes>>), + Brotli(Decompressor<Cursor<Bytes>>), + Plain, } fn prepare_devtools_request(request_id: String, url: ServoUrl, method: Method, - headers: Headers, + headers: HeaderMap, body: Option<Vec<u8>>, pipeline_id: PipelineId, now: Tm, @@ -351,7 +303,7 @@ fn send_request_to_devtools(msg: ChromeToDevtoolsControlMsg, fn send_response_to_devtools(devtools_chan: &Sender<DevtoolsControlMsg>, request_id: String, - headers: Option<Headers>, + headers: Option<HeaderMap>, status: Option<(u16, Vec<u8>)>, pipeline_id: PipelineId) { let response = DevtoolsHttpResponse { headers: headers, status: status, body: None, pipeline_id: pipeline_id }; @@ -361,111 +313,93 @@ fn send_response_to_devtools(devtools_chan: &Sender<DevtoolsControlMsg>, let _ = devtools_chan.send(DevtoolsControlMsg::FromChrome(msg)); } -fn auth_from_cache(auth_cache: &RwLock<AuthCache>, origin: &ImmutableOrigin) -> Option<Basic> { +fn auth_from_cache(auth_cache: &RwLock<AuthCache>, origin: &ImmutableOrigin) -> Option<Authorization<Basic>> { if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(&origin.ascii_serialization()) { - let user_name = auth_entry.user_name.clone(); - let password = Some(auth_entry.password.clone()); - Some(Basic { username: user_name, password: password }) + let user_name = &auth_entry.user_name; + let password = &auth_entry.password; + Some(Authorization::basic(user_name, password)) } else { None } } -fn obtain_response(connector: &Pool<Connector>, +fn obtain_response(client: &Client<Connector, WrappedBody>, url: &ServoUrl, method: &Method, - request_headers: &Headers, + request_headers: &HeaderMap, data: &Option<Vec<u8>>, load_data_method: &Method, pipeline_id: &Option<PipelineId>, iters: u32, request_id: Option<&str>, is_xhr: bool) - -> Result<(HyperResponse, Option<ChromeToDevtoolsControlMsg>), NetworkError> { - let null_data = None; - - // loop trying connections in connection pool - // they may have grown stale (disconnected), in which case we'll get - // a ConnectionAborted error. this loop tries again with a new - // connection. - loop { - let mut headers = request_headers.clone(); - - // Avoid automatically sending request body if a redirect has occurred. - // - // TODO - This is the wrong behaviour according to the RFC. However, I'm not - // sure how much "correctness" vs. real-world is important in this case. - // - // https://tools.ietf.org/html/rfc7231#section-6.4 - let is_redirected_request = iters != 1; - let request_body; - match data { - &Some(ref d) if !is_redirected_request => { - headers.set(ContentLength(d.len() as u64)); - request_body = data; - } - _ => { - if *load_data_method != Method::Get && *load_data_method != Method::Head { - headers.set(ContentLength(0)) - } - request_body = &null_data; - } + -> Box<Future<Item=(HyperResponse<WrappedBody>, Option<ChromeToDevtoolsControlMsg>), + Error = NetworkError>> { + let mut headers = request_headers.clone(); + + // Avoid automatically sending request body if a redirect has occurred. + // + // TODO - This is the wrong behaviour according to the RFC. However, I'm not + // sure how much "correctness" vs. real-world is important in this case. + // + // https://tools.ietf.org/html/rfc7231#section-6.4 + let is_redirected_request = iters != 1; + let request_body; + match data { + &Some(ref d) if !is_redirected_request => { + headers.typed_insert(ContentLength(d.len() as u64)); + request_body = d.clone(); } - - if log_enabled!(log::Level::Info) { - info!("{} {}", method, url); - for header in headers.iter() { - info!(" - {}", header); + _ => { + if *load_data_method != Method::GET && *load_data_method != Method::HEAD { + headers.typed_insert(ContentLength(0)) } - info!("{:?}", data); + request_body = vec![]; } + } - let connect_start = precise_time_ms(); - - let request = HyperRequest::with_connector(method.clone(), - url.clone().into_url(), - &*connector); - let mut request = match request { - Ok(request) => request, - Err(e) => return Err(NetworkError::from_hyper_error(&url, e)), - }; - *request.headers_mut() = headers.clone(); - - let connect_end = precise_time_ms(); - - let send_start = precise_time_ms(); + if log_enabled!(log::Level::Info) { + info!("{} {}", method, url); + for header in headers.iter() { + info!(" - {:?}", header); + } + info!("{:?}", data); + } - let mut request_writer = match request.start() { - Ok(streaming) => streaming, - Err(e) => return Err(NetworkError::Internal(e.description().to_owned())), - }; + let connect_start = precise_time_ms(); + // https://url.spec.whatwg.org/#percent-encoded-bytes + let request = HyperRequest::builder() + .method(method) + .uri(url.clone().into_url().as_ref().replace("|", "%7C").replace("{", "%7B").replace("}", "%7D")) + .body(WrappedBody::new(request_body.clone().into())); - if let Some(ref data) = *request_body { - if let Err(e) = request_writer.write_all(&data) { - return Err(NetworkError::Internal(e.description().to_owned())) - } - } + let mut request = match request { + Ok(request) => request, + Err(e) => return Box::new(future::result(Err(NetworkError::from_http_error(&e)))), + }; + *request.headers_mut() = headers.clone(); + let connect_end = precise_time_ms(); - let response = match request_writer.send() { - Ok(w) => w, - Err(HttpError::Io(ref io_error)) - if io_error.kind() == io::ErrorKind::ConnectionAborted || - io_error.kind() == io::ErrorKind::ConnectionReset => { - debug!("connection aborted ({:?}), possibly stale, trying new connection", io_error.description()); - continue; - }, - Err(e) => return Err(NetworkError::Internal(e.description().to_owned())), - }; + let request_id = request_id.map(|v| v.to_owned()); + let pipeline_id = pipeline_id.clone(); + let closure_url = url.clone(); + let method = method.clone(); + let send_start = precise_time_ms(); + Box::new(client.request(request).and_then(move |res| { let send_end = precise_time_ms(); let msg = if let Some(request_id) = request_id { - if let Some(pipeline_id) = *pipeline_id { + if let Some(pipeline_id) = pipeline_id { Some(prepare_devtools_request( - request_id.into(), - url.clone(), method.clone(), headers, - request_body.clone(), pipeline_id, time::now(), + request_id, + closure_url, method.clone(), headers, + Some(request_body.clone()), pipeline_id, time::now(), connect_end - connect_start, send_end - send_start, is_xhr)) + // TODO: ^This is not right, connect_start is taken before contructing the + // request and connect_end at the end of it. send_start is takend before the + // connection too. I'm not sure it's currently possible to get the time at the + // point between the connection and the start of a request. } else { debug!("Not notifying devtools (no pipeline_id)"); None @@ -474,8 +408,9 @@ fn obtain_response(connector: &Pool<Connector>, debug!("Not notifying devtools (no request_id)"); None }; - return Ok((response, msg)); - } + let decoder = Decoder::from_http_response(&res); + Ok((res.map(move |r| WrappedBody::new_with_decoder(r, decoder)), msg)) + }).map_err(move |e| NetworkError::from_hyper_error(&e))) } /// [HTTP fetch](https://fetch.spec.whatwg.org#http-fetch) @@ -543,8 +478,8 @@ pub fn http_fetch(request: &mut Request, let method_mismatch = !method_cache_match && (!is_cors_safelisted_method(&request.method) || request.use_cors_preflight); - let header_mismatch = request.headers.iter().any(|view| - !cache.match_header(&*request, view.name()) && !is_cors_safelisted_request_header(&view) + let header_mismatch = request.headers.iter().any(|(name, value)| + !cache.match_header(&*request, &name) && !is_cors_safelisted_request_header(&name, &value) ); // Sub-substep 1 @@ -579,16 +514,19 @@ pub fn http_fetch(request: &mut Request, let mut response = response.unwrap(); // Step 5 - if response.actual_response().status.map_or(false, is_redirect_status) { + if response.actual_response().status.as_ref().map_or(false, is_redirect_status) { // Substep 1. - if response.actual_response().status.map_or(true, |s| s != StatusCode::SeeOther) { + if response.actual_response().status.as_ref().map_or(true, |s| s.0 != StatusCode::SEE_OTHER) { // TODO: send RST_STREAM frame } // Substep 2-3. - let location = response.actual_response().headers.get::<Location>().map( - |l| ServoUrl::parse_with_base(response.actual_response().url(), l) - .map_err(|err| err.description().into())); + let location = response.actual_response().headers.get(header::LOCATION) + .and_then(|v| HeaderValue::to_str(v).map(|l| { + ServoUrl::parse_with_base(response.actual_response().url(), &l) + .map_err(|err| err.description().into()) + }).ok() + ); // Substep 4. response.actual_response_mut().location_url = location; @@ -664,7 +602,7 @@ pub fn http_redirect_fetch(request: &mut Request, } // Step 9 - if response.actual_response().status.map_or(true, |s| s != StatusCode::SeeOther) && + if response.actual_response().status.as_ref().map_or(true, |s| s.0 != StatusCode::SEE_OTHER) && request.body.as_ref().map_or(false, |b| b.is_empty()) { return Response::network_error(NetworkError::Internal("Request body is not done".into())); } @@ -675,10 +613,10 @@ pub fn http_redirect_fetch(request: &mut Request, } // Step 11 - if response.actual_response().status.map_or(false, |code| - ((code == StatusCode::MovedPermanently || code == StatusCode::Found) && request.method == Method::Post) || - (code == StatusCode::SeeOther && request.method != Method::Head)) { - request.method = Method::Get; + if response.actual_response().status.as_ref().map_or(false, |(code, _)| + ((*code == StatusCode::MOVED_PERMANENTLY || *code == StatusCode::FOUND) && request.method == Method::POST) || + (*code == StatusCode::SEE_OTHER && request.method != Method::HEAD)) { + request.method = Method::GET; request.body = None; } @@ -701,10 +639,9 @@ pub fn http_redirect_fetch(request: &mut Request, fn try_immutable_origin_to_hyper_origin(url_origin: &ImmutableOrigin) -> Option<HyperOrigin> { match *url_origin { - // TODO (servo/servo#15569) Set "Origin: null" when hyper supports it - ImmutableOrigin::Opaque(_) => None, + ImmutableOrigin::Opaque(_) => Some(HyperOrigin::NULL), ImmutableOrigin::Tuple(ref scheme, ref host, ref port) => - Some(HyperOrigin::new(scheme.clone(), host.to_string(), Some(port.clone()))) + HyperOrigin::try_from_parts(&scheme, &host.to_string(), Some(port.clone())).ok() } } @@ -742,7 +679,7 @@ fn http_network_or_cache_fetch(request: &mut Request, None => match http_request.method { // Step 6 - Method::Post | Method::Put => + Method::POST | Method::PUT => Some(0), // Step 5 _ => None @@ -753,7 +690,7 @@ fn http_network_or_cache_fetch(request: &mut Request, // Step 8 if let Some(content_length_value) = content_length_value { - http_request.headers.set(ContentLength(content_length_value)); + http_request.headers.typed_insert(ContentLength(content_length_value)); if http_request.keep_alive { // Step 9 TODO: needs request's client object } @@ -764,7 +701,7 @@ fn http_network_or_cache_fetch(request: &mut Request, match http_request.referrer { Referrer::NoReferrer => (), Referrer::ReferrerUrl(ref http_request_referrer) => - http_request.headers.set(Referer(http_request_referrer.to_string())), + http_request.headers.typed_insert::<Referer>(http_request_referrer.to_string().parse().unwrap()), Referrer::Client => // it should be impossible for referrer to be anything else during fetching // https://fetch.spec.whatwg.org/#concept-request-referrer @@ -772,19 +709,19 @@ fn http_network_or_cache_fetch(request: &mut Request, }; // Step 11 - if cors_flag || (http_request.method != Method::Get && http_request.method != Method::Head) { + if cors_flag || (http_request.method != Method::GET && http_request.method != Method::HEAD) { debug_assert_ne!(http_request.origin, Origin::Client); if let Origin::Origin(ref url_origin) = http_request.origin { if let Some(hyper_origin) = try_immutable_origin_to_hyper_origin(url_origin) { - http_request.headers.set(hyper_origin) + http_request.headers.typed_insert(hyper_origin) } } } // Step 12 - if !http_request.headers.has::<UserAgent>() { + if !http_request.headers.contains_key(header::USER_AGENT) { let user_agent = context.user_agent.clone().into_owned(); - http_request.headers.set(UserAgent(user_agent)); + http_request.headers.typed_insert::<UserAgent>(user_agent.parse().unwrap()); } match http_request.cache_mode { @@ -794,20 +731,20 @@ fn http_network_or_cache_fetch(request: &mut Request, }, // Step 14 - CacheMode::NoCache if !http_request.headers.has::<CacheControl>() => { - http_request.headers.set(CacheControl(vec![CacheDirective::MaxAge(0)])); + CacheMode::NoCache if !http_request.headers.contains_key(header::CACHE_CONTROL) => { + http_request.headers.typed_insert(CacheControl::new().with_max_age(Duration::from_secs(0))); }, // Step 15 CacheMode::Reload | CacheMode::NoStore => { // Substep 1 - if !http_request.headers.has::<Pragma>() { - http_request.headers.set(Pragma::NoCache); + if !http_request.headers.contains_key(header::PRAGMA) { + http_request.headers.typed_insert(Pragma::no_cache()); } // Substep 2 - if !http_request.headers.has::<CacheControl>() { - http_request.headers.set(CacheControl(vec![CacheDirective::NoCache])); + if !http_request.headers.contains_key(header::CACHE_CONTROL) { + http_request.headers.typed_insert(CacheControl::new().with_no_cache()); } }, @@ -816,11 +753,13 @@ fn http_network_or_cache_fetch(request: &mut Request, // Step 16 let current_url = http_request.current_url(); - let host = Host { - hostname: current_url.host_str().unwrap().to_owned(), - port: current_url.port() - }; - http_request.headers.set(host); + let host = Host::from( + format!("{}{}", current_url.host_str().unwrap(), + current_url.port().map(|v| format!(":{}", v)).unwrap_or("".into()) + ).parse::<Authority>().unwrap() + ); + + http_request.headers.typed_insert(host); // unlike http_loader, we should not set the accept header // here, according to the fetch spec set_default_accept_encoding(&mut http_request.headers); @@ -835,7 +774,7 @@ fn http_network_or_cache_fetch(request: &mut Request, &mut http_request.headers, &context.state.cookie_jar); // Substep 2 - if !http_request.headers.has::<Authorization<String>>() { + if !http_request.headers.contains_key(header::AUTHORIZATION) { // Substep 3 let mut authorization_value = None; @@ -849,16 +788,16 @@ fn http_network_or_cache_fetch(request: &mut Request, // Substep 5 if authentication_fetch_flag && authorization_value.is_none() { if has_credentials(¤t_url) { - authorization_value = Some(Basic { - username: current_url.username().to_owned(), - password: current_url.password().map(str::to_owned) - }) + authorization_value = Some(Authorization::basic( + current_url.username(), + current_url.password().unwrap_or("") + )); } } // Substep 6 if let Some(basic) = authorization_value { - http_request.headers.set(Authorization(basic)); + http_request.headers.typed_insert(basic); } } } @@ -886,17 +825,12 @@ fn http_network_or_cache_fetch(request: &mut Request, if needs_revalidation { revalidating_flag = true; // Substep 5 - // TODO: find out why the typed header getter return None from the headers of cached responses. - if let Some(date_slice) = response_headers.get_raw("Last-Modified") { - let date_string = String::from_utf8_lossy(&date_slice[0]); - if let Ok(http_date) = HttpDate::from_str(&date_string) { - http_request.headers.set(IfModifiedSince(http_date)); - } + if let Some(http_date) = response_headers.typed_get::<LastModified>() { + let http_date: SystemTime = http_date.into(); + http_request.headers.typed_insert(IfModifiedSince::from(http_date)); } - if let Some(entity_tag) = - response_headers.get_raw("ETag") { - http_request.headers.set_raw("If-None-Match", entity_tag.to_vec()); - + if let Some(entity_tag) = response_headers.get(header::ETAG) { + http_request.headers.insert(header::IF_NONE_MATCH, entity_tag.clone()); } } else { // Substep 6 @@ -945,14 +879,14 @@ fn http_network_or_cache_fetch(request: &mut Request, done_chan, context); // Substep 3 if let Some((200...399, _)) = forward_response.raw_status { - if !http_request.method.safe() { + if !http_request.method.is_safe() { if let Ok(mut http_cache) = context.state.http_cache.write() { http_cache.invalidate(&http_request, &forward_response); } } } // Substep 4 - if revalidating_flag && forward_response.status.map_or(false, |s| s == StatusCode::NotModified) { + if revalidating_flag && forward_response.status.as_ref().map_or(false, |s| s.0 == StatusCode::NOT_MODIFIED) { if let Ok(mut http_cache) = context.state.http_cache.write() { response = http_cache.refresh(&http_request, forward_response.clone(), done_chan); wait_for_cached_response(done_chan, &mut response); @@ -976,7 +910,9 @@ fn http_network_or_cache_fetch(request: &mut Request, // Step 23 // FIXME: Figure out what to do with request window objects - if let (Some(StatusCode::Unauthorized), false, true) = (response.status, cors_flag, credentials_flag) { + if let (Some((StatusCode::UNAUTHORIZED, _)), false, true) = + (response.status.as_ref(), cors_flag, credentials_flag) + { // Substep 1 // TODO: Spec says requires testing on multiple WWW-Authenticate headers @@ -1002,7 +938,7 @@ fn http_network_or_cache_fetch(request: &mut Request, } // Step 24 - if let Some(StatusCode::ProxyAuthenticationRequired) = response.status { + if let Some((StatusCode::PROXY_AUTHENTICATION_REQUIRED, _)) = response.status.as_ref() { // Step 1 if request_has_no_window { return Response::network_error(NetworkError::Internal("Can't find Window object".into())); @@ -1062,38 +998,38 @@ fn http_network_fetch(request: &Request, // do not. Once we support other kinds of fetches we'll need to be more fine grained here // since things like image fetches are classified differently by devtools let is_xhr = request.destination == Destination::None; - let wrapped_response = obtain_response(&context.state.connector, - &url, - &request.method, - &request.headers, - &request.body, &request.method, - &request.pipeline_id, request.redirect_count + 1, - request_id.as_ref().map(Deref::deref), is_xhr); + let response_future = obtain_response(&context.state.client, + &url, + &request.method, + &request.headers, + &request.body, &request.method, + &request.pipeline_id, request.redirect_count + 1, + request_id.as_ref().map(Deref::deref), is_xhr); let pipeline_id = request.pipeline_id; - let (res, msg) = match wrapped_response { + // This will only get the headers, the body is read later + let (res, msg) = match response_future.wait() { Ok(wrapped_response) => wrapped_response, Err(error) => return Response::network_error(error), }; if log_enabled!(log::Level::Info) { info!("response for {}", url); - for header in res.headers.iter() { - info!(" - {}", header); + for header in res.headers().iter() { + info!(" - {:?}", header); } } let mut response = Response::new(url.clone()); - response.status = Some(res.status); - response.raw_status = Some((res.status_raw().0, - res.status_raw().1.as_bytes().to_vec())); - response.headers = res.headers.clone(); + response.status = Some((res.status(), res.status().canonical_reason().unwrap_or("").into())); + response.raw_status = Some((res.status().as_u16(), res.status().canonical_reason().unwrap_or("").into())); + response.headers = res.headers().clone(); response.referrer = request.referrer.to_url().cloned(); response.referrer_policy = request.referrer_policy.clone(); let res_body = response.body.clone(); - // We're about to spawn a thread to be waited on here + // We're about to spawn a future to be waited on here let (done_sender, done_receiver) = channel(); *done_chan = Some((done_sender.clone(), done_receiver)); let meta = match response.metadata().expect("Response metadata should exist at this stage") { @@ -1107,68 +1043,56 @@ fn http_network_fetch(request: &Request, if cancellation_listener.lock().unwrap().cancelled() { return Response::network_error(NetworkError::Internal("Fetch aborted".into())) } - thread::Builder::new().name(format!("fetch worker thread")).spawn(move || { - match StreamedResponse::from_http_response(res) { - Ok(mut res) => { - *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); - - if let Some(ref sender) = devtools_sender { - if let Some(m) = msg { - send_request_to_devtools(m, &sender); - } - // --- Tell devtools that we got a response - // Send an HttpResponse message to devtools with the corresponding request_id - if let Some(pipeline_id) = pipeline_id { - send_response_to_devtools( - &sender, request_id.unwrap(), - meta_headers.map(Serde::into_inner), - meta_status, - pipeline_id); - } - } + *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); - loop { - if cancellation_listener.lock().unwrap().cancelled() { - *res_body.lock().unwrap() = ResponseBody::Done(vec![]); - let _ = done_sender.send(Data::Cancelled); - return; - } - match read_block(&mut res) { - Ok(Data::Payload(chunk)) => { - if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { - body.extend_from_slice(&chunk); - let _ = done_sender.send(Data::Payload(chunk)); - } - }, - Ok(Data::Done) | Err(_) => { - let mut body = res_body.lock().unwrap(); - let completed_body = match *body { - ResponseBody::Receiving(ref mut body) => { - mem::replace(body, vec![]) - }, - _ => vec![], - }; - *body = ResponseBody::Done(completed_body); - let _ = done_sender.send(Data::Done); - break; - } - Ok(Data::Cancelled) => unreachable!() // read_block doesn't return Data::Cancelled - } - } - } - Err(_) => { - // XXXManishearth we should propagate this error somehow - *res_body.lock().unwrap() = ResponseBody::Done(vec![]); - let _ = done_sender.send(Data::Done); - } + if let Some(ref sender) = devtools_sender { + if let Some(m) = msg { + send_request_to_devtools(m, &sender); } - }).expect("Thread spawning failed"); - // TODO these substeps aren't possible yet - // Substep 1 + // --- Tell devtools that we got a response + // Send an HttpResponse message to devtools with the corresponding request_id + if let Some(pipeline_id) = pipeline_id { + send_response_to_devtools( + &sender, request_id.unwrap(), + meta_headers.map(Serde::into_inner), + meta_status, + pipeline_id); + } + } - // Substep 2 + let done_sender = done_sender.clone(); + let done_sender2 = done_sender.clone(); + HANDLE.lock().unwrap().spawn(res.into_body().map_err(|_|()).fold(res_body, move |res_body, chunk| { + if cancellation_listener.lock().unwrap().cancelled() { + *res_body.lock().unwrap() = ResponseBody::Done(vec![]); + let _ = done_sender.send(Data::Cancelled); + return future::failed(()); + } + if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { + let bytes = chunk.into_bytes(); + body.extend_from_slice(&*bytes); + let _ = done_sender.send(Data::Payload(bytes.to_vec())); + } + future::ok(res_body) + }).and_then(move |res_body| { + let mut body = res_body.lock().unwrap(); + let completed_body = match *body { + ResponseBody::Receiving(ref mut body) => { + mem::replace(body, vec![]) + }, + _ => vec![], + }; + *body = ResponseBody::Done(completed_body); + let _ = done_sender2.send(Data::Done); + future::ok(()) + }).map_err(|_| ())); + + // TODO these substeps aren't possible yet + // Substep 1 + + // Substep 2 // TODO Determine if response was retrieved over HTTPS // TODO Servo needs to decide what ciphers are to be treated as "deprecated" @@ -1183,11 +1107,11 @@ fn http_network_fetch(request: &Request, // TODO when https://bugzilla.mozilla.org/show_bug.cgi?id=1030660 // is resolved, this step will become uneccesary // TODO this step - if let Some(encoding) = response.headers.get::<ContentEncoding>() { - if encoding.contains(&Encoding::Gzip) { + if let Some(encoding) = response.headers.typed_get::<ContentEncoding>() { + if encoding.contains("gzip") { } - else if encoding.contains(&Encoding::Compress) { + else if encoding.contains("compress") { } }; @@ -1228,7 +1152,7 @@ fn cors_preflight_fetch(request: &Request, -> Response { // Step 1 let mut preflight = Request::new(request.current_url(), Some(request.origin.clone()), request.pipeline_id); - preflight.method = Method::Options; + preflight.method = Method::OPTIONS; preflight.initiator = request.initiator.clone(); preflight.destination = request.destination.clone(); preflight.origin = request.origin.clone(); @@ -1236,20 +1160,24 @@ fn cors_preflight_fetch(request: &Request, preflight.referrer_policy = request.referrer_policy; // Step 2 - preflight.headers.set::<AccessControlRequestMethod>( - AccessControlRequestMethod(request.method.clone())); + preflight.headers.typed_insert::<AccessControlRequestMethod>( + AccessControlRequestMethod::from(request.method.clone())); // Step 3 let mut headers = request.headers .iter() - .filter(|view| !is_cors_safelisted_request_header(view)) - .map(|view| UniCase(view.name().to_ascii_lowercase().to_owned())) - .collect::<Vec<UniCase<String>>>(); + .filter(|(name, value)| !is_cors_safelisted_request_header(&name, &value)) + .map(|(name, _)| name.as_str()) + .collect::<Vec<&str>>(); headers.sort(); + let headers = headers + .iter() + .map(|name| HeaderName::from_str(name).unwrap()) + .collect::<Vec<HeaderName>>(); // Step 4 if !headers.is_empty() { - preflight.headers.set::<AccessControlRequestHeaders>(AccessControlRequestHeaders(headers)); + preflight.headers.typed_insert(AccessControlRequestHeaders::from_iter(headers)); } // Step 5 @@ -1257,11 +1185,11 @@ fn cors_preflight_fetch(request: &Request, // Step 6 if cors_check(&request, &response).is_ok() && - response.status.map_or(false, |status| status.is_success()) { + response.status.as_ref().map_or(false, |(status, _)| status.is_success()) { // Substep 1, 2 - let mut methods = if response.headers.has::<AccessControlAllowMethods>() { - match response.headers.get::<AccessControlAllowMethods>() { - Some(&AccessControlAllowMethods(ref m)) => m.clone(), + let mut methods = if response.headers.contains_key(header::ACCESS_CONTROL_ALLOW_METHODS) { + match response.headers.typed_get::<AccessControlAllowMethods>() { + Some(methods) => methods.iter().collect(), // Substep 4 None => return Response::network_error(NetworkError::Internal("CORS ACAM check failed".into())) } @@ -1270,9 +1198,9 @@ fn cors_preflight_fetch(request: &Request, }; // Substep 3 - let header_names = if response.headers.has::<AccessControlAllowHeaders>() { - match response.headers.get::<AccessControlAllowHeaders>() { - Some(&AccessControlAllowHeaders(ref hn)) => hn.clone(), + let header_names = if response.headers.contains_key(header::ACCESS_CONTROL_ALLOW_HEADERS) { + match response.headers.typed_get::<AccessControlAllowHeaders>() { + Some(names) => names.iter().collect(), // Substep 4 None => return Response::network_error(NetworkError::Internal("CORS ACAH check failed".into())) } @@ -1282,7 +1210,7 @@ fn cors_preflight_fetch(request: &Request, // Substep 5 if (methods.iter().any(|m| m.as_ref() == "*") || - header_names.iter().any(|hn| &**hn == "*")) && + header_names.iter().any(|hn| hn.as_str() == "*")) && request.credentials_mode == CredentialsMode::Include { return Response::network_error( NetworkError::Internal("CORS ACAH/ACAM and request credentials mode mismatch".into())); @@ -1304,22 +1232,24 @@ fn cors_preflight_fetch(request: &Request, // Substep 8 if request.headers.iter().any( - |header| header.name() == "authorization" && - header_names.iter().all(|hn| *hn != UniCase(header.name()))) { + |(name, _)| name == header::AUTHORIZATION && + header_names.iter().all(|hn| hn != name)) { return Response::network_error(NetworkError::Internal("CORS authorization check failed".into())); } // Substep 9 debug!("CORS check: Allowed headers: {:?}, current headers: {:?}", header_names, request.headers); - let set: HashSet<&UniCase<String>> = HashSet::from_iter(header_names.iter()); + let set: HashSet<&HeaderName> = HashSet::from_iter(header_names.iter()); if request.headers.iter().any( - |ref hv| !set.contains(&UniCase(hv.name().to_owned())) && !is_cors_safelisted_request_header(hv)) { + |(name, value)| !set.contains(name) && !is_cors_safelisted_request_header(&name, &value)) { return Response::network_error(NetworkError::Internal("CORS headers check failed".into())); } // Substep 10, 11 - let max_age = response.headers.get::<AccessControlMaxAge>().map(|acma| acma.0).unwrap_or(0); - + let max_age: Duration = response.headers.typed_get::<AccessControlMaxAge>() + .map(|acma| acma.into()) + .unwrap_or(Duration::from_secs(0)); + let max_age = max_age.as_secs() as u32; // Substep 12 // TODO: Need to define what an imposed limit on max-age is @@ -1346,26 +1276,26 @@ fn cors_preflight_fetch(request: &Request, /// [CORS check](https://fetch.spec.whatwg.org#concept-cors-check) fn cors_check(request: &Request, response: &Response) -> Result<(), ()> { // Step 1 - let origin = response.headers.get::<AccessControlAllowOrigin>().cloned(); + let origin = response.headers.typed_get::<AccessControlAllowOrigin>(); // Step 2 let origin = origin.ok_or(())?; // Step 3 if request.credentials_mode != CredentialsMode::Include && - origin == AccessControlAllowOrigin::Any { + origin == AccessControlAllowOrigin::ANY { return Ok(()); } // Step 4 - let origin = match origin { - AccessControlAllowOrigin::Value(origin) => origin, + let origin = match origin.origin() { + Some(origin) => origin, // if it's Any or Null at this point, there's nothing to do but return Err(()) - _ => return Err(()) + None => return Err(()) }; match request.origin { - Origin::Origin(ref o) if o.ascii_serialization() == origin.trim() => {}, + Origin::Origin(ref o) if o.ascii_serialization() == origin.to_string().trim() => {}, _ => return Err(()) } @@ -1375,7 +1305,7 @@ fn cors_check(request: &Request, response: &Response) -> Result<(), ()> { } // Step 6 - let credentials = response.headers.get::<AccessControlAllowCredentials>().cloned(); + let credentials = response.headers.typed_get::<AccessControlAllowCredentials>(); // Step 7 if credentials.is_some() { @@ -1390,20 +1320,20 @@ fn has_credentials(url: &ServoUrl) -> bool { !url.username().is_empty() || url.password().is_some() } -fn is_no_store_cache(headers: &Headers) -> bool { - headers.has::<IfModifiedSince>() | headers.has::<IfNoneMatch>() | - headers.has::<IfUnmodifiedSince>() | headers.has::<IfMatch>() | - headers.has::<IfRange>() +fn is_no_store_cache(headers: &HeaderMap) -> bool { + headers.contains_key(header::IF_MODIFIED_SINCE) | headers.contains_key(header::IF_NONE_MATCH) | + headers.contains_key(header::IF_UNMODIFIED_SINCE) | headers.contains_key(header::IF_MATCH) | + headers.contains_key(header::IF_RANGE) } /// <https://fetch.spec.whatwg.org/#redirect-status> -pub fn is_redirect_status(status: StatusCode) -> bool { - match status { - StatusCode::MovedPermanently | - StatusCode::Found | - StatusCode::SeeOther | - StatusCode::TemporaryRedirect | - StatusCode::PermanentRedirect => true, +pub fn is_redirect_status(status: &(StatusCode, String)) -> bool { + match status.0 { + StatusCode::MOVED_PERMANENTLY | + StatusCode::FOUND | + StatusCode::SEE_OTHER | + StatusCode::TEMPORARY_REDIRECT | + StatusCode::PERMANENT_REDIRECT => true, _ => false, } } |