diff options
author | bors-servo <lbergstrom+bors@mozilla.com> | 2018-11-26 10:34:31 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-11-26 10:34:31 -0500 |
commit | d282618baac130a95e7b4a6f9d99312f35401dab (patch) | |
tree | 59013df1fdff12f4ebd3312f68d97a1c6acf2c43 | |
parent | 7c65505df3fff47f43062da20088113631ed9ae0 (diff) | |
parent | b23dd0587b5d31d34915fc3906e550d05108a185 (diff) | |
download | servo-d282618baac130a95e7b4a6f9d99312f35401dab.tar.gz servo-d282618baac130a95e7b4a6f9d99312f35401dab.zip |
Auto merge of #22134 - ferjm:load_better_blob, r=jdm
Support range requests for blob URLs
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #21467 and fix #22053
- [X] There are tests for these changes.
<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22134)
<!-- Reviewable:end -->
-rw-r--r-- | components/net/blob_loader.rs | 108 | ||||
-rw-r--r-- | components/net/fetch/methods.rs | 264 | ||||
-rw-r--r-- | components/net/filemanager_thread.rs | 253 | ||||
-rw-r--r-- | components/net/lib.rs | 1 | ||||
-rw-r--r-- | components/net/tests/fetch.rs | 42 | ||||
-rw-r--r-- | components/net_traits/blob_url_store.rs | 4 | ||||
-rw-r--r-- | tests/wpt/mozilla/meta/MANIFEST.json | 20 | ||||
-rw-r--r-- | tests/wpt/mozilla/tests/mozilla/range_request_blob_url.html | 60 | ||||
-rw-r--r-- | tests/wpt/mozilla/tests/mozilla/range_request_file_url.html | 4 |
9 files changed, 520 insertions, 236 deletions
diff --git a/components/net/blob_loader.rs b/components/net/blob_loader.rs deleted file mode 100644 index e6d64acc16e..00000000000 --- a/components/net/blob_loader.rs +++ /dev/null @@ -1,108 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -use crate::filemanager_thread::FileManager; -use headers_core::HeaderMapExt; -use headers_ext::{ContentLength, ContentType}; -use http::header::{self, HeaderValue}; -use http::HeaderMap; -use ipc_channel::ipc; -use mime::{self, Mime}; -use net_traits::blob_url_store::parse_blob_url; -use net_traits::filemanager_thread::ReadFileProgress; -use net_traits::{http_percent_encode, NetworkError}; -use servo_url::ServoUrl; - -// TODO: Check on GET -// https://w3c.github.io/FileAPI/#requestResponseModel - -/// https://fetch.spec.whatwg.org/#concept-basic-fetch (partial) -// TODO: make async. -pub fn load_blob_sync( - url: ServoUrl, - filemanager: FileManager, -) -> Result<(HeaderMap, Vec<u8>), NetworkError> { - let (id, origin) = match parse_blob_url(&url) { - Ok((id, origin)) => (id, origin), - Err(()) => { - let e = format!("Invalid blob URL format {:?}", url); - return Err(NetworkError::Internal(e)); - }, - }; - - let (sender, receiver) = ipc::channel().unwrap(); - let check_url_validity = true; - filemanager.read_file(sender, id, check_url_validity, origin); - - let blob_buf = match receiver.recv().unwrap() { - Ok(ReadFileProgress::Meta(blob_buf)) => blob_buf, - Ok(_) => { - return Err(NetworkError::Internal( - "Invalid filemanager reply".to_string(), - )); - }, - Err(e) => { - return Err(NetworkError::Internal(format!("{:?}", e))); - }, - }; - - let content_type: Mime = blob_buf.type_string.parse().unwrap_or(mime::TEXT_PLAIN); - let charset = content_type.get_param(mime::CHARSET); - - let mut headers = HeaderMap::new(); - - if let Some(name) = blob_buf.filename { - let charset = charset - .map(|c| c.as_ref().into()) - .unwrap_or("us-ascii".to_owned()); - // TODO(eijebong): Replace this once the typed header is there - headers.insert( - header::CONTENT_DISPOSITION, - HeaderValue::from_bytes( - format!( - "inline; {}", - if charset.to_lowercase() == "utf-8" { - format!( - "filename=\"{}\"", - String::from_utf8(name.as_bytes().into()).unwrap() - ) - } else { - format!( - "filename*=\"{}\"''{}", - charset, - http_percent_encode(name.as_bytes()) - ) - } - ) - .as_bytes(), - ) - .unwrap(), - ); - } - - // Basic fetch, Step 4. - headers.typed_insert(ContentLength(blob_buf.size as u64)); - // Basic fetch, Step 5. - headers.typed_insert(ContentType::from(content_type.clone())); - - let mut bytes = blob_buf.bytes; - loop { - match receiver.recv().unwrap() { - Ok(ReadFileProgress::Partial(ref mut new_bytes)) => { - bytes.append(new_bytes); - }, - Ok(ReadFileProgress::EOF) => { - return Ok((headers, bytes)); - }, - Ok(_) => { - return Err(NetworkError::Internal( - "Invalid filemanager reply".to_string(), - )); - }, - Err(e) => { - return Err(NetworkError::Internal(format!("{:?}", e))); - }, - } - } -} diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 70dcf4d7983..7eb963fcd5e 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -2,10 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use crate::blob_loader::load_blob_sync; use crate::data_loader::decode; use crate::fetch::cors_cache::CorsCache; -use crate::filemanager_thread::FileManager; +use crate::filemanager_thread::{fetch_file_in_chunks, FileManager, FILE_CHUNK_SIZE}; use crate::http_loader::{determine_request_referrer, http_fetch, HttpState}; use crate::http_loader::{set_default_accept, set_default_accept_language}; use crate::subresource_integrity::is_response_integrity_valid; @@ -19,6 +18,8 @@ use hyper::StatusCode; use ipc_channel::ipc::IpcReceiver; use mime::{self, Mime}; use mime_guess::guess_mime_type; +use net_traits::blob_url_store::{parse_blob_url, BlobURLStoreError}; +use net_traits::filemanager_thread::RelativePos; use net_traits::request::{CredentialsMode, Destination, Referrer, Request, RequestMode}; use net_traits::request::{Origin, ResponseTainting, Window}; use net_traits::response::{Response, ResponseBody, ResponseType}; @@ -26,21 +27,18 @@ use net_traits::{FetchTaskTarget, NetworkError, ReferrerPolicy, ResourceFetchTim use servo_url::ServoUrl; use std::borrow::Cow; use std::fs::File; -use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::io::{BufReader, Seek, SeekFrom}; use std::mem; use std::ops::Bound; use std::str; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex}; -use std::thread; lazy_static! { static ref X_CONTENT_TYPE_OPTIONS: HeaderName = HeaderName::from_static("x-content-type-options"); } -const FILE_CHUNK_SIZE: usize = 32768; //32 KB - pub type Target<'a> = &'a mut (dyn FetchTaskTarget + Send); pub enum Data { @@ -492,6 +490,74 @@ fn wait_for_response(response: &mut Response, target: Target, done_chan: &mut Do } } +/// Range header start and end values. +pub enum RangeRequestBounds { + /// The range bounds are known and set to final values. + Final(RelativePos), + /// We need extra information to set the range bounds. + /// i.e. buffer or file size. + Pending(u64), +} + +impl RangeRequestBounds { + pub fn get_final(&self, len: Option<u64>) -> Result<RelativePos, ()> { + match self { + RangeRequestBounds::Final(pos) => { + if let Some(len) = len { + if pos.start <= len as i64 { + return Ok(pos.clone()); + } + } + Err(()) + }, + RangeRequestBounds::Pending(offset) => Ok(RelativePos::from_opts( + if let Some(len) = len { + Some((len - u64::min(len, *offset)) as i64) + } else { + Some(0) + }, + None, + )), + } + } +} + +/// Get the range bounds if the `Range` header is present. +fn get_range_request_bounds(range: Option<Range>) -> RangeRequestBounds { + if let Some(ref range) = range { + let (start, end) = match range + .iter() + .collect::<Vec<(Bound<u64>, Bound<u64>)>>() + .first() + { + Some(&(Bound::Included(start), Bound::Unbounded)) => (start, None), + Some(&(Bound::Included(start), Bound::Included(end))) => { + // `end` should be less or equal to `start`. + (start, Some(i64::max(start as i64, end as i64))) + }, + Some(&(Bound::Unbounded, Bound::Included(offset))) => { + return RangeRequestBounds::Pending(offset); + }, + _ => (0, None), + }; + RangeRequestBounds::Final(RelativePos::from_opts(Some(start as i64), end)) + } else { + RangeRequestBounds::Final(RelativePos::from_opts(Some(0), None)) + } +} + +fn partial_content(response: &mut Response) { + let reason = "Partial Content".to_owned(); + response.status = Some((StatusCode::PARTIAL_CONTENT, reason.clone())); + response.raw_status = Some((StatusCode::PARTIAL_CONTENT.as_u16(), reason.into())); +} + +fn range_not_satisfiable_error(response: &mut Response) { + let reason = "Range Not Satisfiable".to_owned(); + response.status = Some((StatusCode::RANGE_NOT_SATISFIABLE, reason.clone())); + response.raw_status = Some((StatusCode::RANGE_NOT_SATISFIABLE.as_u16(), reason.into())); +} + /// [Scheme fetch](https://fetch.spec.whatwg.org#scheme-fetch) fn scheme_fetch( request: &mut Request, @@ -537,106 +603,56 @@ fn scheme_fetch( } if let Ok(file_path) = url.to_file_path() { if let Ok(file) = File::open(file_path.clone()) { - let mime = guess_mime_type(file_path); + // Get range bounds (if any) and try to seek to the requested offset. + // If seeking fails, bail out with a NetworkError. + let file_size = match file.metadata() { + Ok(metadata) => Some(metadata.len()), + Err(_) => None, + }; let mut response = Response::new(url, ResourceFetchTiming::new(request.timing_type())); + + let range_header = request.headers.typed_get::<Range>(); + let is_range_request = range_header.is_some(); + let range = match get_range_request_bounds(range_header).get_final(file_size) { + Ok(range) => range, + Err(_) => { + range_not_satisfiable_error(&mut response); + return response; + }, + }; + let mut reader = BufReader::with_capacity(FILE_CHUNK_SIZE, file); + if reader.seek(SeekFrom::Start(range.start as u64)).is_err() { + return Response::network_error(NetworkError::Internal( + "Unexpected method for file".into(), + )); + } + + // Set response status to 206 if Range header is present. + // At this point we should have already validated the header. + if is_range_request { + partial_content(&mut response); + } + + // Set Content-Type header. + let mime = guess_mime_type(file_path); response.headers.typed_insert(ContentType::from(mime)); + // Setup channel to receive cross-thread messages about the file fetch + // operation. let (done_sender, done_receiver) = unbounded(); *done_chan = Some((done_sender.clone(), done_receiver)); *response.body.lock().unwrap() = ResponseBody::Receiving(vec![]); - let res_body = response.body.clone(); - - let cancellation_listener = context.cancellation_listener.clone(); + fetch_file_in_chunks( + done_sender, + reader, + response.body.clone(), + context.cancellation_listener.clone(), + range, + ); - let (start, end) = if let Some(ref range) = request.headers.typed_get::<Range>() - { - match range - .iter() - .collect::<Vec<(Bound<u64>, Bound<u64>)>>() - .first() - { - Some(&(Bound::Included(start), Bound::Unbounded)) => (start, None), - Some(&(Bound::Included(start), Bound::Included(end))) => { - // `end` should be less or equal to `start`. - (start, Some(u64::max(start, end))) - }, - Some(&(Bound::Unbounded, Bound::Included(offset))) => { - if let Ok(metadata) = file.metadata() { - // `offset` cannot be bigger than the file size. - (metadata.len() - u64::min(metadata.len(), offset), None) - } else { - (0, None) - } - }, - _ => (0, None), - } - } else { - (0, None) - }; - - thread::Builder::new() - .name("fetch file worker thread".to_string()) - .spawn(move || { - let mut reader = BufReader::with_capacity(FILE_CHUNK_SIZE, file); - if reader.seek(SeekFrom::Start(start)).is_err() { - warn!("Fetch - could not seek to {:?}", start); - } - - loop { - if cancellation_listener.lock().unwrap().cancelled() { - *res_body.lock().unwrap() = ResponseBody::Done(vec![]); - let _ = done_sender.send(Data::Cancelled); - return; - } - let length = { - let buffer = reader.fill_buf().unwrap().to_vec(); - let mut buffer_len = buffer.len(); - if let ResponseBody::Receiving(ref mut body) = - *res_body.lock().unwrap() - { - let offset = usize::min( - { - if let Some(end) = end { - let remaining_bytes = - end as usize - start as usize - body.len(); - if remaining_bytes <= FILE_CHUNK_SIZE { - // This is the last chunk so we set buffer - // len to 0 to break the reading loop. - buffer_len = 0; - remaining_bytes - } else { - FILE_CHUNK_SIZE - } - } else { - FILE_CHUNK_SIZE - } - }, - buffer.len(), - ); - body.extend_from_slice(&buffer[0..offset]); - let _ = done_sender.send(Data::Payload(buffer)); - } - buffer_len - }; - if length == 0 { - let mut body = res_body.lock().unwrap(); - let completed_body = match *body { - ResponseBody::Receiving(ref mut body) => { - mem::replace(body, vec![]) - }, - _ => vec![], - }; - *body = ResponseBody::Done(completed_body); - let _ = done_sender.send(Data::Done); - break; - } - reader.consume(length); - } - }) - .expect("Failed to create fetch file worker thread"); response } else { Response::network_error(NetworkError::Internal("Opening file failed".into())) @@ -649,7 +665,7 @@ fn scheme_fetch( }, "blob" => { - println!("Loading blob {}", url.as_str()); + debug!("Loading blob {}", url.as_str()); // Step 2. if request.method != Method::GET { return Response::network_error(NetworkError::Internal( @@ -657,19 +673,51 @@ fn scheme_fetch( )); } - match load_blob_sync(url.clone(), context.filemanager.clone()) { - Ok((headers, bytes)) => { - let mut response = - Response::new(url, ResourceFetchTiming::new(request.timing_type())); - response.headers = headers; - *response.body.lock().unwrap() = ResponseBody::Done(bytes); - response - }, - Err(e) => { - debug!("Failed to load {}: {:?}", url, e); - Response::network_error(e) + let range_header = request.headers.typed_get::<Range>(); + let is_range_request = range_header.is_some(); + // We will get a final version of this range once we have + // the length of the data backing the blob. + let range = get_range_request_bounds(range_header); + + let (id, origin) = match parse_blob_url(&url) { + Ok((id, origin)) => (id, origin), + Err(()) => { + return Response::network_error(NetworkError::Internal( + "Invalid blob url".into(), + )); }, + }; + + let mut response = Response::new(url, ResourceFetchTiming::new(request.timing_type())); + if is_range_request { + partial_content(&mut response); } + + let (done_sender, done_receiver) = unbounded(); + *done_chan = Some((done_sender.clone(), done_receiver)); + *response.body.lock().unwrap() = ResponseBody::Receiving(vec![]); + let check_url_validity = true; + if let Err(err) = context.filemanager.fetch_file( + &done_sender, + context.cancellation_listener.clone(), + id, + check_url_validity, + origin, + &mut response, + range, + ) { + let _ = done_sender.send(Data::Done); + let err = match err { + BlobURLStoreError::InvalidRange => { + range_not_satisfiable_error(&mut response); + return response; + }, + _ => format!("{:?}", err), + }; + return Response::network_error(NetworkError::Internal(err)); + }; + + response }, "ftp" => { diff --git a/components/net/filemanager_thread.rs b/components/net/filemanager_thread.rs index 8f0fb05b229..f0779df632b 100644 --- a/components/net/filemanager_thread.rs +++ b/components/net/filemanager_thread.rs @@ -2,26 +2,37 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +use crate::fetch::methods::{CancellationListener, Data, RangeRequestBounds}; +use crossbeam_channel::Sender; use embedder_traits::{EmbedderMsg, EmbedderProxy, FilterPattern}; +use headers_ext::{ContentLength, ContentType, HeaderMap, HeaderMapExt}; +use http::header::{self, HeaderValue}; use ipc_channel::ipc::{self, IpcSender}; +use mime::{self, Mime}; use mime_guess::guess_mime_type_opt; use net_traits::blob_url_store::{BlobBuf, BlobURLStoreError}; use net_traits::filemanager_thread::{FileManagerResult, FileManagerThreadMsg, FileOrigin}; use net_traits::filemanager_thread::{ FileManagerThreadError, ReadFileProgress, RelativePos, SelectedFile, }; +use net_traits::http_percent_encode; +use net_traits::response::{Response, ResponseBody}; +use servo_arc::Arc as ServoArc; use servo_config::prefs::PREFS; use std::collections::HashMap; use std::fs::File; -use std::io::{Read, Seek, SeekFrom}; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; +use std::mem; use std::ops::Index; use std::path::{Path, PathBuf}; use std::sync::atomic::{self, AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, Mutex, RwLock}; use std::thread; use url::Url; use uuid::Uuid; +pub const FILE_CHUNK_SIZE: usize = 32768; //32 KB + /// FileManagerStore's entry struct FileStoreEntry { /// Origin of the entry's "creator" @@ -91,6 +102,30 @@ impl FileManager { .expect("Thread spawning failed"); } + // Read a file for the Fetch implementation. + // It gets the required headers synchronously and reads the actual content + // in a separate thread. + pub fn fetch_file( + &self, + done_sender: &Sender<Data>, + cancellation_listener: Arc<Mutex<CancellationListener>>, + id: Uuid, + check_url_validity: bool, + origin: FileOrigin, + response: &mut Response, + range: RangeRequestBounds, + ) -> Result<(), BlobURLStoreError> { + self.store.fetch_blob_buf( + done_sender, + cancellation_listener, + &id, + &origin, + range, + check_url_validity, + response, + ) + } + pub fn promote_memory( &self, blob_buf: BlobBuf, @@ -452,7 +487,7 @@ impl FileManagerStore { None => "".to_string(), }; - chunked_read(sender, &mut file, range.len(), opt_filename, type_string); + read_file_in_chunks(sender, &mut file, range.len(), opt_filename, type_string); Ok(()) } else { Err(BlobURLStoreError::InvalidEntry) @@ -489,6 +524,109 @@ impl FileManagerStore { ) } + fn fetch_blob_buf( + &self, + done_sender: &Sender<Data>, + cancellation_listener: Arc<Mutex<CancellationListener>>, + id: &Uuid, + origin_in: &FileOrigin, + range: RangeRequestBounds, + check_url_validity: bool, + response: &mut Response, + ) -> Result<(), BlobURLStoreError> { + let file_impl = self.get_impl(id, origin_in, check_url_validity)?; + match file_impl { + FileImpl::Memory(buf) => { + let range = match range.get_final(Some(buf.size)) { + Ok(range) => range, + Err(_) => { + return Err(BlobURLStoreError::InvalidRange); + }, + }; + + let range = range.to_abs_range(buf.size as usize); + let len = range.len() as u64; + + set_headers( + &mut response.headers, + len, + buf.type_string.parse().unwrap_or(mime::TEXT_PLAIN), + /* filename */ None, + ); + + let mut bytes = vec![]; + bytes.extend_from_slice(buf.bytes.index(range)); + + let _ = done_sender.send(Data::Payload(bytes)); + let _ = done_sender.send(Data::Done); + + Ok(()) + }, + FileImpl::MetaDataOnly(metadata) => { + /* XXX: Snapshot state check (optional) https://w3c.github.io/FileAPI/#snapshot-state. + Concretely, here we create another file, and this file might not + has the same underlying file state (meta-info plus content) as the time + create_entry is called. + */ + + let file = File::open(&metadata.path) + .map_err(|e| BlobURLStoreError::External(e.to_string()))?; + + let range = match range.get_final(Some(metadata.size)) { + Ok(range) => range, + Err(_) => { + return Err(BlobURLStoreError::InvalidRange); + }, + }; + + let mut reader = BufReader::with_capacity(FILE_CHUNK_SIZE, file); + if reader.seek(SeekFrom::Start(range.start as u64)).is_err() { + return Err(BlobURLStoreError::External( + "Unexpected method for blob".into(), + )); + } + + let filename = metadata + .path + .file_name() + .and_then(|osstr| osstr.to_str()) + .map(|s| s.to_string()); + + set_headers( + &mut response.headers, + metadata.size, + guess_mime_type_opt(metadata.path).unwrap_or(mime::TEXT_PLAIN), + filename, + ); + + fetch_file_in_chunks( + done_sender.clone(), + reader, + response.body.clone(), + cancellation_listener, + range, + ); + + Ok(()) + }, + FileImpl::Sliced(parent_id, inner_rel_pos) => { + // Next time we don't need to check validity since + // we have already done that for requesting URL if necessary. + return self.fetch_blob_buf( + done_sender, + cancellation_listener, + &parent_id, + origin_in, + RangeRequestBounds::Final( + RelativePos::full_range().slice_inner(&inner_rel_pos), + ), + false, + response, + ); + }, + } + } + fn dec_ref(&self, id: &Uuid, origin_in: &FileOrigin) -> Result<(), BlobURLStoreError> { let (do_remove, opt_parent_id) = match self.entries.read().unwrap().get(id) { Some(entry) => { @@ -609,9 +747,7 @@ fn select_files_pref_enabled() -> bool { .unwrap_or(false) } -const CHUNK_SIZE: usize = 8192; - -fn chunked_read( +fn read_file_in_chunks( sender: &IpcSender<FileManagerResult<ReadFileProgress>>, file: &mut File, size: usize, @@ -619,7 +755,7 @@ fn chunked_read( type_string: String, ) { // First chunk - let mut buf = vec![0; CHUNK_SIZE]; + let mut buf = vec![0; FILE_CHUNK_SIZE]; match file.read(&mut buf) { Ok(n) => { buf.truncate(n); @@ -639,7 +775,7 @@ fn chunked_read( // Send the remaining chunks loop { - let mut buf = vec![0; CHUNK_SIZE]; + let mut buf = vec![0; FILE_CHUNK_SIZE]; match file.read(&mut buf) { Ok(0) => { let _ = sender.send(Ok(ReadFileProgress::EOF)); @@ -656,3 +792,104 @@ fn chunked_read( } } } + +pub fn fetch_file_in_chunks( + done_sender: Sender<Data>, + mut reader: BufReader<File>, + res_body: ServoArc<Mutex<ResponseBody>>, + cancellation_listener: Arc<Mutex<CancellationListener>>, + range: RelativePos, +) { + thread::Builder::new() + .name("fetch file worker thread".to_string()) + .spawn(move || { + loop { + if cancellation_listener.lock().unwrap().cancelled() { + *res_body.lock().unwrap() = ResponseBody::Done(vec![]); + let _ = done_sender.send(Data::Cancelled); + return; + } + let length = { + let buffer = reader.fill_buf().unwrap().to_vec(); + let mut buffer_len = buffer.len(); + if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { + let offset = usize::min( + { + if let Some(end) = range.end { + // HTTP Range requests are specified with closed ranges, + // while Rust uses half-open ranges. We add +1 here so + // we don't skip the last requested byte. + let remaining_bytes = + end as usize - range.start as usize - body.len() + 1; + if remaining_bytes <= FILE_CHUNK_SIZE { + // This is the last chunk so we set buffer + // len to 0 to break the reading loop. + buffer_len = 0; + remaining_bytes + } else { + FILE_CHUNK_SIZE + } + } else { + FILE_CHUNK_SIZE + } + }, + buffer.len(), + ); + let chunk = &buffer[0..offset]; + body.extend_from_slice(chunk); + let _ = done_sender.send(Data::Payload(chunk.to_vec())); + } + buffer_len + }; + if length == 0 { + let mut body = res_body.lock().unwrap(); + let completed_body = match *body { + ResponseBody::Receiving(ref mut body) => mem::replace(body, vec![]), + _ => vec![], + }; + *body = ResponseBody::Done(completed_body); + let _ = done_sender.send(Data::Done); + break; + } + reader.consume(length); + } + }) + .expect("Failed to create fetch file worker thread"); +} + +fn set_headers(headers: &mut HeaderMap, content_length: u64, mime: Mime, filename: Option<String>) { + headers.typed_insert(ContentLength(content_length)); + headers.typed_insert(ContentType::from(mime.clone())); + let name = match filename { + Some(name) => name, + None => return, + }; + let charset = mime.get_param(mime::CHARSET); + let charset = charset + .map(|c| c.as_ref().into()) + .unwrap_or("us-ascii".to_owned()); + // TODO(eijebong): Replace this once the typed header is there + // https://github.com/hyperium/headers/issues/8 + headers.insert( + header::CONTENT_DISPOSITION, + HeaderValue::from_bytes( + format!( + "inline; {}", + if charset.to_lowercase() == "utf-8" { + format!( + "filename=\"{}\"", + String::from_utf8(name.as_bytes().into()).unwrap() + ) + } else { + format!( + "filename*=\"{}\"''{}", + charset, + http_percent_encode(name.as_bytes()) + ) + } + ) + .as_bytes(), + ) + .unwrap(), + ); +} diff --git a/components/net/lib.rs b/components/net/lib.rs index 04d41dd8d30..d5943fa380c 100644 --- a/components/net/lib.rs +++ b/components/net/lib.rs @@ -17,7 +17,6 @@ extern crate profile_traits; #[macro_use] extern crate serde; -mod blob_loader; pub mod connector; pub mod cookie; pub mod cookie_storage; diff --git a/components/net/tests/fetch.rs b/components/net/tests/fetch.rs index 51f25059ad1..6831ca7aa46 100644 --- a/components/net/tests/fetch.rs +++ b/components/net/tests/fetch.rs @@ -29,14 +29,15 @@ use mime::{self, Mime}; use msg::constellation_msg::TEST_PIPELINE_ID; use net::connector::create_ssl_connector_builder; use net::fetch::cors_cache::CorsCache; -use net::fetch::methods::{CancellationListener, FetchContext}; +use net::fetch::methods::{self, CancellationListener, FetchContext}; use net::filemanager_thread::FileManager; use net::hsts::HstsEntry; use net::test::HttpState; use net_traits::request::{Destination, Origin, RedirectMode, Referrer, Request, RequestMode}; use net_traits::response::{CacheState, Response, ResponseBody, ResponseType}; use net_traits::{ - IncludeSubdomains, NetworkError, ReferrerPolicy, ResourceFetchTiming, ResourceTimingType, + FetchTaskTarget, IncludeSubdomains, NetworkError, ReferrerPolicy, ResourceFetchTiming, + ResourceTimingType, }; use servo_url::{ImmutableOrigin, ServoUrl}; use std::fs::File; @@ -127,7 +128,27 @@ fn test_fetch_blob() { use ipc_channel::ipc; use net_traits::blob_url_store::BlobBuf; - let mut context = new_fetch_context(None, None); + struct FetchResponseCollector { + sender: Sender<Response>, + buffer: Vec<u8>, + expected: Vec<u8>, + } + + impl FetchTaskTarget for FetchResponseCollector { + fn process_request_body(&mut self, _: &Request) {} + fn process_request_eof(&mut self, _: &Request) {} + fn process_response(&mut self, _: &Response) {} + fn process_response_chunk(&mut self, chunk: Vec<u8>) { + self.buffer.extend_from_slice(chunk.as_slice()); + } + /// Fired when the response is fully fetched + fn process_response_eof(&mut self, response: &Response) { + assert_eq!(self.buffer, self.expected); + let _ = self.sender.send(response.clone()); + } + } + + let context = new_fetch_context(None, None); let bytes = b"content"; let blob_buf = BlobBuf { @@ -147,7 +168,18 @@ fn test_fetch_blob() { let url = ServoUrl::parse(&format!("blob:{}{}", origin.as_str(), id.to_simple())).unwrap(); let mut request = Request::new(url, Some(Origin::Origin(origin.origin())), None); - let fetch_response = fetch_with_context(&mut request, &mut context); + + let (sender, receiver) = unbounded(); + + let mut target = FetchResponseCollector { + sender, + buffer: vec![], + expected: bytes.to_vec(), + }; + + methods::fetch(&mut request, &mut target, &context); + + let fetch_response = receiver.recv().unwrap(); assert!(!fetch_response.is_network_error()); @@ -165,7 +197,7 @@ fn test_fetch_blob() { assert_eq!( *fetch_response.body.lock().unwrap(), - ResponseBody::Done(bytes.to_vec()) + ResponseBody::Receiving(vec![]) ); } diff --git a/components/net_traits/blob_url_store.rs b/components/net_traits/blob_url_store.rs index a429c6880bc..c9bb670ee29 100644 --- a/components/net_traits/blob_url_store.rs +++ b/components/net_traits/blob_url_store.rs @@ -9,7 +9,7 @@ use url::Url; use uuid::Uuid; /// Errors returned to Blob URL Store request -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum BlobURLStoreError { /// Invalid File UUID InvalidFileID, @@ -17,6 +17,8 @@ pub enum BlobURLStoreError { InvalidOrigin, /// Invalid entry content InvalidEntry, + /// Invalid range + InvalidRange, /// External error, from like file system, I/O etc. External(String), } diff --git a/tests/wpt/mozilla/meta/MANIFEST.json b/tests/wpt/mozilla/meta/MANIFEST.json index eddddddeb97..66b49e51489 100644 --- a/tests/wpt/mozilla/meta/MANIFEST.json +++ b/tests/wpt/mozilla/meta/MANIFEST.json @@ -14270,6 +14270,12 @@ {} ] ], + "mozilla/range_request_blob_url.html": [ + [ + "/_mozilla/mozilla/range_request_blob_url.html", + {} + ] + ], "mozilla/range_request_file_url.html": [ [ "/_mozilla/mozilla/range_request_file_url.html", @@ -27064,7 +27070,7 @@ "testharness" ], "mozilla/interfaces.html": [ - "ad17e930ddb5bc2daecb86216efe8885ae399173", + "ad17e930ddb5bc2daecb86216efe8885ae399173", "testharness" ], "mozilla/interfaces.js": [ @@ -27072,7 +27078,7 @@ "support" ], "mozilla/interfaces.worker.js": [ - "a5f2e00f234ea66b80e8a9bd4dbbc5433926191f", + "a5f2e00f234ea66b80e8a9bd4dbbc5433926191f", "testharness" ], "mozilla/invalid-this.html": [ @@ -27239,8 +27245,12 @@ "8de03455bcb0d18258f76af20f58c14868fe1c21", "testharness" ], + "mozilla/range_request_blob_url.html": [ + "075397620e989dafc814c0ed2bca46bd476bccf6", + "testharness" + ], "mozilla/range_request_file_url.html": [ - "65fe13fe93d97cebc2846ff7d7deab3eb84c1787", + "4fd4ddc8b1a9959e90b243795267c220d6a05f5e", "testharness" ], "mozilla/referrer-policy/OWNERS": [ @@ -32904,11 +32914,11 @@ "testharness" ], "mozilla/window_performance.html": [ - "690870b7080e179481ca0255f7c30337e8b6636a", + "690870b7080e179481ca0255f7c30337e8b6636a", "testharness" ], "mozilla/window_performance_topLevelDomComplete.html": [ - "50bbc2917b5ac900b5061a0b2c30b6c1fef1067e", + "50bbc2917b5ac900b5061a0b2c30b6c1fef1067e", "testharness" ], "mozilla/window_requestAnimationFrame.html": [ diff --git a/tests/wpt/mozilla/tests/mozilla/range_request_blob_url.html b/tests/wpt/mozilla/tests/mozilla/range_request_blob_url.html new file mode 100644 index 00000000000..075397620e9 --- /dev/null +++ b/tests/wpt/mozilla/tests/mozilla/range_request_blob_url.html @@ -0,0 +1,60 @@ +<html> +<head> +<script src="/resources/testharness.js"></script> +<script src="/resources/testharnessreport.js"></script> +<script> +[{ + range: "bytes=0-", + status: 206, + expected: "abcdefghijklmnopqrstuvwxyz" +}, { + range: "bytes=0-9", + status: 206, + expected: "abcdefghi" +}, { + range: "bytes=1-9", + status: 206, + expected: "bcdefghi" +}, { + range: "bytes=-10", + status: 206, + expected: "qrstuvwxyz" +}, { + range: "bytes=0-100", + status: 206, + expected: "abcdefghijklmnopqrstuvwxyz" +}, { + range: "bytes=100-", + status: 416, + expected: "" +}, { + range: "bytes=-100", + status: 206, + expected: "abcdefghijklmnopqrstuvwxyz" +}].forEach(test => { + promise_test(function() { + const abc = "abcdefghijklmnopqrstuvwxyz"; + const blob = new Blob([abc], { "type": "text/plain" }); + return fetch(URL.createObjectURL(blob), { + headers: { + "Range": test.range + } + }) + .then(response => { + assert_equals(response.status, test.status); + if (response.status != 206) { + return ""; + } + return response.text(); + }) + .then(response => { + assert_equals(response, test.expected); + }); + }); +}); + +</script> +</head> +</body> +</html> + diff --git a/tests/wpt/mozilla/tests/mozilla/range_request_file_url.html b/tests/wpt/mozilla/tests/mozilla/range_request_file_url.html index 65fe13fe93d..4fd4ddc8b1a 100644 --- a/tests/wpt/mozilla/tests/mozilla/range_request_file_url.html +++ b/tests/wpt/mozilla/tests/mozilla/range_request_file_url.html @@ -3,6 +3,10 @@ <script src="/resources/testharness.js"></script> <script src="/resources/testharnessreport.js"></script> <script> + +// FIXME(ferjm) https://github.com/servo/servo/issues/22269 +// This is not using file:// urls. + [{ file: "resources/range_small.txt", range: "bytes=0-", |