From 036e74524a72920ffca6deb4d6a850cf04b31736 Mon Sep 17 00:00:00 2001 From: Martin Robinson Date: Wed, 16 Oct 2024 09:53:24 -0700 Subject: net: Start reducing number of IPCs channels used for fetch with a `FetchThread` (#33863) Instead of creating a `ROUTER` for each fetch, create a fetch thread which handles all incoming and outcoming fetch requests. Now messages involving fetches carry a "request id" which indicates which fetch is being addressed by the message. This greatly reduces the number of file descriptors used by fetch. In addition, the interface for kicking off fetches is simplified when using the `Listener` with `Document`s and the `GlobalScope`. This does not fix all leaked file descriptors / mach ports, but greatly eliminates the number used. Now tests can be run without limiting procesess on modern macOS systems. Followup work: 1. There are more instances where fetch is done using the old method. Some of these require more changes in order to be converted to the `FetchThread` approach. 2. Eliminate usage of IPC channels when doing redirects. 3. Also eliminate the IPC channel used for cancel handling. 4. This change opens up the possiblity of controlling the priority of fetch requests. Fixes #29834. Signed-off-by: Martin Robinson --- components/net/tests/fetch.rs | 297 +++++++++++------------------------------- 1 file changed, 77 insertions(+), 220 deletions(-) (limited to 'components/net/tests/fetch.rs') diff --git a/components/net/tests/fetch.rs b/components/net/tests/fetch.rs index bc0d19a5b9a..81321770a4e 100644 --- a/components/net/tests/fetch.rs +++ b/components/net/tests/fetch.rs @@ -33,15 +33,15 @@ use net::test::HttpState; use net_traits::filemanager_thread::FileTokenCheck; use net_traits::http_status::HttpStatus; use net_traits::request::{ - Destination, Origin, RedirectMode, Referrer, Request, RequestBuilder, RequestMode, + Destination, RedirectMode, Referrer, Request, RequestBuilder, RequestMode, }; -use net_traits::response::{CacheState, HttpsState, Response, ResponseBody, ResponseType}; +use net_traits::response::{CacheState, Response, ResponseBody, ResponseType}; use net_traits::{ FetchTaskTarget, IncludeSubdomains, NetworkError, ReferrerPolicy, ResourceFetchTiming, ResourceTimingType, }; use servo_arc::Arc as ServoArc; -use servo_url::{ImmutableOrigin, ServoUrl}; +use servo_url::ServoUrl; use tokio_test::block_on; use uuid::Uuid; @@ -61,14 +61,9 @@ fn test_fetch_response_is_not_network_error() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -80,14 +75,9 @@ fn test_fetch_response_is_not_network_error() { #[test] fn test_fetch_on_bad_port_is_network_error() { let url = ServoUrl::parse("http://www.example.org:6667").unwrap(); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); assert!(fetch_response.is_network_error()); let fetch_error = fetch_response.get_network_error().unwrap(); @@ -105,14 +95,9 @@ fn test_fetch_response_body_matches_const_message() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -130,14 +115,9 @@ fn test_fetch_response_body_matches_const_message() { #[test] fn test_fetch_aboutblank() { let url = ServoUrl::parse("about:blank").unwrap(); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); // We should see an opaque-filtered response. @@ -168,12 +148,12 @@ fn test_fetch_blob() { impl FetchTaskTarget for FetchResponseCollector { fn process_request_body(&mut self, _: &Request) {} fn process_request_eof(&mut self, _: &Request) {} - fn process_response(&mut self, _: &Response) {} - fn process_response_chunk(&mut self, chunk: Vec) { + fn process_response(&mut self, _: &Request, _: &Response) {} + fn process_response_chunk(&mut self, _: &Request, chunk: Vec) { self.buffer.extend_from_slice(chunk.as_slice()); } /// Fired when the response is fully fetched - fn process_response_eof(&mut self, response: &Response) { + fn process_response_eof(&mut self, _: &Request, response: &Response) { assert_eq!(self.buffer, self.expected); let _ = self.sender.send(response.clone()); } @@ -200,13 +180,9 @@ fn test_fetch_blob() { ); let url = ServoUrl::parse(&format!("blob:{}{}", origin.as_str(), id.simple())).unwrap(); - let mut request = Request::new( - url, - Some(Origin::Origin(origin.origin())), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(origin.origin()) + .build(); let (sender, receiver) = unbounded(); @@ -246,14 +222,9 @@ fn test_file() { .unwrap(); let url = ServoUrl::from_file_path(path.clone()).unwrap(); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let pool = CoreResourceThreadPool::new(1); let pool_handle = Arc::new(pool); @@ -294,14 +265,9 @@ fn test_file() { #[test] fn test_fetch_ftp() { let url = ServoUrl::parse("ftp://not-supported").unwrap(); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); assert!(fetch_response.is_network_error()); } @@ -309,14 +275,9 @@ fn test_fetch_ftp() { #[test] fn test_fetch_bogus_scheme() { let url = ServoUrl::parse("bogus://whatever").unwrap(); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); assert!(fetch_response.is_network_error()); } @@ -360,15 +321,7 @@ fn test_cors_preflight_fetch() { let (server, url) = make_server(handler); let target_url = url.clone().join("a.html").unwrap(); - - let origin = Origin::Origin(ImmutableOrigin::new_opaque()); - let mut request = Request::new( - url.clone(), - Some(origin), - Referrer::ReferrerUrl(target_url), - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url, Referrer::ReferrerUrl(target_url)).build(); request.referrer_policy = Some(ReferrerPolicy::Origin); request.use_cors_preflight = true; request.mode = RequestMode::CorsMode; @@ -418,14 +371,7 @@ fn test_cors_preflight_cache_fetch() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(ImmutableOrigin::new_opaque()); - let mut request = Request::new( - url.clone(), - Some(origin.clone()), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build(); request.use_cors_preflight = true; request.mode = RequestMode::CorsMode; let mut wrapped_request0 = request.clone(); @@ -485,14 +431,7 @@ fn test_cors_preflight_fetch_network_error() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(ImmutableOrigin::new_opaque()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build(); request.method = Method::from_bytes(b"CHICKEN").unwrap(); request.use_cors_preflight = true; request.mode = RequestMode::CorsMode; @@ -519,14 +458,9 @@ fn test_fetch_response_is_basic_filtered() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -587,14 +521,7 @@ fn test_fetch_response_is_cors_filtered() { let (server, url) = make_server(handler); // an origin mis-match will stop it from defaulting to a basic filtered response - let origin = Origin::Origin(ImmutableOrigin::new_opaque()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build(); request.mode = RequestMode::CorsMode; let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -626,14 +553,7 @@ fn test_fetch_response_is_opaque_filtered() { let (server, url) = make_server(handler); // an origin mis-match will fall through to an Opaque filtered response - let origin = Origin::Origin(ImmutableOrigin::new_opaque()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -679,14 +599,9 @@ fn test_fetch_response_is_opaque_redirect_filtered() { let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); request.redirect_mode = RedirectMode::Manual; let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -718,14 +633,9 @@ fn test_fetch_with_local_urls_only() { let (server, server_url) = make_server(handler); let do_fetch = |url: ServoUrl| { - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); // Set the flag. request.local_urls_only = true; @@ -786,14 +696,9 @@ fn test_fetch_with_hsts() { HstsEntry::new("localhost".to_owned(), IncludeSubdomains::NotIncluded, None).unwrap(), ); } - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); // Set the flag. request.local_urls_only = false; let response = fetch_with_context(&mut request, &mut context); @@ -935,14 +840,9 @@ fn test_fetch_with_sri_network_error() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); // To calulate hash use : // echo -n "alert('Hello, Network Error');" | openssl dgst -sha384 -binary | openssl base64 -A request.integrity_metadata = @@ -964,14 +864,9 @@ fn test_fetch_with_sri_sucess() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); // To calulate hash use : // echo -n "alert('Hello, Network Error');" | openssl dgst -sha384 -binary | openssl base64 -A request.integrity_metadata = @@ -1009,15 +904,10 @@ fn test_fetch_blocked_nosniff() { let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); - request.destination = destination; + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .destination(destination) + .build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -1059,14 +949,9 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); fetch_response @@ -1152,15 +1037,10 @@ fn test_fetch_redirect_updates_method_runner( let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); - request.method = method; + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .method(method) + .build(); let _ = fetch(&mut request, None); let _ = server.close(); @@ -1240,15 +1120,9 @@ fn test_fetch_async_returns_complete_response() { }; let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); - + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -1264,15 +1138,7 @@ fn test_opaque_filtered_fetch_async_returns_complete_response() { let (server, url) = make_server(handler); // an origin mis-match will fall through to an Opaque filtered response - let origin = Origin::Origin(ImmutableOrigin::new_opaque()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); - + let mut request = RequestBuilder::new(url, Referrer::NoReferrer).build(); let fetch_response = fetch(&mut request, None); let _ = server.close(); @@ -1304,16 +1170,10 @@ fn test_opaque_redirect_filtered_fetch_async_returns_complete_response() { }; let (server, url) = make_server(handler); - - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url, - Some(origin), - Referrer::NoReferrer, - None, - HttpsState::None, - ); - request.redirect_mode = RedirectMode::Manual; + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .redirect_mode(RedirectMode::Manual) + .build(); let fetch_response = fetch(&mut request, None); @@ -1333,14 +1193,11 @@ fn test_fetch_with_devtools() { let (server, url) = make_server(handler); - let origin = Origin::Origin(url.origin()); - let mut request = Request::new( - url.clone(), - Some(origin), - Referrer::NoReferrer, - Some(TEST_PIPELINE_ID), - HttpsState::None, - ); + let mut request = RequestBuilder::new(url.clone(), Referrer::NoReferrer) + .origin(url.origin()) + .redirect_mode(RedirectMode::Manual) + .pipeline_id(Some(TEST_PIPELINE_ID)) + .build(); let (devtools_chan, devtools_port) = unbounded(); -- cgit v1.2.3