fix tor total timeout

This commit is contained in:
Dylan Knutson
2025-07-27 17:05:43 +00:00
parent bc4143ae12
commit 53a9498386
2 changed files with 4 additions and 5 deletions

View File

@@ -65,7 +65,6 @@ class Scraper::CurlHttpPerformer
curl = get_curl
start_at = Time.now
curl.timeout = 30
curl.url = request.uri.normalize.to_s
curl.follow_location = request.follow_redirects
request.request_headers.each { |key, value| curl.headers[key.to_s] = value }
@@ -133,6 +132,8 @@ class Scraper::CurlHttpPerformer
end
curl = t.thread_variable_get(:curl)
proxy_url = ENV["HTTP_PROXY_URL"]
curl.timeout = 30
curl.connect_timeout = 2
curl.proxy_url = proxy_url
curl.headers = {}
curl

View File

@@ -17,11 +17,9 @@ class Scraper::TorCurlHttpPerformer < Scraper::CurlHttpPerformer
end
curl = T.cast(t.thread_variable_get(:curl), Curl::Easy)
curl.proxy_url = "socks5h://tor:9050"
curl.timeout_ms = 60_000
curl.connect_timeout_ms = 10_000
curl.timeout = 120
curl.connect_timeout = 10
curl.headers = {
"User-Agent" =>
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0",
"Accept-Encoding" => "gzip, deflate",
"Connection" => "keep-alive",
}