fix tor total timeout
This commit is contained in:
@@ -65,7 +65,6 @@ class Scraper::CurlHttpPerformer
|
||||
curl = get_curl
|
||||
start_at = Time.now
|
||||
|
||||
curl.timeout = 30
|
||||
curl.url = request.uri.normalize.to_s
|
||||
curl.follow_location = request.follow_redirects
|
||||
request.request_headers.each { |key, value| curl.headers[key.to_s] = value }
|
||||
@@ -133,6 +132,8 @@ class Scraper::CurlHttpPerformer
|
||||
end
|
||||
curl = t.thread_variable_get(:curl)
|
||||
proxy_url = ENV["HTTP_PROXY_URL"]
|
||||
curl.timeout = 30
|
||||
curl.connect_timeout = 2
|
||||
curl.proxy_url = proxy_url
|
||||
curl.headers = {}
|
||||
curl
|
||||
|
||||
@@ -17,11 +17,9 @@ class Scraper::TorCurlHttpPerformer < Scraper::CurlHttpPerformer
|
||||
end
|
||||
curl = T.cast(t.thread_variable_get(:curl), Curl::Easy)
|
||||
curl.proxy_url = "socks5h://tor:9050"
|
||||
curl.timeout_ms = 60_000
|
||||
curl.connect_timeout_ms = 10_000
|
||||
curl.timeout = 120
|
||||
curl.connect_timeout = 10
|
||||
curl.headers = {
|
||||
"User-Agent" =>
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0",
|
||||
"Accept-Encoding" => "gzip, deflate",
|
||||
"Connection" => "keep-alive",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user