107 lines
2.7 KiB
Ruby
107 lines
2.7 KiB
Ruby
class Scraper::CurlHttpPerformer
|
|
Response =
|
|
Struct.new(:response_code, :response_headers, :response_time_ms, :body)
|
|
|
|
def get(url, request_headers)
|
|
do_request(:http_get, url, request_headers)
|
|
end
|
|
|
|
def post(url, request_headers)
|
|
do_request(:http_post, url, request_headers)
|
|
end
|
|
|
|
def name
|
|
"direct"
|
|
end
|
|
|
|
METHOD_MAP = { http_get: "GET", http_post: "POST" }
|
|
|
|
def do_request(method, url, request_headers)
|
|
t, curl = get_curl
|
|
start_at = Time.now
|
|
curl.url = Addressable::URI.encode url
|
|
curl.follow_location = true
|
|
request_headers.each { |key, value| curl.headers[key.to_s] = value }
|
|
curl.headers["User-Agent"] = "FurryArchiver/1.0 / dhelta"
|
|
curl.send(method)
|
|
response_time_ms = ((Time.now - start_at) * 1000).to_i
|
|
response_code = curl.response_code
|
|
body_str = curl.body_str
|
|
response_headers = self.class.parse_header_str(curl.header_str).to_h
|
|
|
|
rc_color =
|
|
case response_code / 100
|
|
when 2
|
|
[:green]
|
|
when 3
|
|
[:cyan]
|
|
when 4
|
|
[:red]
|
|
when 5
|
|
%i[red bold]
|
|
else
|
|
[:white]
|
|
end
|
|
|
|
rc_string = response_code.to_s
|
|
rc_color.each { |m| rc_string = rc_string.send(m) }
|
|
|
|
cf_cache_status =
|
|
if response_headers["cf-cache-status"]
|
|
"(#{response_headers["cf-cache-status"].light_blue})"
|
|
else
|
|
nil
|
|
end
|
|
|
|
puts [
|
|
"[#{Process.pid.to_s.black.bold} / #{t.name.to_s.black.bold}]",
|
|
"[#{METHOD_MAP[method]} #{rc_string}",
|
|
"#{self.class.humansize(body_str.size).to_s.bold}",
|
|
(response_time_ms.to_s + "ms").light_blue.bold + "]",
|
|
cf_cache_status,
|
|
"#{url.bold}"
|
|
].reject(&:nil?).join(" ")
|
|
|
|
Response.new(response_code, response_headers, response_time_ms, body_str)
|
|
end
|
|
|
|
private
|
|
|
|
def get_curl
|
|
t = Thread.current
|
|
unless t.thread_variable?(:curl)
|
|
t.thread_variable_set(:curl, Curl::Easy.new)
|
|
end
|
|
curl = t.thread_variable_get(:curl)
|
|
curl.headers = {}
|
|
[t, curl]
|
|
end
|
|
|
|
def self.humansize(size)
|
|
units = %w[B KiB MiB GiB TiB Pib EiB ZiB]
|
|
return "0.0 B" if size == 0
|
|
exp = (Math.log(size) / Math.log(1024)).to_i
|
|
exp += 1 if (size.to_f / 1024**exp >= 1024 - 0.05)
|
|
exp = units.size - 1 if exp > units.size - 1
|
|
unit = units[exp]
|
|
if unit == "B"
|
|
"%d B" % [size]
|
|
else
|
|
"%.1f %s" % [size.to_f / 1024**exp, unit]
|
|
end
|
|
end
|
|
|
|
def self.parse_header_str(header_str)
|
|
header_str
|
|
.split("\r\n")
|
|
.each_with_index
|
|
.map do |str, idx|
|
|
next nil if idx == 0
|
|
idx = str.index(": ")
|
|
next nil unless idx
|
|
[str[0...idx], str[idx + 2..-1]]
|
|
end
|
|
.reject(&:nil?)
|
|
end
|
|
end
|