9e8f2651db
- Updated the FA cookie management in the Scraper::FaHttpClientConfig class to validate cookie formats and handle cookies more robustly. - Removed the fa.yml configuration file and integrated cookie retrieval directly within the class. - Added comprehensive tests for cookie handling in the new fa_http_client_config_spec.rb file. - Updated the Gemfile to include parallel_tests in the test and development group. - Modified the .rspec file to enable color output for better readability in test results. - Simplified test commands in the justfile for improved usability. - Adjusted the rails_helper.rb to ensure the test environment is correctly set up.
51 lines
1.4 KiB
Ruby
51 lines
1.4 KiB
Ruby
class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig
|
|
DEFAULT_ALLOWED_DOMAINS = %w[*.furaffinity.net *.facdn.net ipinfo.io]
|
|
UUID_PATTERN =
|
|
/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i
|
|
OAID_PATTERN = /\A[0-9a-f]{32}\z/i
|
|
|
|
def cookies
|
|
a_cookie = GlobalState.get("furaffinity-cookie-a")
|
|
b_cookie = GlobalState.get("furaffinity-cookie-b")
|
|
oaid_cookie = GlobalState.get("furaffinity-cookie-oaid")
|
|
|
|
validate_cookie!("a", a_cookie, UUID_PATTERN)
|
|
validate_cookie!("b", b_cookie, UUID_PATTERN)
|
|
validate_cookie!("OAID", oaid_cookie, OAID_PATTERN)
|
|
|
|
[
|
|
{
|
|
domain: ".furaffinity.net",
|
|
cookies: [
|
|
{ name: "a", value: a_cookie, path: "/" },
|
|
{ name: "b", value: b_cookie, path: "/" },
|
|
],
|
|
},
|
|
{
|
|
domain: "rv.furaffinity.net",
|
|
cookies: [{ name: "OAID", value: oaid_cookie, path: "/" }],
|
|
},
|
|
]
|
|
end
|
|
|
|
def ratelimit
|
|
# number represents minimum delay in seconds between requests to the same domain
|
|
[["d.furaffinity.net", :none], ["*.facdn.net", :none], ["*", 1]]
|
|
end
|
|
|
|
def allowed_domains
|
|
DEFAULT_ALLOWED_DOMAINS
|
|
end
|
|
|
|
def redirect_limit
|
|
4
|
|
end
|
|
|
|
private
|
|
|
|
def validate_cookie!(name, value, pattern)
|
|
raise "#{name} cookie is not set" if value.nil? || value.empty?
|
|
raise "#{name} cookie has invalid format" unless value.match?(pattern)
|
|
end
|
|
end
|