From 9e8f2651db517ddc0f16f497247d02263ea66cf4 Mon Sep 17 00:00:00 2001 From: Dylan Knutson Date: Mon, 30 Dec 2024 02:00:30 +0000 Subject: [PATCH] Refactor FA cookie management and enhance testing - Updated the FA cookie management in the Scraper::FaHttpClientConfig class to validate cookie formats and handle cookies more robustly. - Removed the fa.yml configuration file and integrated cookie retrieval directly within the class. - Added comprehensive tests for cookie handling in the new fa_http_client_config_spec.rb file. - Updated the Gemfile to include parallel_tests in the test and development group. - Modified the .rspec file to enable color output for better readability in test results. - Simplified test commands in the justfile for improved usability. - Adjusted the rails_helper.rb to ensure the test environment is correctly set up. --- .rspec | 2 +- Gemfile | 5 +- app/lib/scraper/fa_http_client_config.rb | 33 ++++- config/application.rb | 1 - config/cookies/fa.yml | 104 ---------------- justfile | 4 +- .../lib/scraper/fa_http_client_config_spec.rb | 115 ++++++++++++++++++ spec/rails_helper.rb | 10 +- 8 files changed, 157 insertions(+), 117 deletions(-) delete mode 100644 config/cookies/fa.yml create mode 100644 spec/lib/scraper/fa_http_client_config_spec.rb diff --git a/.rspec b/.rspec index c99d2e73..4e1e0d2f 100644 --- a/.rspec +++ b/.rspec @@ -1 +1 @@ ---require spec_helper +--color diff --git a/Gemfile b/Gemfile index 9cb3fbde..b274b017 100644 --- a/Gemfile +++ b/Gemfile @@ -89,11 +89,14 @@ group :test do gem "webdrivers" gem "shoulda-matchers" gem "factory_bot_rails" - gem "parallel_tests" gem "pundit-matchers", "~> 4.0" gem "db-query-matchers", "~> 0.14" end +group :test, :development do + gem "parallel_tests" +end + gem "xdiff", path: "/gems/xdiff-rb" # for legacy import diff --git a/app/lib/scraper/fa_http_client_config.rb b/app/lib/scraper/fa_http_client_config.rb index 7f06bbe6..9b755eef 100644 --- a/app/lib/scraper/fa_http_client_config.rb +++ b/app/lib/scraper/fa_http_client_config.rb @@ -1,9 +1,31 @@ class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig DEFAULT_ALLOWED_DOMAINS = %w[*.furaffinity.net *.facdn.net ipinfo.io] + UUID_PATTERN = + /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i + OAID_PATTERN = /\A[0-9a-f]{32}\z/i def cookies - Rails.application.config.x.cookies.fa["direct"] || - raise("no fa cookies defined for direct") + a_cookie = GlobalState.get("furaffinity-cookie-a") + b_cookie = GlobalState.get("furaffinity-cookie-b") + oaid_cookie = GlobalState.get("furaffinity-cookie-oaid") + + validate_cookie!("a", a_cookie, UUID_PATTERN) + validate_cookie!("b", b_cookie, UUID_PATTERN) + validate_cookie!("OAID", oaid_cookie, OAID_PATTERN) + + [ + { + domain: ".furaffinity.net", + cookies: [ + { name: "a", value: a_cookie, path: "/" }, + { name: "b", value: b_cookie, path: "/" }, + ], + }, + { + domain: "rv.furaffinity.net", + cookies: [{ name: "OAID", value: oaid_cookie, path: "/" }], + }, + ] end def ratelimit @@ -18,4 +40,11 @@ class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig def redirect_limit 4 end + + private + + def validate_cookie!(name, value, pattern) + raise "#{name} cookie is not set" if value.nil? || value.empty? + raise "#{name} cookie has invalid format" unless value.match?(pattern) + end end diff --git a/config/application.rb b/config/application.rb index 84a86e20..76a1f30d 100644 --- a/config/application.rb +++ b/config/application.rb @@ -30,6 +30,5 @@ module ReduxScraper config.time_zone = "Pacific Time (US & Canada)" # config.eager_load_paths << Rails.root.join("extras") - config.x.cookies.fa = ReduxScraper::Application.config_for("cookies/fa") end end diff --git a/config/cookies/fa.yml b/config/cookies/fa.yml deleted file mode 100644 index 86d4ae1e..00000000 --- a/config/cookies/fa.yml +++ /dev/null @@ -1,104 +0,0 @@ -ddwhatnow: &ddwhatnow - - domain: .furaffinity.net - cookies: - - name: a - value: ebf15f81-6978-4be0-b687-a4c950030d1b - path: / - - name: b - value: 9daf8e3e-1867-44f4-bf38-e248dcfba3dd - path: / - - domain: rv.furaffinity.net - cookies: - - name: OAID - value: 36dcd165758a3b5ed55565030c41a3b0 - path: / - -vipvillageworker: &vipvillageworker - - domain: .furaffinity.net - cookies: - - name: a - value: 64be5a6d-3f9b-4cc5-9a3d-303799d34277 - - name: b - value: c508792f-8694-485c-b077-e6626ceeb63a - - domain: rv.furaffinity.net - cookies: - - name: OAID - value: 14ee7a5203d4b59fd7cb3f8513281fcc - path: / - - domain: www.furaffinity.net - cookies: - - name: sz - value: 1905x976 - path: / - -blazeandwish: &blazeandwish - - domain: .furaffinity.net - cookies: - - name: a - value: 343f3180-1ba4-4e20-af45-b0ce39201a78 - - name: b - value: 05784dcd-1fe7-482a-8191-4470e68b776a - - domain: rv.furaffinity.net - cookies: - - name: OAID - value: a0196c5a7bdc3fa4865b86dab9be9ce4 - path: / - - domain: www.furaffinity.net - cookies: - - name: sz - value: 1600x944 - path: / - -cottoniq: &cottoniq - - domain: .furaffinity.net - cookies: - - name: a - value: dfe43e98-84ad-43b5-8ed5-f03b2c184832 - - name: b - value: ffa1b62d-ab55-4912-b8a1-aae83723afe6 - - domain: rv.furaffinity.net - cookies: - - name: OAID - value: 0e0618e596550ccd2644525f569db0b8 - path: / - - domain: www.furaffinity.net - cookies: - - name: sz - value: 1680x810 - path: / - -testcookies: &testcookies - - domain: .furaffinity.net - cookies: - - name: a - value: avalue - path: / - - name: b - value: bvalue - path: / - - domain: rv.furaffinity.net - cookies: - - name: OAID - value: oaidvalue - path: / - -development: - direct: *vipvillageworker - -production: - direct: *ddwhatnow - proxy-1: *vipvillageworker - dedipath-1: *blazeandwish - serverhost-1: *cottoniq - -worker: - direct: *ddwhatnow - proxy-1: *vipvillageworker - dedipath-1: *blazeandwish - serverhost-1: *cottoniq - -test: - direct: *testcookies - proxy-1: *testcookies - dedipath-1: *testcookies - serverhost-1: *testcookies diff --git a/justfile b/justfile index 519c249c..641cf570 100644 --- a/justfile +++ b/justfile @@ -17,5 +17,5 @@ psql-dump-domain-fa-favs: @psql -P pager=off -c 'select user_id, post_id, 1 from domain_fa_favs limit 10000000;' -d redux_prod -h 10.166.33.171 -U scraper_redux -t -A -F ' ' test: - RAILS_ENV=test bin/rails test - RAILS_ENV=test bin/rake parallel:spec + bin/rails test + bin/rake parallel:spec diff --git a/spec/lib/scraper/fa_http_client_config_spec.rb b/spec/lib/scraper/fa_http_client_config_spec.rb new file mode 100644 index 00000000..056d0a63 --- /dev/null +++ b/spec/lib/scraper/fa_http_client_config_spec.rb @@ -0,0 +1,115 @@ +require "rails_helper" + +RSpec.describe Scraper::FaHttpClientConfig do + describe "#cookies" do + let(:config) { described_class.new } + + context "when all cookies are valid" do + before do + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-a", + ).and_return("12345678-1234-1234-1234-123456789abc") + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-b", + ).and_return("abcdef12-3456-7890-1234-567890abcdef") + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-oaid", + ).and_return("0123456789abcdef0123456789abcdef") + end + + it "returns the correct cookie structure" do + cookies = config.cookies + expect(cookies).to match( + [ + { + domain: ".furaffinity.net", + cookies: [ + { + name: "a", + value: "12345678-1234-1234-1234-123456789abc", + path: "/", + }, + { + name: "b", + value: "abcdef12-3456-7890-1234-567890abcdef", + path: "/", + }, + ], + }, + { + domain: "rv.furaffinity.net", + cookies: [ + { + name: "OAID", + value: "0123456789abcdef0123456789abcdef", + path: "/", + }, + ], + }, + ], + ) + end + end + + context "when cookies are missing" do + before do + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-a", + ).and_return(nil) + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-b", + ).and_return("abcdef12-3456-7890-1234-567890abcdef") + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-oaid", + ).and_return("0123456789abcdef0123456789abcdef") + end + + it "raises an error" do + expect { config.cookies }.to raise_error("a cookie is not set") + end + end + + context "when cookies have invalid format" do + before do + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-a", + ).and_return("12345678-1234-1234-1234-123456789abc") + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-b", + ).and_return("invalid-uuid-format") + allow(GlobalState).to receive(:get).with( + "furaffinity-cookie-oaid", + ).and_return("0123456789abcdef0123456789abcdef") + end + + it "raises an error" do + expect { config.cookies }.to raise_error("b cookie has invalid format") + end + end + end + + describe "#allowed_domains" do + it "returns the default allowed domains" do + config = described_class.new + expect(config.allowed_domains).to eq( + %w[*.furaffinity.net *.facdn.net ipinfo.io], + ) + end + end + + describe "#redirect_limit" do + it "returns the configured redirect limit" do + config = described_class.new + expect(config.redirect_limit).to eq(4) + end + end + + describe "#ratelimit" do + it "returns the configured rate limits" do + config = described_class.new + expect(config.ratelimit).to eq( + [["d.furaffinity.net", :none], ["*.facdn.net", :none], ["*", 1]], + ) + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 3279fc9a..6784c790 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -1,13 +1,11 @@ # This file is copied to spec/ when you run 'rails generate rspec:install' -ENV["RAILS_ENV"] ||= "test" +ENV["RAILS_ENV"] = "test" require_relative "../config/environment" require "spec_helper" require "rspec/rails" -require "pundit/matchers" -require "db-query-matchers" -# Prevent database truncation if the environment is production -if Rails.env.production? - abort("The Rails environment is running in production mode!") + +unless Rails.env.test? + abort("The Rails environment is not running in test mode!") end # Add additional requires below this line. Rails is not loaded until this point!