Refactor FA cookie management and enhance testing

- Updated the FA cookie management in the Scraper::FaHttpClientConfig class to validate cookie formats and handle cookies more robustly.
- Removed the fa.yml configuration file and integrated cookie retrieval directly within the class.
- Added comprehensive tests for cookie handling in the new fa_http_client_config_spec.rb file.
- Updated the Gemfile to include parallel_tests in the test and development group.
- Modified the .rspec file to enable color output for better readability in test results.
- Simplified test commands in the justfile for improved usability.
- Adjusted the rails_helper.rb to ensure the test environment is correctly set up.

.rspec

@@ -1 +1 @@
---require spec_helper
+--color

Gemfile

@@ -89,11 +89,14 @@ group :test do
   gem "webdrivers"
   gem "shoulda-matchers"
   gem "factory_bot_rails"
-  gem "parallel_tests"
   gem "pundit-matchers", "~> 4.0"
   gem "db-query-matchers", "~> 0.14"
 end
 
+group :test, :development do
+  gem "parallel_tests"
+end
+
 gem "xdiff", path: "/gems/xdiff-rb"
 
 # for legacy import

app/lib/scraper/fa_http_client_config.rb

@@ -1,9 +1,31 @@
 class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig
   DEFAULT_ALLOWED_DOMAINS = %w[*.furaffinity.net *.facdn.net ipinfo.io]
+  UUID_PATTERN =
+    /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i
+  OAID_PATTERN = /\A[0-9a-f]{32}\z/i
 
   def cookies
-    Rails.application.config.x.cookies.fa["direct"] ||
+    a_cookie = GlobalState.get("furaffinity-cookie-a")
-      raise("no fa cookies defined for direct")
+    b_cookie = GlobalState.get("furaffinity-cookie-b")
+    oaid_cookie = GlobalState.get("furaffinity-cookie-oaid")
+
+    validate_cookie!("a", a_cookie, UUID_PATTERN)
+    validate_cookie!("b", b_cookie, UUID_PATTERN)
+    validate_cookie!("OAID", oaid_cookie, OAID_PATTERN)
+
+    [
+      {
+        domain: ".furaffinity.net",
+        cookies: [
+          { name: "a", value: a_cookie, path: "/" },
+          { name: "b", value: b_cookie, path: "/" },
+        ],
+      },
+      {
+        domain: "rv.furaffinity.net",
+        cookies: [{ name: "OAID", value: oaid_cookie, path: "/" }],
+      },
+    ]
   end
 
   def ratelimit
@@ -18,4 +40,11 @@ class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig
   def redirect_limit
     4
   end
+
+  private
+
+  def validate_cookie!(name, value, pattern)
+    raise "#{name} cookie is not set" if value.nil? || value.empty?
+    raise "#{name} cookie has invalid format" unless value.match?(pattern)
+  end
 end

config/application.rb

@@ -30,6 +30,5 @@ module ReduxScraper
     config.time_zone = "Pacific Time (US & Canada)"
 
     # config.eager_load_paths << Rails.root.join("extras")
-    config.x.cookies.fa = ReduxScraper::Application.config_for("cookies/fa")
   end
 end

config/cookies/fa.yml

@@ -1,104 +0,0 @@
-ddwhatnow: &ddwhatnow
-  - domain: .furaffinity.net
-    cookies:
-      - name: a
-        value: ebf15f81-6978-4be0-b687-a4c950030d1b
-        path: /
-      - name: b
-        value: 9daf8e3e-1867-44f4-bf38-e248dcfba3dd
-        path: /
-  - domain: rv.furaffinity.net
-    cookies:
-      - name: OAID
-        value: 36dcd165758a3b5ed55565030c41a3b0
-        path: /
-
-vipvillageworker: &vipvillageworker
-  - domain: .furaffinity.net
-    cookies:
-      - name: a
-        value: 64be5a6d-3f9b-4cc5-9a3d-303799d34277
-      - name: b
-        value: c508792f-8694-485c-b077-e6626ceeb63a
-  - domain: rv.furaffinity.net
-    cookies:
-      - name: OAID
-        value: 14ee7a5203d4b59fd7cb3f8513281fcc
-        path: /
-  - domain: www.furaffinity.net
-    cookies:
-      - name: sz
-        value: 1905x976
-        path: /
-
-blazeandwish: &blazeandwish
-  - domain: .furaffinity.net
-    cookies:
-      - name: a
-        value: 343f3180-1ba4-4e20-af45-b0ce39201a78
-      - name: b
-        value: 05784dcd-1fe7-482a-8191-4470e68b776a
-  - domain: rv.furaffinity.net
-    cookies:
-      - name: OAID
-        value: a0196c5a7bdc3fa4865b86dab9be9ce4
-        path: /
-  - domain: www.furaffinity.net
-    cookies:
-      - name: sz
-        value: 1600x944
-        path: /
-
-cottoniq: &cottoniq
-  - domain: .furaffinity.net
-    cookies:
-      - name: a
-        value: dfe43e98-84ad-43b5-8ed5-f03b2c184832
-      - name: b
-        value: ffa1b62d-ab55-4912-b8a1-aae83723afe6
-  - domain: rv.furaffinity.net
-    cookies:
-      - name: OAID
-        value: 0e0618e596550ccd2644525f569db0b8
-        path: /
-  - domain: www.furaffinity.net
-    cookies:
-      - name: sz
-        value: 1680x810
-        path: /
-
-testcookies: &testcookies
-  - domain: .furaffinity.net
-    cookies:
-      - name: a
-        value: avalue
-        path: /
-      - name: b
-        value: bvalue
-        path: /
-  - domain: rv.furaffinity.net
-    cookies:
-      - name: OAID
-        value: oaidvalue
-        path: /
-
-development:
-  direct: *vipvillageworker
-
-production:
-  direct: *ddwhatnow
-  proxy-1: *vipvillageworker
-  dedipath-1: *blazeandwish
-  serverhost-1: *cottoniq
-
-worker:
-  direct: *ddwhatnow
-  proxy-1: *vipvillageworker
-  dedipath-1: *blazeandwish
-  serverhost-1: *cottoniq
-
-test:
-  direct: *testcookies
-  proxy-1: *testcookies
-  dedipath-1: *testcookies
-  serverhost-1: *testcookies

justfile

@@ -17,5 +17,5 @@ psql-dump-domain-fa-favs:
     @psql -P pager=off -c 'select user_id, post_id, 1 from domain_fa_favs limit 10000000;' -d redux_prod -h 10.166.33.171 -U scraper_redux -t -A -F ' '
 
 test:
-    RAILS_ENV=test bin/rails test
+    bin/rails test
-    RAILS_ENV=test bin/rake parallel:spec
+    bin/rake parallel:spec

spec/lib/scraper/fa_http_client_config_spec.rb

@@ -0,0 +1,115 @@
+require "rails_helper"
+
+RSpec.describe Scraper::FaHttpClientConfig do
+  describe "#cookies" do
+    let(:config) { described_class.new }
+
+    context "when all cookies are valid" do
+      before do
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-a",
+        ).and_return("12345678-1234-1234-1234-123456789abc")
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-b",
+        ).and_return("abcdef12-3456-7890-1234-567890abcdef")
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-oaid",
+        ).and_return("0123456789abcdef0123456789abcdef")
+      end
+
+      it "returns the correct cookie structure" do
+        cookies = config.cookies
+        expect(cookies).to match(
+          [
+            {
+              domain: ".furaffinity.net",
+              cookies: [
+                {
+                  name: "a",
+                  value: "12345678-1234-1234-1234-123456789abc",
+                  path: "/",
+                },
+                {
+                  name: "b",
+                  value: "abcdef12-3456-7890-1234-567890abcdef",
+                  path: "/",
+                },
+              ],
+            },
+            {
+              domain: "rv.furaffinity.net",
+              cookies: [
+                {
+                  name: "OAID",
+                  value: "0123456789abcdef0123456789abcdef",
+                  path: "/",
+                },
+              ],
+            },
+          ],
+        )
+      end
+    end
+
+    context "when cookies are missing" do
+      before do
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-a",
+        ).and_return(nil)
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-b",
+        ).and_return("abcdef12-3456-7890-1234-567890abcdef")
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-oaid",
+        ).and_return("0123456789abcdef0123456789abcdef")
+      end
+
+      it "raises an error" do
+        expect { config.cookies }.to raise_error("a cookie is not set")
+      end
+    end
+
+    context "when cookies have invalid format" do
+      before do
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-a",
+        ).and_return("12345678-1234-1234-1234-123456789abc")
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-b",
+        ).and_return("invalid-uuid-format")
+        allow(GlobalState).to receive(:get).with(
+          "furaffinity-cookie-oaid",
+        ).and_return("0123456789abcdef0123456789abcdef")
+      end
+
+      it "raises an error" do
+        expect { config.cookies }.to raise_error("b cookie has invalid format")
+      end
+    end
+  end
+
+  describe "#allowed_domains" do
+    it "returns the default allowed domains" do
+      config = described_class.new
+      expect(config.allowed_domains).to eq(
+        %w[*.furaffinity.net *.facdn.net ipinfo.io],
+      )
+    end
+  end
+
+  describe "#redirect_limit" do
+    it "returns the configured redirect limit" do
+      config = described_class.new
+      expect(config.redirect_limit).to eq(4)
+    end
+  end
+
+  describe "#ratelimit" do
+    it "returns the configured rate limits" do
+      config = described_class.new
+      expect(config.ratelimit).to eq(
+        [["d.furaffinity.net", :none], ["*.facdn.net", :none], ["*", 1]],
+      )
+    end
+  end
+end

spec/rails_helper.rb

@@ -1,13 +1,11 @@
 # This file is copied to spec/ when you run 'rails generate rspec:install'
-ENV["RAILS_ENV"] ||= "test"
+ENV["RAILS_ENV"] = "test"
 require_relative "../config/environment"
 require "spec_helper"
 require "rspec/rails"
-require "pundit/matchers"
+
-require "db-query-matchers"
+unless Rails.env.test?
-# Prevent database truncation if the environment is production
+  abort("The Rails environment is not running in test mode!")
-if Rails.env.production?
-  abort("The Rails environment is running in production mode!")
 end
 # Add additional requires below this line. Rails is not loaded until this point!