dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor tests and enhance functionality for Domain::Fa and Domain::E621

Browse Source 
- Removed outdated tests for Domain::Fa::PostsController and Domain::Fa::UsersController.
- Added new tests for Domain::Fa::PostsController to verify successful responses for the show action.
- Introduced a new UsersController spec to test the show action for user retrieval.
- Created a factory for Domain::E621::Post to streamline test data creation.
- Added comprehensive tests for BlobFile model, ensuring correct functionality and file operations.
- Implemented tests for HttpLogEntryHeader and LogStoreSstEntry models to validate header scrubbing and version parsing.
- Deleted obsolete test files and directories to clean up the test suite.
This commit is contained in:
Dylan Knutson
2024-12-30 20:11:06 +00:00
parent 15c11b2b89
commit 52498b3cc2
27 changed files with 196 additions and 418 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
justfile
View File

@@ -17,5 +17,4 @@ psql-dump-domain-fa-favs:
@psql -P pager=off -c 'select user_id, post_id, 1 from domain_fa_favs limit 10000000;' -d redux_prod -h 10.166.33.171 -U scraper_redux -t -A -F ' '
test:
bin/rails test
bin/rake parallel:spec

25
spec/controllers/domain/fa/posts_controller_spec.rb
View File

@@ -2,7 +2,19 @@ require "rails_helper"
RSpec.describe Domain::Fa::PostsController, type: :controller do
render_views
include Domain::Fa::UsersHelper
describe "GET #show" do
let(:post) { create(:domain_fa_post, :with_creator) }
it "returns a successful response" do
get :show,
params: {
fa_id: post.fa_id,
user_url_name: post.creator.url_name,
}
expect(response).to be_successful
end
end
describe "GET #index" do
let(:user) { create(:domain_fa_user, :with_avatar) }
@@ -22,7 +34,6 @@ RSpec.describe Domain::Fa::PostsController, type: :controller do
it "renders index template with posts" do
get :index, params: { user_url_name: user.url_name }
expect(response).to be_successful
expect(response).to render_template(:index)
expect(response.body).to include(user.name)
expect(response.body).to include("posts")
posts.each do |post|
@@ -56,14 +67,4 @@ RSpec.describe Domain::Fa::PostsController, type: :controller do
end
end
end
describe "GET #show" do
let(:user) { create(:domain_fa_user) }
let(:post) { create(:domain_fa_post, creator: user) }
it "returns http success" do
get :show, params: { user_url_name: user.url_name, fa_id: post.fa_id }
expect(response).to be_successful
end
end
end

14
spec/controllers/domain/fa/users_controller_spec.rb Normal file
View File

@@ -0,0 +1,14 @@
require "rails_helper"
RSpec.describe Domain::Fa::UsersController, type: :controller do
render_views
describe "GET #show" do
let(:user) { create(:domain_fa_user, registered_at: 1.year.ago) }
it "returns a successful response" do
get :show, params: { url_name: user.url_name }
expect(response).to be_successful
end
end
end

12
spec/factories/domain/e621/posts.rb Normal file
View File

@@ -0,0 +1,12 @@
FactoryBot.define do
factory :domain_e621_post, class: "Domain::E621::Post" do
sequence(:e621_id) { |n| n }
state { :ok }
state_detail { {} }
flags_array { [] }
pools_array { [] }
sources_array { [] }
artists_array { [] }
tags_array { {} }
end
end

84
spec/models/blob_file_spec.rb Normal file
View File

@@ -0,0 +1,84 @@
require "rails_helper"
RSpec.describe BlobFile, type: :model do
before(:all) do
# safeguard against running this test in a non-test environment
root_dir =
File.absolute_path(Rails.application.config_for("blob_file_location"))
expect(root_dir).to match(%r{^#{Rails.root}/tmp})
FileUtils.rm_rf(root_dir)
end
describe "basic functionality" do
it "can be built and saved" do
content_bytes = SecureRandom.alphanumeric(1024)
blob_file =
BlobFile.new(content_bytes: content_bytes, content_type: "text")
expect(blob_file).to be_valid
expect(blob_file.size_bytes).to eq(1024)
expect(blob_file.content_bytes).to eq(content_bytes)
expect(blob_file.content_type).to eq("text")
expect(blob_file.content_bytes.encoding).to eq(Encoding::ASCII_8BIT)
expect(blob_file.save).to be true
end
it "counts bytes with unicode characters" do
content_bytes = "fooばr"
blob_file =
BlobFile.new(content_bytes: content_bytes, content_type: "text")
expect(blob_file).to be_valid
expect(blob_file.size_bytes).to eq(7)
expect(blob_file.save).to be true
sha256 = blob_file.sha256
blob_file = BlobFile.find(sha256)
expect(blob_file.content_bytes).to eq(content_bytes)
end
end
describe "file operations" do
it "puts the file in the right place" do
content_bytes = SecureRandom.alphanumeric(1024)
blob_file =
BlobFile.new(content_bytes: content_bytes, content_type: "text")
expect(blob_file.save).to be true
expect(File.exist?(blob_file.absolute_file_path)).to be true
expect(File.binread(blob_file.absolute_file_path)).to eq(content_bytes)
end
it "handles saving when file already exists" do
content_bytes = SecureRandom.alphanumeric(1024)
blob_file =
BlobFile.new(content_bytes: content_bytes, content_type: "text")
FileUtils.mkdir_p(File.dirname(blob_file.absolute_file_path))
File.binwrite(blob_file.absolute_file_path, content_bytes)
expect(blob_file.save).to be true
expect(blob_file.content_bytes).to eq(content_bytes)
end
end
describe "path segmentation" do
it "segments paths correctly" do
test_cases = [
["abcd1234", [2], %w[ab abcd1234]],
["abcd1234", [4, 2], %w[abcd 12 abcd1234]],
["abcd1234", [4, 2, 2], %w[abcd 12 34 abcd1234]],
["abcd1234", [2, 2, 1], %w[ab cd 1 abcd1234]],
]
test_cases.each do |sha256_hex, pattern, expected|
expect(BlobFile.path_segments(pattern, sha256_hex)).to eq(expected)
end
end
end
describe "blob entry integration" do
it "can be initialized from a BlobEntry" do
blob_entry = create(:blob_entry)
blob_file = BlobFile.initialize_from_blob_entry(blob_entry)
expect(blob_file.save).to be true
expect(blob_file.content_bytes).to eq(blob_entry.contents)
expect(blob_file.content_type).to eq(blob_entry.content_type)
end
end
end

54
spec/models/domain/e621/post_spec.rb
View File

@@ -1,13 +1,49 @@
require "rails_helper"
describe Domain::E621::Post do
it "ensures indexed_post is created" do
post = SpecUtil.build_e621_post(created_at: 1.day.ago)
post.save!
expect(post.indexed_post).to be_present
expect(post.indexed_post.created_at).to eq(post.created_at)
expect(post.indexed_post.postable_id).to eq(post.id)
expect(post.indexed_post.postable_type).to eq("Domain::E621::Post")
expect(post.indexed_post.postable).to eq(post)
RSpec.describe Domain::E621::Post, type: :model do
describe "versioning" do
it "handles lite trail versions correctly" do
# Verify the generated model exists
expect(LiteTrail::PerTable::DomainE621PostVersions).not_to be_nil
post = described_class.new(e621_id: 12_345)
expect(post).to be_valid
expect(post.save).to be true
expect(post.versions.length).to eq(0)
post.reload
expect(post.versions.length).to eq(0)
post.rating = "s"
expect(post.save).to be true
# Check version after save
expect(post.versions.length).to eq(1)
updated_rating_version = post.versions.last
expect(updated_rating_version.event).to eq("update")
expect(updated_rating_version.created_at).to eq(post.updated_at)
# Check version after reload
post.reload
expect(post.versions.length).to eq(1)
updated_rating_version = post.versions.last
expect(updated_rating_version.event).to eq("update")
expect(updated_rating_version.created_at).to eq(post.updated_at)
# Check destroy
post.destroy
expect(post).to be_destroyed
end
end
describe "indexed post" do
it "ensures indexed_post is created" do
post = create(:domain_e621_post, created_at: 1.day.ago)
expect(post.indexed_post).to be_present
expect(post.indexed_post.created_at).to eq(post.created_at)
expect(post.indexed_post.postable_id).to eq(post.id)
expect(post.indexed_post.postable_type).to eq("Domain::E621::Post")
expect(post.indexed_post.postable).to eq(post)
end
end
end

17
spec/models/http_log_entry_header_spec.rb Normal file
View File

@@ -0,0 +1,17 @@
require "rails_helper"
RSpec.describe HttpLogEntryHeader, type: :model do
describe "header scrubbing" do
it "scrubs blacklisted headers" do
header =
HttpLogEntryHeader.build(
headers: {
"foo" => "bar",
"date" => "Tue, 15 Nov 1994 08:12:31 GMT",
},
)
expect(header.headers["foo"]).to eq("bar")
expect(header.headers["date"]).to eq("Tue, 15 Nov 1994 (scrubbed) GMT")
end
end
end

11
spec/models/log_store_sst_entry_spec.rb Normal file
View File

@@ -0,0 +1,11 @@
require "rails_helper"
RSpec.describe LogStoreSstEntry, type: :model do
describe "version string parsing" do
it "correctly parses version and flags" do
entry = LogStoreSstEntry.new(value: "\x00\x00\x00\x01\x00\x00\x00\x02")
expect(entry.value_version).to eq(1)
expect(entry.value_flags).to eq(2)
end
end
end

5
test/application_system_test_case.rb
View File

@@ -1,5 +0,0 @@
require "test_helper"
class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
driven_by :selenium, using: :chrome, screen_size: [1400, 1400]
end

11
test/channels/application_cable/connection_test.rb
View File

@@ -1,11 +0,0 @@
require "test_helper"
class ApplicationCable::ConnectionTest < ActionCable::Connection::TestCase
# test "connects with cookies" do
# cookies.signed[:user_id] = 42
#
# connect
#
# assert_equal connection.user_id, "42"
# end
end

0
test/controllers/.keep
View File

13
test/controllers/domain/fa/posts_controller_test.rb
View File

@@ -1,13 +0,0 @@
require "test_helper"
class Domain::Fa::PostsControllerTest < ActionDispatch::IntegrationTest
setup do
@post = TestUtil.build_fa_post
@post.save!
end
test "should show domain_fa_post" do
get domain_fa_post_url(fa_id: @post.fa_id)
assert_response :success
end
end

14
test/controllers/domain/fa/users_controller_test.rb
View File

@@ -1,14 +0,0 @@
require "test_helper"
class Domain::Fa::UsersControllerTest < ActionDispatch::IntegrationTest
setup do
@user = TestUtil.build_fa_user
@user.registered_at = 1.year.ago
@user.save!
end
test "should show user" do
get domain_fa_user_url(@user)
assert_response :success
end
end

7
test/controllers/indexable_posts_controller_test.rb
View File

@@ -1,7 +0,0 @@
require "test_helper"
class IndexablePostsControllerTest < ActionDispatch::IntegrationTest
# test "the truth" do
# assert true
# end
end

7
test/controllers/log_entries_controller_test.rb
View File

@@ -1,7 +0,0 @@
require "test_helper"
class LogEntriesControllerTest < ActionDispatch::IntegrationTest
# test "the truth" do
# assert true
# end
end

0
test/helpers/.keep
View File

0
test/integration/.keep
View File

0
test/mailers/.keep
View File

0
test/models/.keep
View File

48
test/models/blob_entry_test.rb
View File

@@ -1,48 +0,0 @@
require "test_helper"
class BlobEntryTest < ActiveSupport::TestCase
test "building a blob works" do
blob = TestUtil.build_blob_entry
assert blob.valid?
assert blob.save
end
test "model cannot be updated" do
model = TestUtil.build_blob_entry
model.save!
model.contents = "new contenets"
assert_raises(ActiveRecord::ReadOnlyRecord) { model.save! }
end
test "model cannot be deleted" do
model = TestUtil.build_blob_entry
model.save!
assert_raises(ActiveRecord::ReadOnlyRecord) { model.destroy }
end
test "model dual-writes a BlobEntry model" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntry.find_by(sha256: model.sha256)
assert_same_blob_entry model, model_p
end
test "ensure works for creating a blob entry" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntry.ensure(model.sha256)
assert_same_blob_entry model, model_p
end
def assert_same_blob_entry(model, model_p)
%i[sha256 base_sha256 contents size created_at content_type].each do |attr|
expected = model.send(attr)
actual = model_p.send(attr)
if expected.nil?
assert_nil actual, "#{attr} mismatch"
else
assert_equal expected, actual, "#{attr} mismatch"
end
end
end
end

70
test/models/blob_file_test.rb
View File

@@ -1,70 +0,0 @@
class BlobFileTest < ActiveSupport::TestCase
setup do
# safeguard against running this test in a non-test environment
root_dir =
File.absolute_path(Rails.application.config_for("blob_file_location"))
assert_match %r{^#{Rails.root}/tmp}, root_dir
FileUtils.rm_rf(root_dir)
end
test "building a blob file works" do
content_bytes = TestUtil.random_string(1024)
blob_file = BlobFile.new(content_bytes: content_bytes, content_type: "text")
assert blob_file.valid?
assert_equal blob_file.size_bytes, 1024
assert_equal blob_file.content_bytes, content_bytes
assert_equal blob_file.content_type, "text"
assert_equal blob_file.content_bytes.encoding, Encoding::ASCII_8BIT
assert blob_file.save
end
test "counts bytes with unicode characters" do
content_bytes = "fooばr"
blob_file = BlobFile.new(content_bytes: content_bytes, content_type: "text")
assert blob_file.valid?
assert_equal blob_file.size_bytes, 7
assert blob_file.save
sha256 = blob_file.sha256
blob_file = BlobFile.find(sha256)
assert_equal blob_file.content_bytes, content_bytes
end
test "puts the file in the right place" do
content_bytes = TestUtil.random_string(1024)
blob_file = BlobFile.new(content_bytes: content_bytes, content_type: "text")
assert blob_file.save
assert File.exist?(blob_file.absolute_file_path)
assert_equal File.binread(blob_file.absolute_file_path), content_bytes
end
test "saving blob when the file already exists works" do
content_bytes = TestUtil.random_string(1024)
blob_file = BlobFile.new(content_bytes: content_bytes, content_type: "text")
FileUtils.mkdir_p(File.dirname(blob_file.absolute_file_path))
File.binwrite(blob_file.absolute_file_path, content_bytes)
assert blob_file.save
assert_equal blob_file.content_bytes, content_bytes
end
test "file path segmentation works" do
test_cases = [
["abcd1234", [2], %w[ab abcd1234]],
["abcd1234", [4, 2], %w[abcd 12 abcd1234]],
["abcd1234", [4, 2, 2], %w[abcd 12 34 abcd1234]],
["abcd1234", [2, 2, 1], %w[ab cd 1 abcd1234]],
]
test_cases.each do |sha256_hex, pattern, expected|
assert_equal BlobFile.path_segments(pattern, sha256_hex), expected
end
end
test "from an initialized BlobEntry" do
blob_entry = TestUtil.build_blob_entry
blob_file = BlobFile.initialize_from_blob_entry(blob_entry)
assert blob_file.save
assert_equal blob_file.content_bytes, blob_entry.contents
assert_equal blob_file.content_type, blob_entry.content_type
end
end

34
test/models/domain/e621/post_test.rb
View File

@@ -1,34 +0,0 @@
class Domain::E621::PostTest < ActiveSupport::TestCase
Subject = Domain::E621::Post
test "lite trail versions works" do
# the generated model should exist
refute_nil LiteTrail::PerTable::DomainE621PostVersions
post = Subject.new({ e621_id: 12_345 })
assert post.valid?, post.errors.full_messages
assert post.save
assert_equal 0, post.versions.length
post.reload
assert_equal 0, post.versions.length
check_update =
proc do
assert_equal 1, post.versions.length
updated_rating_version = post.versions.last
assert_equal "update", updated_rating_version.event
assert_equal post.updated_at, updated_rating_version.created_at
end
post.rating = "s"
assert post.save
check_update.call
post.reload
check_update.call
post.destroy
assert post.destroyed?
end
end

15
test/models/http_log_entry_header_test.rb
View File

@@ -1,15 +0,0 @@
require "test_helper"
class HttpLogEntryHeaderTest < ActiveSupport::TestCase
test "blacklisted headers are scrubbed" do
header =
HttpLogEntryHeader.build(
headers: {
"foo" => "bar",
"date" => "Tue, 15 Nov 1994 08:12:31 GMT"
}
)
assert_equal "bar", header.headers["foo"]
assert_equal "Tue, 15 Nov 1994 (scrubbed) GMT", header.headers["date"]
end
end

9
test/models/log_store_sst_entry_test.rb
View File

@@ -1,9 +0,0 @@
require "test_helper"
class LogStoreSstEntryTest < ActiveSupport::TestCase
test "version string works" do
entry = LogStoreSstEntry.new(value: "\x00\x00\x00\x01\x00\x00\x00\x02")
assert_equal 1, entry.value_version
assert_equal 2, entry.value_flags
end
end

7
test/models/user_test.rb
View File

@@ -1,7 +0,0 @@
require "test_helper"
class UserTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end

32
test/test_helper.rb
View File

@@ -1,32 +0,0 @@
ENV["RAILS_ENV"] ||= "test"
require_relative "../config/environment"
require "rails/test_help"
require "minitest/mock"
require_relative "./test_util"
class ActiveSupport::TestCase
# Run tests in parallel with specified workers
# parallelize(workers: :number_of_processors)
# Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
fixtures :all
# Add more helper methods to be used by all tests here...
parallelize_teardown do
Rails.logger.error "Deleting all records: #{ReduxApplicationRecord.subclasses.map(&:name)}"
ReduxApplicationRecord.subclasses.each(&:delete_all)
end
def start_logging!
@old_logger = ActiveRecord::Base.logger
ActiveRecord::Base.logger = Logger.new(STDOUT)
end
def end_logging!
ActiveRecord::Base.logger = @old_logger
end
def read_fixture_file(path)
File.read File.join("test/fixtures/files", path)
end
end

124
test/test_util.rb
View File

@@ -1,124 +0,0 @@
module TestUtil
def self.random_string(length)
(0...length).map { (65 + rand(26)).chr }.join
end
def self.build_http_log_entry(
uri: "http://example.com",
status_code: 200,
content_type: "text/plain",
contents: nil
)
entry =
::HttpLogEntry.new(
{
uri: uri,
verb: :get,
performed_by: "direct",
status_code: status_code,
response_time_ms: rand(20..100),
request_headers: create_http_headers,
response_headers: create_http_headers,
response:
build_blob_entry(content_type: content_type, contents: contents),
content_type: content_type,
requested_at: Time.now,
},
)
raise entry.errors.full_messages.join(", ") unless entry.valid?
entry
end
def self.build_blob_entry(content_type: "text/plain", contents: nil)
BlobEntry.find_or_build(
content_type: content_type,
contents: contents || random_string(1024),
)
end
def self.create_blob_entry
be = build_blob_entry
be.save!
be
end
def self.create_http_headers
::HttpLogEntryHeader.find_or_create(
headers: {
test_header_key: "test header value #{random_string(16)}",
},
)
end
def self.build_fa_user
str = random_string(8)
user =
::Domain::Fa::User.new(
{ name: "Test-User-#{str}", url_name: "test-user-#{str.downcase}" },
)
user.valid? || raise(user.errors.full_messages.join(", "))
user
end
def self.build_fa_post
creator = build_fa_user
creator.save!
post = ::Domain::Fa::Post.new({ fa_id: 10, creator: creator })
post
end
def self.mock_curl_easy(
expected_url,
response_code: 200,
body_str: "http body",
request_headers: {},
response_headers_str: "HTTP 200\r\nresp: respheader\r\ncontent-type: text/plain\r\n\r\n"
)
mock = MiniTest::Mock.new
def mock.singleton_method_added(_)
{}
end
mock.expect(:url=, nil, [expected_url])
def mock.headers
{}
end
def mock.headers=(_)
{}
end
mock.expect(:perform, nil)
mock.expect(:response_code, response_code)
mock.expect(:body_str, body_str)
mock.expect(:header_str, response_headers_str)
mock.expect(:headers, request_headers)
mock
end
def self.mock_http_performer(
expected_url,
request_headers: {},
response_code: 200,
response_time_ms: 15,
response_headers: { "content-type" => "text/plain" },
response_body: "http body"
)
mock = MiniTest::Mock.new
mock.expect(:is_a?, false, [String])
mock.expect(:is_a?, false, [String])
mock.expect(:name, "direct")
mock.expect(
:get,
Scraper::CurlHttpPerformer::Response.new(
response_code,
response_headers,
response_time_ms,
response_body,
),
[expected_url, request_headers],
)
mock
end
end