dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rake job fixes, optimize fa favs backfill

Browse Source
This commit is contained in:
Dylan Knutson
2025-09-10 16:30:51 +00:00
parent 4f7217abf0
commit 3174032ac3
10 changed files with 205 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
spec/lib/tasks/create_post_file_fingerprints_task_spec.rb
View File

@@ -2,8 +2,19 @@
require "rails_helper"
RSpec.describe Tasks::CreatePostFileFingerprintsTask do
let(:mode) { raise("mode is required") }
let(:user_param) { nil }
let(:start_at) { nil }
let(:log_sink) { StringIO.new }
let(:task) { described_class.new(log_sink: log_sink) }
let(:task) do
described_class.new(
mode: mode,
user_param: user_param,
start_at: start_at,
log_sink: log_sink,
)
end
before do
# Mock external dependencies
@@ -28,17 +39,19 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
describe "#run" do
context "with user mode missing user_param" do
let(:mode) { Tasks::CreatePostFileFingerprintsTask::Mode::User }
it "raises an error when user mode is used without user_param" do
expect {
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::User,
user_param: nil,
)
}.to raise_error("need 'user_param' when mode is Mode::User")
expect { task.run }.to raise_error(
"need 'user_param' when mode is Mode::User",
)
end
end
context "with PostFileDescending mode" do
let(:mode) do
Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending
end
let!(:post_files) do
# Create test post files with descending IDs
5
@@ -57,9 +70,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
create(:domain_post_file, post: post, state: "ok")
end
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
)
task.run
# Should have saved progress at least once
saved_progress = GlobalState.get("task-create-post-file-fingerprints")
@@ -67,20 +78,16 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
expect(saved_progress.to_i).to be > 0
end
it "starts from specified start_at ID" do
# Create post files with specific IDs
target_post_file = post_files[2] # Third post file
start_at_id = target_post_file.id.to_s
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
start_at: start_at_id,
)
expect(log_sink.string).to include("resuming from:")
context "with start_at specified" do
let(:start_at) { post_files[2].id.to_s }
it "starts from specified start_at ID" do
task.run
expect(log_sink.string).to include("resuming from:")
end
end
context "with start_at='last' and existing progress" do
let(:start_at) { "last" }
before do
# Set up saved progress
target_post_file = post_files[2]
@@ -91,22 +98,15 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
end
it "resumes from saved progress" do
task.run(
mode:
Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
start_at: "last",
)
task.run
expect(log_sink.string).to include("resuming from:")
end
end
context "with start_at='last' and no existing progress" do
let(:start_at) { "last" }
it "starts from beginning when no saved progress found" do
task.run(
mode:
Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
start_at: "last",
)
task.run
expect(log_sink.string).to include("no saved progress")
end
@@ -114,6 +114,9 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
end
context "with PostsDescending mode" do
let(:mode) do
Tasks::CreatePostFileFingerprintsTask::Mode::PostsDescending
end
let!(:posts) do
3.times.map { |i| create(:domain_post_fa_post, fa_id: 400 + i) }
end
@@ -132,9 +135,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).exactly(3).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostsDescending,
)
task.run
end
it "uses correct progress bar total" do
@@ -147,9 +148,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
)
allow(ProgressBar).to receive(:create).and_return(progress_bar)
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostsDescending,
)
task.run
expect(ProgressBar).to have_received(:create).with(
total: 66_431_808,
@@ -161,6 +160,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
end
context "with User mode" do
let(:mode) { Tasks::CreatePostFileFingerprintsTask::Mode::User }
let!(:user) { create(:domain_user_fa_user, url_name: "testuser") }
let!(:posts) do
3.times.map do |i|
@@ -173,56 +173,54 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
posts.each { |post| create(:domain_post_file, post: post) }
end
it "processes posts for specified user" do
# Set up mocks before running the task
expect(Domain::PostFile::Thumbnail).to receive(
:create_for_post_file!,
).exactly(3).times
expect(Domain::PostFile::BitFingerprint).to receive(
:create_for_post_file!,
).exactly(3).times
context "with an existing user " do
let(:user_param) { user.to_param }
it "processes posts for specified user" do
# Set up mocks before running the task
expect(Domain::PostFile::Thumbnail).to receive(
:create_for_post_file!,
).exactly(3).times
expect(Domain::PostFile::BitFingerprint).to receive(
:create_for_post_file!,
).exactly(3).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::User,
user_param: user.to_param,
)
task.run
expect(log_sink.string).to include(
"migrating posts for #{user.to_param}",
)
end
it "raises error when user is not found" do
expect {
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::User,
user_param: "nonexistent_user",
expect(log_sink.string).to include(
"migrating posts for #{user.to_param}",
)
}.to raise_error(
ActionController::BadRequest,
"invalid id: \"nonexistent_user\"",
)
end
it "handles posts without creators" do
# Create a post without a creator
create(:domain_post_fa_post, creator: nil)
expect(Domain::PostFile::Thumbnail).to receive(
:create_for_post_file!,
).exactly(3).times
expect(Domain::PostFile::BitFingerprint).to receive(
:create_for_post_file!,
).exactly(3).times
task.run
end
end
it "handles posts without creators" do
# Create a post without a creator
create(:domain_post_fa_post, creator: nil)
expect(Domain::PostFile::Thumbnail).to receive(
:create_for_post_file!,
).exactly(3).times
expect(Domain::PostFile::BitFingerprint).to receive(
:create_for_post_file!,
).exactly(3).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::User,
user_param: user.to_param,
)
context "with a nonexistent user" do
let(:user_param) { "nonexistent_user" }
it "raises error when user is not found" do
expect { task.run }.to raise_error(
ActionController::BadRequest,
"invalid id: \"nonexistent_user\"",
)
end
end
end
context "with UsersDescending mode" do
let(:mode) do
Tasks::CreatePostFileFingerprintsTask::Mode::UsersDescending
end
let!(:users) do
3.times.map { |i| create(:domain_user_fa_user, url_name: "user#{i}") }
end
@@ -244,9 +242,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).exactly(3).times.and_return([])
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::UsersDescending,
)
task.run
users.each do |user|
expect(log_sink.string).to include(
@@ -264,9 +260,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).exactly(3).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::UsersDescending,
)
task.run
# Check that the migrated file was created and contains user params
if File.exist?("migrated_files.txt")
@@ -285,6 +279,10 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
end
describe "#save_progress" do
let(:mode) do
Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending
end
it "saves progress to GlobalState" do
task.send(:save_progress, 12_345.to_s)
@@ -295,6 +293,9 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
describe "error handling" do
let!(:post_file) { create(:domain_post_file, state: "ok") }
let(:mode) do
Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending
end
it "handles errors in migrate_post_file gracefully" do
# Mock the job to raise an error
@@ -302,9 +303,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).and_raise(StandardError.new("Test error"))
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
)
task.run
expect(log_sink.string).to include("error: Test error")
end
@@ -318,15 +317,17 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).and_raise(StandardError.new("Test error"))
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostsDescending,
)
task.run
expect(log_sink.string).to include("error: Test error")
end
end
describe "utility methods" do
let(:mode) do
Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending
end
describe "#create_progress_bar" do
it "creates a progress bar with the correct format" do
progress_bar = instance_double(ProgressBar::Base)
@@ -354,6 +355,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
end
describe "interrupt functionality" do
let(:mode) { Tasks::CreatePostFileFingerprintsTask::Mode::PostsDescending }
let!(:posts) do
3.times.map { |i| create(:domain_post_fa_post, fa_id: 400 + i) }
end
@@ -383,9 +385,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).at_most(1).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostsDescending,
)
task.run
end
it "stops processing when interrupted in post_file_descending mode" do
@@ -409,39 +409,7 @@ RSpec.describe Tasks::CreatePostFileFingerprintsTask do
:create_for_post_file!,
).at_most(1).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
)
end
it "saves progress before interruption in post_file_descending mode" do
# Create enough post files to trigger progress saving
105.times do |i|
post = create(:domain_post_fa_post, fa_id: 600 + i)
create(:domain_post_file, post: post, state: "ok")
end
# Mock the interrupt to happen after progress saving
call_count = 0
allow(task).to receive(:interrupted?) do
call_count += 1
call_count > 100 # Let it save progress first
end
expect(Domain::PostFile::Thumbnail).to receive(
:create_for_post_file!,
).at_least(1).times
expect(Domain::PostFile::BitFingerprint).to receive(
:create_for_post_file!,
).at_least(1).times
task.run(
mode: Tasks::CreatePostFileFingerprintsTask::Mode::PostFileDescending,
)
# Should have saved progress before interruption
saved_progress = GlobalState.get("task-create-post-file-fingerprints")
expect(saved_progress).to be_present
task.run
end
end
end