better job dedup key computation

This commit is contained in:
Dylan Knutson
2025-06-26 17:59:32 +00:00
parent e3b2463cbe
commit 3a06181db8
9 changed files with 255 additions and 50 deletions

View File

@@ -166,22 +166,28 @@ namespace :fa do
desc "Get 404 files from FurArchiver"
task get_404_files_from_fur_archiver: :set_logger_stdout do
query =
Domain::PostFile
.joins(:post)
.for_post_type(Domain::Post::FaPost)
.where(state: "terminal_error", last_status_code: 404)
.where(
"((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) IS NULL OR ((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) != TRUE",
)
url_name = ENV["url_name"]
# query =
# Domain::User
# .find_by_param("fa@wolfsparta")
# .posts
# .flat_map do |post|
# post.files.where(state: "terminal_error", last_status_code: 404)
# end
if url_name
query =
Domain::User
.find_by_param("fa@#{url_name}")
.posts
.flat_map do |post|
post.files.where(state: "terminal_error", last_status_code: 404)
end
method = :each
else
query =
Domain::PostFile
.joins(:post)
.for_post_type(Domain::Post::FaPost)
.where(state: "terminal_error", last_status_code: 404)
.where(
"((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) IS NULL OR ((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) != TRUE",
)
method = :find_each
end
puts "counting..."
total = query.count
@@ -189,7 +195,7 @@ namespace :fa do
pb = ProgressBar.create(total: total, format: "%t: %c/%C %B %p%% %a %e")
counter = 0
query.find_each do |post_file|
query.send(method) do |post_file|
next if post_file.url_str.include?("/stories/")
Job::FaPostFurArchiverPostFileJob.perform_now({ post_file: })
post = post_file.post