better job dedup key computation
This commit is contained in:
38
rake/fa.rake
38
rake/fa.rake
@@ -166,22 +166,28 @@ namespace :fa do
|
||||
|
||||
desc "Get 404 files from FurArchiver"
|
||||
task get_404_files_from_fur_archiver: :set_logger_stdout do
|
||||
query =
|
||||
Domain::PostFile
|
||||
.joins(:post)
|
||||
.for_post_type(Domain::Post::FaPost)
|
||||
.where(state: "terminal_error", last_status_code: 404)
|
||||
.where(
|
||||
"((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) IS NULL OR ((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) != TRUE",
|
||||
)
|
||||
url_name = ENV["url_name"]
|
||||
|
||||
# query =
|
||||
# Domain::User
|
||||
# .find_by_param("fa@wolfsparta")
|
||||
# .posts
|
||||
# .flat_map do |post|
|
||||
# post.files.where(state: "terminal_error", last_status_code: 404)
|
||||
# end
|
||||
if url_name
|
||||
query =
|
||||
Domain::User
|
||||
.find_by_param("fa@#{url_name}")
|
||||
.posts
|
||||
.flat_map do |post|
|
||||
post.files.where(state: "terminal_error", last_status_code: 404)
|
||||
end
|
||||
method = :each
|
||||
else
|
||||
query =
|
||||
Domain::PostFile
|
||||
.joins(:post)
|
||||
.for_post_type(Domain::Post::FaPost)
|
||||
.where(state: "terminal_error", last_status_code: 404)
|
||||
.where(
|
||||
"((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) IS NULL OR ((\"post\".\"json_attributes\"->>'tried_from_fur_archiver')::bool) != TRUE",
|
||||
)
|
||||
method = :find_each
|
||||
end
|
||||
|
||||
puts "counting..."
|
||||
total = query.count
|
||||
@@ -189,7 +195,7 @@ namespace :fa do
|
||||
pb = ProgressBar.create(total: total, format: "%t: %c/%C %B %p%% %a %e")
|
||||
|
||||
counter = 0
|
||||
query.find_each do |post_file|
|
||||
query.send(method) do |post_file|
|
||||
next if post_file.url_str.include?("/stories/")
|
||||
Job::FaPostFurArchiverPostFileJob.perform_now({ post_file: })
|
||||
post = post_file.post
|
||||
|
||||
Reference in New Issue
Block a user