Files
redux-scraper/app/jobs/domain/inkbunny/job/file_job.rb
2025-01-01 03:29:53 +00:00

53 lines
1.5 KiB
Ruby

# typed: true
module Domain::Inkbunny::Job
class FileJob < Base
queue_as :static_file
def perform(args)
file = args[:file] || fatal_error("file is required")
caused_by_entry = args[:caused_by_entry]
logger.prefix =
proc do
"[#{file.id.to_s.bold} / " + "#{file.ib_file_id.to_s.bold} / " +
"#{file.state.to_s.bold}] "
end
url_str = file.url_str
if file.state == "error"
retry_count = file.state_detail&.[]("error")&.[]("retry_count") || 0
if retry_count >= 3
logger.error("file has been retried 3 times, giving up")
return
end
end
response = http_client.get(url_str, caused_by_entry: caused_by_entry)
if response.status_code != 200
file.state = :error
fe = (file.state_detail["error"] ||= {})
fe["status_code"] = response.status_code
fe["log_entry_id"] = response.log_entry.id
fe["retry_count"] ||= 0
fe["retry_count"] += 1
file.save!
if response.status_code == 404
logger.error("#{response.status_code}, not retrying download")
else
fatal_error("#{response.status_code}, will retry later")
end
return
end
file.state = :ok
file.log_entry = response.log_entry
file.blob_entry = response.log_entry.response
file.state_detail.delete("error")
file.save!
logger.info "downloaded file"
end
end
end