dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add JobHelper module and enhance Inkbunny job processing

Browse Source 
- Introduced a new `JobHelper` module to encapsulate job retrieval and execution logic, enforcing strict typing with Sorbet.
- Updated `UpdatePostsJob` to improve logging and error handling for file processing, including handling files with null MD5 sums.
- Enhanced validation in `Domain::Inkbunny::File` model to conditionally require presence of certain attributes based on file state.
- Added a new fixture for testing scenarios involving submissions with null MD5 sums, improving test coverage and robustness.

These changes aim to improve type safety, maintainability, and error handling in job processing logic.
This commit is contained in:
Dylan Knutson
2025-01-02 17:52:11 +00:00
parent 76c701a97f
commit 025b4ac8e4
5 changed files with 476 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
app/jobs/domain/inkbunny/job/update_posts_job.rb
View File

@@ -39,6 +39,7 @@ class Domain::Inkbunny::Job::UpdatePostsJob < Domain::Inkbunny::Job::Base
deep_update_post_from_submission_json(submission_json, log_entry)
end
end
logger.prefix = ""
end
sig do
@@ -48,6 +49,8 @@ class Domain::Inkbunny::Job::UpdatePostsJob < Domain::Inkbunny::Job::Base
).void
end
def deep_update_post_from_submission_json(submission_json, log_entry)
logger.prefix = "ib_post_id #{submission_json["submission_id"].to_s.bold}"
post =
Domain::Inkbunny::Post.find_by!(
ib_post_id: submission_json["submission_id"],
@@ -87,9 +90,13 @@ class Domain::Inkbunny::Job::UpdatePostsJob < Domain::Inkbunny::Job::Base
md5_initial = file_json["initial_file_md5"]
next if post_files_by_md5[md5_initial]
# We create all files, even those with null MD5 sums (which also do not have
# a valid download URL), so that post.files.count will be accurate and match
# pagecount.
file =
post.files.create!(
post.files.create(
{
state: md5_initial.present? ? "ok" : "error",
ib_file_id: file_json["file_id"]&.to_i,
ib_created_at: Time.parse(file_json["create_datetime"]),
file_order: file_json["submission_file_order"]&.to_i,
@@ -108,6 +115,18 @@ class Domain::Inkbunny::Job::UpdatePostsJob < Domain::Inkbunny::Job::Base
},
)
if file.state == "error"
logger.error "file #{file.ib_file_id.to_s.bold} is poorly formed, skipping enqueue"
next
end
if file.invalid?
logger.error "file #{file.ib_file_id.to_s.bold} (ib_post_id #{post.ib_post_id.to_s.bold}) is invalid: #{file.errors.full_messages.join(", ")}"
fatal_error(
"file #{file.ib_file_id.to_s.bold} is invalid: #{file.errors.full_messages.join(", ")}",
)
end
logger.info "[ib_post_id #{post.ib_post_id.to_s.bold}] " +
"new file #{file.ib_file_id.to_s.bold} - #{file.file_name&.black&.bold}"

32
app/jobs/job_helper.rb Normal file
View File

@@ -0,0 +1,32 @@
# typed: strict
module JobHelper
extend T::Sig
# @param good_job_uuid [String]
# @return [GoodJob::Job]
sig { params(good_job_uuid: String).returns(GoodJob::Job) }
def self.find_job(good_job_uuid)
GoodJob::Job.find(good_job_uuid)
end
# @param good_job_uuid [String]
# @return [void]
sig { params(good_job_uuid: String).void }
def self.perform_job(good_job_uuid)
job = find_job(good_job_uuid)
job_args_deserialized =
T.let(
ActiveJob::Arguments.deserialize(job.serialized_params),
T::Array[[String, T.untyped]],
)
job_args = job_args_deserialized.find { |key, _| key == "arguments" }
raise("no arguments for #{job.job_class}") if job_args.nil?
job_args = T.cast(job_args[1], T::Array[T.untyped])
job_instance =
job.job_class&.constantize&.new ||
raise("no job_class for #{job.job_class}")
job_instance.arguments = job_args
job_instance&.perform(*job_args)
end
end

7
app/models/domain/inkbunny/file.rb
View File

@@ -18,6 +18,11 @@ class Domain::Inkbunny::File < ReduxApplicationRecord
end
validates_presence_of(
%i[ib_file_id file_name url_str ib_created_at file_order md5_initial md5s],
%i[ib_file_id file_name url_str ib_created_at file_order],
)
validates_presence_of(
%i[md5_initial md5s],
unless: -> { T.bind(self, Domain::Inkbunny::File).state == "error" },
)
end

74
spec/jobs/domain/inkbunny/job/update_posts_job_spec.rb
View File

@@ -254,4 +254,78 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
expect(file_jobs[1][:args][0][:caused_by_entry]).to eq(log_entries[1])
end
end
context "when a submission has files with null MD5 sums" do
let(:api_submissions_url) do
"https://inkbunny.net/api_submissions.php?submission_ids=2637105&show_description=yes&show_writing=yes&show_pools=yes"
end
let! :log_entries do
SpecUtil.init_http_client_mock(
http_client_mock,
[
{
method: :get,
uri: api_submissions_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions_null_md5sum.json",
),
},
],
)
end
let!(:user_friar) do
create(:domain_inkbunny_user, ib_user_id: 1664, name: "Friar")
end
let!(:post_2637105) do
create(
:domain_inkbunny_post,
ib_post_id: 2_637_105,
creator: user_friar,
num_files: 5,
)
end
it "handles files with null MD5 sums correctly" do
perform_now({ ib_post_ids: [2_637_105], caused_by_entry: nil })
post_2637105.reload
expect(post_2637105.description).to eq(
"Berry and Friar cheer up Barrett.",
)
expect(post_2637105.writing).to eq("")
expect(post_2637105.num_views).to eq(170)
expect(post_2637105.num_favs).to eq(11)
expect(post_2637105.num_comments).to eq(1)
expect(post_2637105.deep_update_log_entry).to eq(log_entries[0])
expect(post_2637105.deep_updated_at).to be_within(1.second).of(Time.now)
# Check that files were created, even those with null MD5 sums
expect(post_2637105.files.count).to eq(5)
file_3897070 = post_2637105.files.find_by!(ib_file_id: 3_897_070)
expect(file_3897070.url_str).to eq(
"https://tx.ib.metapix.net/files/full/3897/3897070_Friar_ffrbb4.png",
)
expect(file_3897070.attributes).to include(
"md5_initial" => "",
"md5_full" => "",
)
expect(file_3897070.state).to eq("error")
# Check file jobs were enqueued for valid files
file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
expect(file_jobs.length).to eq(4)
expect(
file_jobs.map { |job| job[:args][0][:file].ib_file_id }.sort,
).to eq([3_897_058, 3_897_061, 3_897_065, 3_897_067])
file_jobs.each do |job|
expect(job[:args][0][:caused_by_entry]).to eq(log_entries[0])
end
end
end
end

344
test/fixtures/files/domain/inkbunny/job/api_submissions_null_md5sum.json vendored Normal file
View File

@@ -0,0 +1,344 @@
{
"sid": "9B5BPQqTZ,Mj-nYbL2tuIPoO,b",
"results_count": 100,
"user_location": "",
"submissions": [
{
"submission_id": "2637105",
"keywords": [
{
"keyword_id": "27",
"keyword_name": "babyfur",
"contributed": "f",
"submissions_count": "38576"
},
{
"keyword_id": "1260",
"keyword_name": "babyfurs",
"contributed": "f",
"submissions_count": "5044"
},
{
"keyword_id": "18450",
"keyword_name": "care bears",
"contributed": "f",
"submissions_count": "1162"
},
{
"keyword_id": "28",
"keyword_name": "diaper",
"contributed": "f",
"submissions_count": "76689"
},
{
"keyword_id": "5864",
"keyword_name": "diaperfur",
"contributed": "f",
"submissions_count": "17622"
},
{
"keyword_id": "14621",
"keyword_name": "diaperfurs",
"contributed": "f",
"submissions_count": "3142"
},
{
"keyword_id": "744",
"keyword_name": "diapers",
"contributed": "f",
"submissions_count": "21738"
},
{
"keyword_id": "33",
"keyword_name": "fox",
"contributed": "f",
"submissions_count": "245655"
},
{
"keyword_id": "174",
"keyword_name": "lion",
"contributed": "f",
"submissions_count": "42337"
},
{
"keyword_id": "165",
"keyword_name": "male",
"contributed": "f",
"submissions_count": "1181363"
},
{
"keyword_id": "28542",
"keyword_name": "messy diaper",
"contributed": "f",
"submissions_count": "6461"
},
{
"keyword_id": "66434",
"keyword_name": "messydiaper",
"contributed": "f",
"submissions_count": "430"
},
{
"keyword_id": "41896",
"keyword_name": "messy diapers",
"contributed": "f",
"submissions_count": "1753"
},
{
"keyword_id": "116323",
"keyword_name": "messydiapers",
"contributed": "f",
"submissions_count": "206"
},
{
"keyword_id": "25479",
"keyword_name": "nirvana",
"contributed": "f",
"submissions_count": "52"
}
],
"hidden": "f",
"scraps": "f",
"favorite": "f",
"favorites_count": "11",
"create_datetime": "2022-01-29 00:35:54.901218+00",
"create_datetime_usertime": "29 Jan 2022 01:35 CET",
"last_file_update_datetime": "2022-01-29 00:23:53.403546+00",
"last_file_update_datetime_usertime": "29 Jan 2022 01:23 CET",
"username": "Friar",
"user_id": "1664",
"user_icon_file_name": "320573_Friar_friar_byleopuppad.png",
"user_icon_url_large": "https://tx.ib.metapix.net/usericons/large/320/320573_Friar_friar_byleopuppad.png",
"user_icon_url_medium": "https://tx.ib.metapix.net/usericons/medium/320/320573_Friar_friar_byleopuppad.png",
"user_icon_url_small": "https://tx.ib.metapix.net/usericons/small/320/320573_Friar_friar_byleopuppad.png",
"file_name": "3897058_Friar_ffrbb.png",
"file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897058_Friar_ffrbb.png",
"file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897058_Friar_ffrbb.png",
"file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897058_Friar_ffrbb.jpg",
"thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3897/3897058_Friar_ffrbb.jpg",
"thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3897/3897058_Friar_ffrbb_noncustom.jpg",
"thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3897/3897058_Friar_ffrbb_noncustom.jpg",
"thumb_medium_noncustom_x": "120",
"thumb_medium_noncustom_y": "39",
"thumb_large_noncustom_x": "200",
"thumb_large_noncustom_y": "65",
"thumb_huge_noncustom_x": "300",
"thumb_huge_noncustom_y": "97",
"files": [
{
"file_id": "3897058",
"file_name": "3897058_Friar_ffrbb.png",
"file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897058_Friar_ffrbb.png",
"file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897058_Friar_ffrbb.png",
"file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897058_Friar_ffrbb.jpg",
"mimetype": "image/png",
"submission_id": "2637105",
"user_id": "1664",
"submission_file_order": "0",
"full_size_x": "2869",
"full_size_y": "926",
"screen_size_x": "920",
"screen_size_y": "297",
"preview_size_x": "300",
"preview_size_y": "97",
"initial_file_md5": "068b8fce07284ed4b118e1dd510ae7d8",
"full_file_md5": "77947b3f3d8673b0b9c3f7b65a2815e7",
"large_file_md5": "e4ff99ea22be9d764a93c1c5b85e16c3",
"small_file_md5": "bc4fff9766a8b3aeeb98222068af2f0b",
"thumbnail_md5": "42ec52dad5607f850f63934b2e4c0182",
"deleted": "f",
"create_datetime": "2022-01-28 23:49:21.206209+00",
"create_datetime_usertime": "29 Jan 2022 00:49 CET",
"thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3897/3897058_Friar_ffrbb.jpg",
"thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3897/3897058_Friar_ffrbb_noncustom.jpg",
"thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3897/3897058_Friar_ffrbb_noncustom.jpg",
"thumb_medium_noncustom_x": "120",
"thumb_medium_noncustom_y": "39",
"thumb_large_noncustom_x": "200",
"thumb_large_noncustom_y": "65",
"thumb_huge_noncustom_x": "300",
"thumb_huge_noncustom_y": "97"
},
{
"file_id": "3897061",
"file_name": "3897061_Friar_ffrbb1.png",
"file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897061_Friar_ffrbb1.png",
"file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897061_Friar_ffrbb1.png",
"file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897061_Friar_ffrbb1.jpg",
"mimetype": "image/png",
"submission_id": "2637105",
"user_id": "1664",
"submission_file_order": "1",
"full_size_x": "674",
"full_size_y": "790",
"screen_size_x": "674",
"screen_size_y": "790",
"preview_size_x": "256",
"preview_size_y": "300",
"initial_file_md5": "9b189b59fc838df29e449bbfd8734541",
"full_file_md5": "b2160f9dff1192109bea827847950b14",
"large_file_md5": "b2160f9dff1192109bea827847950b14",
"small_file_md5": "80d59b8411490f69061dbdaa495b0535",
"thumbnail_md5": "2cdd3572d20e155532ef17825d55b311",
"deleted": "f",
"create_datetime": "2022-01-29 00:06:46.735042+00",
"create_datetime_usertime": "29 Jan 2022 01:06 CET",
"thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3897/3897061_Friar_ffrbb1.jpg",
"thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3897/3897061_Friar_ffrbb1_noncustom.jpg",
"thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3897/3897061_Friar_ffrbb1_noncustom.jpg",
"thumb_medium_noncustom_x": "102",
"thumb_medium_noncustom_y": "120",
"thumb_large_noncustom_x": "171",
"thumb_large_noncustom_y": "200",
"thumb_huge_noncustom_x": "256",
"thumb_huge_noncustom_y": "300"
},
{
"file_id": "3897065",
"file_name": "3897065_Friar_ffrbb2.png",
"file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897065_Friar_ffrbb2.png",
"file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897065_Friar_ffrbb2.png",
"file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897065_Friar_ffrbb2.jpg",
"mimetype": "image/png",
"submission_id": "2637105",
"user_id": "1664",
"submission_file_order": "2",
"full_size_x": "670",
"full_size_y": "692",
"screen_size_x": "670",
"screen_size_y": "692",
"preview_size_x": "290",
"preview_size_y": "300",
"initial_file_md5": "9c8d2e149f7382355e44176fc718d72b",
"full_file_md5": "82a90516e278deeb7c0a017314fd72a1",
"large_file_md5": "82a90516e278deeb7c0a017314fd72a1",
"small_file_md5": "6dc62aff4539c097d21c65e5f0784081",
"thumbnail_md5": "a1a19404fb5e8962024152176f6c74b7",
"deleted": "f",
"create_datetime": "2022-01-29 00:12:16.980414+00",
"create_datetime_usertime": "29 Jan 2022 01:12 CET",
"thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3897/3897065_Friar_ffrbb2.jpg",
"thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3897/3897065_Friar_ffrbb2_noncustom.jpg",
"thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3897/3897065_Friar_ffrbb2_noncustom.jpg",
"thumb_medium_noncustom_x": "116",
"thumb_medium_noncustom_y": "120",
"thumb_large_noncustom_x": "194",
"thumb_large_noncustom_y": "200",
"thumb_huge_noncustom_x": "290",
"thumb_huge_noncustom_y": "300"
},
{
"file_id": "3897067",
"file_name": "3897067_Friar_ffrbb3.png",
"file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897067_Friar_ffrbb3.png",
"file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897067_Friar_ffrbb3.png",
"file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897067_Friar_ffrbb3.jpg",
"mimetype": "image/png",
"submission_id": "2637105",
"user_id": "1664",
"submission_file_order": "3",
"full_size_x": "668",
"full_size_y": "694",
"screen_size_x": "668",
"screen_size_y": "694",
"preview_size_x": "289",
"preview_size_y": "300",
"initial_file_md5": "5a295fbd69506869adef0598cd1cb490",
"full_file_md5": "5dfc433ed796e5b68b734ead18c38011",
"large_file_md5": "5dfc433ed796e5b68b734ead18c38011",
"small_file_md5": "c422c6d5e5d93a612034a2aa6e5e6370",
"thumbnail_md5": "44a8702f2361547726e95077a549edb5",
"deleted": "f",
"create_datetime": "2022-01-29 00:18:11.196403+00",
"create_datetime_usertime": "29 Jan 2022 01:18 CET",
"thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3897/3897067_Friar_ffrbb3.jpg",
"thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3897/3897067_Friar_ffrbb3_noncustom.jpg",
"thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3897/3897067_Friar_ffrbb3_noncustom.jpg",
"thumb_medium_noncustom_x": "116",
"thumb_medium_noncustom_y": "120",
"thumb_large_noncustom_x": "193",
"thumb_large_noncustom_y": "200",
"thumb_huge_noncustom_x": "289",
"thumb_huge_noncustom_y": "300"
},
{
"file_id": "3897070",
"file_name": "3897070_Friar_ffrbb4.png",
"file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897070_Friar_ffrbb4.png",
"file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897070_Friar_ffrbb4.png",
"file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897070_Friar_ffrbb4.jpg",
"mimetype": "image/png",
"submission_id": "2637105",
"user_id": "1664",
"submission_file_order": "4",
"full_size_x": null,
"full_size_y": null,
"screen_size_x": null,
"screen_size_y": null,
"preview_size_x": null,
"preview_size_y": null,
"initial_file_md5": "",
"full_file_md5": "",
"large_file_md5": "",
"small_file_md5": "",
"thumbnail_md5": "",
"deleted": "f",
"create_datetime": "2022-01-29 00:23:53.403546+00",
"create_datetime_usertime": "29 Jan 2022 01:23 CET"
}
],
"pools": [
{
"pool_id": "54330",
"name": "Treasured Guests - Weekly Strips",
"description": "Comic strips featuring Friar and his friends, not directly related to the comic in progress.",
"count": "322",
"submission_left_submission_id": "2642791",
"submission_left_file_name": "3905825_Friar_cthbb.png",
"submission_left_thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3905/3905825_Friar_cthbb.jpg",
"submission_left_thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3905/3905825_Friar_cthbb_noncustom.jpg",
"submission_left_thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3905/3905825_Friar_cthbb_noncustom.jpg",
"submission_left_thumb_medium_noncustom_x": "120",
"submission_left_thumb_medium_noncustom_y": "39",
"submission_left_thumb_large_noncustom_x": "200",
"submission_left_thumb_large_noncustom_y": "65",
"submission_left_thumb_huge_noncustom_x": "300",
"submission_left_thumb_huge_noncustom_y": "97",
"submission_right_submission_id": "2631657",
"submission_right_file_name": "3888851_Friar_bbandbb.png",
"submission_right_thumbnail_url_huge_noncustom": "https://tx.ib.metapix.net/files/preview/3888/3888851_Friar_bbandbb.jpg",
"submission_right_thumbnail_url_large_noncustom": "https://tx.ib.metapix.net/thumbnails/large/3888/3888851_Friar_bbandbb_noncustom.jpg",
"submission_right_thumbnail_url_medium_noncustom": "https://tx.ib.metapix.net/thumbnails/medium/3888/3888851_Friar_bbandbb_noncustom.jpg",
"submission_right_thumb_medium_noncustom_x": "120",
"submission_right_thumb_medium_noncustom_y": "39",
"submission_right_thumb_large_noncustom_x": "200",
"submission_right_thumb_large_noncustom_y": "65",
"submission_right_thumb_huge_noncustom_x": "300",
"submission_right_thumb_huge_noncustom_y": "97"
}
],
"description": "Berry and Friar cheer up Barrett.",
"writing": "",
"pools_count": 1,
"title": "Feelings Friends",
"deleted": "f",
"public": "t",
"mimetype": "image/png",
"pagecount": "5",
"rating_id": "0",
"rating_name": "General",
"ratings": [],
"submission_type_id": "1",
"type_name": "Picture/Pinup",
"guest_block": "f",
"friends_only": "f",
"comments_count": "1",
"views": "170",
"latest_file_name": "3897070_Friar_ffrbb4.png",
"latest_mimetype": "image/png",
"latest_file_url_full": "https://tx.ib.metapix.net/files/full/3897/3897070_Friar_ffrbb4.png",
"latest_file_url_screen": "https://tx.ib.metapix.net/files/screen/3897/3897070_Friar_ffrbb4.png",
"latest_file_url_preview": "https://tx.ib.metapix.net/files/preview/3897/3897070_Friar_ffrbb4.jpg"
}
]
}