dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Inkbunny job handling and enhance avatar management

Browse Source 
- Removed the direct assignment of user avatar URL in JobHelper, delegating it to the post update logic.
- Updated UpdatePostsJob to enqueue UserAvatarJob when the avatar URL changes, improving avatar management.
- Refactored tests to utilize new methods for checking enqueued job arguments, enhancing clarity and maintainability.
- Improved user avatar handling in UserAvatarJob, ensuring proper state management and logging.
- Added new utility methods in SpecUtil for better job argument retrieval in tests.
This commit is contained in:
Dylan Knutson
2024-12-31 05:33:28 +00:00
parent 1f3fa0074e
commit f1c91f1119
12 changed files with 144 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/jobs/domain/inkbunny/job/job_helper.rb
View File

@@ -22,7 +22,6 @@ module Domain::Inkbunny::Job::JobHelper
user = Domain::Inkbunny::User.find_or_initialize_by(ib_user_id: ib_user_id)
user.name = submission_json["username"]
user.avatar_url_str = submission_json["user_icon_url_large"]
user.save! if user.changed?
user
end

21
app/jobs/domain/inkbunny/job/update_posts_job.rb
View File

@@ -22,6 +22,7 @@ module Domain::Inkbunny::Job
"&show_description=yes&show_writing=yes&show_pools=yes"
@api_submissions_response =
http_client.get(url, caused_by_entry: @caused_by_entry)
@log_entry = @api_submissions_response.log_entry
if @api_submissions_response.status_code != 200
fatal_error(
"api_submissions failed: #{@api_submissions_response.status_code}",
@@ -38,10 +39,7 @@ module Domain::Inkbunny::Job
end
def deep_update_post_from_submission_json(submission_json)
post =
Domain::Inkbunny::Job::JobHelper.find_or_create_post_from_submission_json(
submission_json,
)
post = JobHelper.find_or_create_post_from_submission_json(submission_json)
logger.info "deep update post #{post.ib_post_id.to_s.bold}"
post.deep_updated_at = Time.now
post.description = submission_json["description"]
@@ -53,10 +51,17 @@ module Domain::Inkbunny::Job
post.last_file_updated_at =
Time.parse(submission_json["last_file_update_datetime"])
# TODO - enqueue avatar download job if needed
if submission_json["user_icon_url_large"]
post.creator.avatar_url_str = submission_json["user_icon_url_large"]
post.creator.save! if post.creator.changed?
user = post.creator
user.avatar_url_str = submission_json["user_icon_url_large"]
if user.avatar_url_str_changed?
logger.info "avatar url changed, enqueuing download for user #{user.name}"
defer_job(
Domain::Inkbunny::Job::UserAvatarJob,
{ user: user, caused_by_entry: @log_entry },
)
end
user.save!
end
post_files_by_md5 = post.files.index_by(&:md5_initial)
@@ -91,7 +96,7 @@ module Domain::Inkbunny::Job
defer_job(
Domain::Inkbunny::Job::FileJob,
{ file: file, caused_by_entry: @api_submissions_response.log_entry },
{ file: file, caused_by_entry: @log_entry },
)
end
post.save!

12
app/jobs/domain/inkbunny/job/user_avatar_job.rb
View File

@@ -1,6 +1,6 @@
module Domain::Inkbunny::Job
class UserAvatarJob < Base
queue_as :inkbunny_user_avatar
queue_as :static_file
ignore_signature_args :caused_by_entry
def perform(args)
@@ -25,22 +25,18 @@ module Domain::Inkbunny::Job
@user.avatar_file_log_entry_id,
].compact
@user.avatar_state_detail["log_entries"] << response.log_entry.id
@user.avatar_file_log_entry_id = response.log_entry.id
@user.avatar_log_entry = response.log_entry
case response.status_code
when 200
@user.avatar_state = :ok
@user.avatar_state_detail.delete("download_error")
@user.avatar_downloaded_at = response.log_entry.created_at
@user.avatar_file_sha256 = response.log_entry.response_sha256
logger.info("downloaded avatar")
when 404
@user.avatar_state = :not_found
if @user.avatar_file_sha256.blank?
@user.avatar_downloaded_at = response.log_entry.created_at
logger.info("avatar 404, and no previous file")
else
logger.info("avatar 404, keeping previous file")
end
logger.info("avatar 404")
else
@user.avatar_state = :error
@user.avatar_state_detail[

2
app/lib/scraper/inkbunny_http_client_config.rb
View File

@@ -41,7 +41,7 @@ class Scraper::InkbunnyHttpClientConfig < Scraper::HttpClientConfig
end
def ratelimit
[["inkbunny.net", 2], ["*.ib.metapix.net", 1]]
[["inkbunny.net", 2], ["*.ib.metapix.net", 0.25]]
end
def allowed_domains

23
spec/jobs/domain/e621/job/scan_post_job_spec.rb
View File

@@ -7,7 +7,7 @@ describe Domain::E621::Job::ScanPostJob do
it "scans the post" do
post = create(:domain_e621_post, e621_id: 2_227_914)
caused_by_entry = create(:http_log_entry)
mock_log_entries =
log_entries =
SpecUtil.init_http_client_mock(
http_client_mock,
[
@@ -37,17 +37,14 @@ describe Domain::E621::Job::ScanPostJob do
hash_including("general" => array_including("black_nose", "facial_tuft")),
)
expect(
SpecUtil.enqueued_jobs(Domain::E621::Job::StaticFileJob).length,
).to eq(1)
expect(
SpecUtil.enqueued_jobs(Domain::E621::Job::StaticFileJob).first[:args],
).to eq([{ post: post, caused_by_entry: mock_log_entries[0] }])
expect(SpecUtil.enqueued_job_args(Domain::E621::Job::StaticFileJob)).to eq(
[{ post: post, caused_by_entry: log_entries[0] }],
)
end
it "handles a post with no file url" do
post = create(:domain_e621_post, e621_id: 5_270_136)
mock_log_entries =
log_entries =
SpecUtil.init_http_client_mock(
http_client_mock,
[
@@ -72,11 +69,9 @@ describe Domain::E621::Job::ScanPostJob do
expect(post.tags_array).to match(
hash_including("general" => array_including("anthro", "duo")),
)
expect(
SpecUtil.enqueued_jobs(Domain::E621::Job::StaticFileJob).length,
).to eq(1)
expect(
SpecUtil.enqueued_jobs(Domain::E621::Job::StaticFileJob).first[:args],
).to eq([{ post: post, caused_by_entry: mock_log_entries[0] }])
expect(SpecUtil.enqueued_job_args(Domain::E621::Job::StaticFileJob)).to eq(
[{ post: post, caused_by_entry: log_entries[0] }],
)
end
end

96
spec/jobs/domain/fa/job/browse_page_job_spec.rb
View File

@@ -24,13 +24,9 @@ describe Domain::Fa::Job::BrowsePageJob do
shared_examples "enqueue post scan" do |expect_to_enqueue|
if expect_to_enqueue
it "enqueues post scan job" do
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::ScanPostJob)).to match(
[
including(
args: [{ post: post.call, caused_by_entry: log_entries[0] }]
)
]
)
expect(
SpecUtil.enqueued_job_args(Domain::Fa::Job::ScanPostJob),
).to match([{ post: post.call, caused_by_entry: log_entries[0] }])
end
end
@@ -47,9 +43,9 @@ describe Domain::Fa::Job::BrowsePageJob do
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::ScanFileJob)).to match(
[
including(
args: [{ post: post.call, caused_by_entry: log_entries[0] }]
)
]
args: [{ post: post.call, caused_by_entry: log_entries[0] }],
),
],
)
end
end
@@ -67,9 +63,9 @@ describe Domain::Fa::Job::BrowsePageJob do
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob)).to match(
[
including(
args: [{ user: user.call, caused_by_entry: log_entries[0] }]
)
]
args: [{ user: user.call, caused_by_entry: log_entries[0] }],
),
],
)
end
end
@@ -85,13 +81,13 @@ describe Domain::Fa::Job::BrowsePageJob do
if expect_to_enqueue
it "enqueues user gallery job" do
expect(
SpecUtil.enqueued_jobs(Domain::Fa::Job::UserGalleryJob)
SpecUtil.enqueued_jobs(Domain::Fa::Job::UserGalleryJob),
).to match(
[
including(
args: [{ user: user.call, caused_by_entry: log_entries[0] }]
)
]
args: [{ user: user.call, caused_by_entry: log_entries[0] }],
),
],
)
end
end
@@ -99,7 +95,7 @@ describe Domain::Fa::Job::BrowsePageJob do
unless expect_to_enqueue
it "does not enqueue user gallery job" do
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserGalleryJob)).to eq(
[]
[],
)
end
end
@@ -139,11 +135,11 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_no_submissions.html"
"domain/fa/job/browse_page_no_submissions.html",
),
caused_by_entry_idx: nil
}
]
caused_by_entry_idx: nil,
},
],
)
end
@@ -163,9 +159,9 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_duplicates.html"
"domain/fa/job/browse_page_duplicates.html",
),
caused_by_entry_idx: nil
caused_by_entry_idx: nil,
},
{
uri: "https://www.furaffinity.net/browse/2/",
@@ -173,11 +169,11 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_no_submissions.html"
"domain/fa/job/browse_page_no_submissions.html",
),
caused_by_entry_idx: 0
}
]
caused_by_entry_idx: 0,
},
],
)
end
@@ -198,9 +194,9 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_one_submission.html"
"domain/fa/job/browse_page_one_submission.html",
),
caused_by_entry_idx: nil
caused_by_entry_idx: nil,
},
{
uri: "https://www.furaffinity.net/browse/2/",
@@ -208,11 +204,11 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_no_submissions.html"
"domain/fa/job/browse_page_no_submissions.html",
),
caused_by_entry_idx: 0
}
]
caused_by_entry_idx: 0,
},
],
)
end
@@ -249,9 +245,9 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_two_submissions.html"
"domain/fa/job/browse_page_two_submissions.html",
),
caused_by_entry_idx: nil
caused_by_entry_idx: nil,
},
{
uri: "https://www.furaffinity.net/browse/2/",
@@ -259,11 +255,11 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_no_submissions.html"
"domain/fa/job/browse_page_no_submissions.html",
),
caused_by_entry_idx: 0
}
]
caused_by_entry_idx: 0,
},
],
)
end
@@ -288,8 +284,8 @@ describe Domain::Fa::Job::BrowsePageJob do
[
including(args: [{ post: post1, caused_by_entry: log_entries[0] }]),
including(args: [{ post: post3, caused_by_entry: log_entries[0] }]),
including(args: [{ post: post2, caused_by_entry: log_entries[0] }])
]
including(args: [{ post: post2, caused_by_entry: log_entries[0] }]),
],
)
end
end
@@ -307,11 +303,11 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_one_submission.html"
"domain/fa/job/browse_page_one_submission.html",
),
caused_by_entry_idx: nil
}
]
caused_by_entry_idx: nil,
},
],
)
end
@@ -394,10 +390,10 @@ describe Domain::Fa::Job::BrowsePageJob do
content_type: "text/html",
contents:
SpecUtil.read_fixture_file(
"domain/fa/job/browse_page_no_submissions.html"
)
}
]
"domain/fa/job/browse_page_no_submissions.html",
),
},
],
)
end

4
spec/jobs/domain/fa/job/favs_job_spec.rb
View File

@@ -20,8 +20,8 @@ describe Domain::Fa::Job::FavsJob do
end
it "enqueues a page scan job" do
perform_now({ url_name: "zzreg" }, should_raise: true)
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob)).to match(
[including(args: [including(url_name: "zzreg")])],
expect(SpecUtil.enqueued_job_args(Domain::Fa::Job::UserPageJob)).to match(
[including({ url_name: "zzreg" })],
)
end
end

39
spec/jobs/domain/inkbunny/job/user_avatar_job_spec.rb
View File

@@ -120,9 +120,7 @@ describe Domain::Inkbunny::Job::UserAvatarJob do
expect(user.avatar).to be_nil
expect(user.avatar_log_entry).to eq(log_entries[0])
expect(user.avatar_state_detail["log_entries"]).to eq([log_entries[0].id])
expect(user.avatar_downloaded_at).to be_within(1.second).of(
log_entries[0].created_at,
)
expect(user.avatar_downloaded_at).to be_nil
end
context "when previous file exists" do
@@ -166,6 +164,13 @@ describe Domain::Inkbunny::Job::UserAvatarJob do
contents: "server error",
caused_by_entry_idx: nil,
},
{
uri: avatar_url_str,
status_code: 200,
content_type: "image/jpeg",
contents: "test",
caused_by_entry_idx: nil,
},
],
)
end
@@ -179,6 +184,20 @@ describe Domain::Inkbunny::Job::UserAvatarJob do
expect(user.avatar).to be_nil
expect(user.avatar_log_entry).to eq(log_entries[0])
expect(user.avatar_state_detail["log_entries"]).to eq([log_entries[0].id])
perform_now({ user: user })
user.reload
expect(user.avatar_state).to eq("ok")
expect(user.avatar_state_detail["download_error"]).to be_nil
expect(user.avatar).to be_present
expect(user.avatar.sha256).to eq(log_entries[1].response_sha256)
expect(user.avatar_log_entry).to eq(log_entries[1])
expect(user.avatar_state_detail["log_entries"]).to eq(
log_entries.map(&:id),
)
expect(user.avatar_downloaded_at).to be_within(1.second).of(
log_entries[1].created_at,
)
end
context "when previous file exists" do
@@ -206,6 +225,20 @@ describe Domain::Inkbunny::Job::UserAvatarJob do
expect(user.avatar_downloaded_at).to be_within(1.second).of(
existing_downloaded_at,
)
perform_now({ user: user })
user.reload
expect(user.avatar_state).to eq("ok")
expect(user.avatar_state_detail["download_error"]).to be_nil
expect(user.avatar).to be_present
expect(user.avatar.sha256).to eq(log_entries[1].response_sha256)
expect(user.avatar_log_entry).to eq(log_entries[1])
expect(user.avatar_state_detail["log_entries"]).to eq(
[existing_log_entry.id] + log_entries.map(&:id),
)
expect(user.avatar_downloaded_at).to be_within(1.second).of(
log_entries[1].created_at,
)
end
end
end

39
spec/jobs/domain/inkbunny/update_posts_job_spec.rb
View File

@@ -77,13 +77,10 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
)
end
let(:ib_post_ids) { [3_104_202, 3_104_200, 3_104_197] }
it "updates posts with detailed information" do
perform_now(
{
ib_post_ids: [3_104_202, 3_104_200, 3_104_197],
caused_by_entry: nil,
},
)
perform_now({ ib_post_ids: ib_post_ids, caused_by_entry: nil })
# Check post details were updated
post_3104202.reload
@@ -127,22 +124,42 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
expect(job[:args][0][:caused_by_entry]).to eq(log_entries[0])
end
end
it "enqueues avatar jobs" do
perform_now({ ib_post_ids: ib_post_ids, caused_by_entry: nil })
user_soulcentinel.reload
expect(user_soulcentinel.avatar_url_str).to eq(
"https://us.ib.metapix.net/usericons/large/208/208598_SoulCentinel_fireb.gif",
)
expect(
SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::UserAvatarJob).length,
).to eq(3)
expect(
SpecUtil.enqueued_job_args(Domain::Inkbunny::Job::UserAvatarJob),
).to match(
[
{ user: user_soulcentinel, caused_by_entry: log_entries[0] },
{ user: user_seff, caused_by_entry: log_entries[0] },
{ user: user_thendyart, caused_by_entry: log_entries[0] },
],
)
end
end
context "when a post's files change" do
let(:api_submissions_url_before) do
let(:api_submissions_url) do
"https://inkbunny.net/api_submissions.php?submission_ids=1047334&show_description=yes&show_writing=yes&show_pools=yes"
end
let(:api_submissions_url_after) { api_submissions_url_before }
let! :log_entries do
SpecUtil.init_http_client_mock(
http_client_mock,
[
{
method: :get,
uri: api_submissions_url_before,
uri: api_submissions_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
@@ -151,7 +168,7 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
},
{
method: :get,
uri: api_submissions_url_after,
uri: api_submissions_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(

2
spec/lib/domain/fa/parser/redux_page_spec.rb
View File

@@ -461,7 +461,7 @@ describe Domain::Fa::Parser::Page do
parser = get_parser("favs_zzreg_page_last_2024_format.html")
assert_page_type parser, :probably_listings_page?
assert_equal parser.submissions_parsed.length, 37
assert_equal parser.favorites_next_button_id, nil
assert_nil parser.favorites_next_button_id
end
def get_parser(file, require_logged_in: true)

4
spec/spec_util.rb
View File

@@ -142,6 +142,10 @@ class SpecUtil
.filter { |job| job_is_class(job_class, job) }
end
def self.enqueued_job_args(job_class = nil)
enqueued_jobs(job_class).map { |job| job[:args][0] }
end
def self.clear_enqueued_jobs!(job_class = nil)
rel = GoodJob::Job
rel = rel.where(job_class: job_class.name) if job_class

0
spec/support/matchers/job_matchers.rb Normal file
View File