Refactor Inkbunny job processing and enhance post management

- Updated Inkbunny job classes to streamline argument handling by removing `ignore_signature_args :caused_by_entry`.
- Enhanced `ApiSearchPageProcessor` and `UpdatePostsJob` to include `caused_by_entry` for better logging and tracking of job origins.
- Introduced `deep_update_log_entry` and `shallow_update_log_entry` associations in Inkbunny post and user models for improved tracking of updates.
- Added `posted_at` attribute to `IndexedPost` model, ensuring synchronization with the postable's posted date.
- Enhanced views to display user posts in a more organized manner, including handling cases where media files are missing.
- Improved tests for Inkbunny jobs and models to ensure robust coverage of new functionality and maintainability.
This commit is contained in:
Dylan Knutson
2025-01-01 00:20:33 +00:00
parent a9bccb00e2
commit 8e9e720695
19 changed files with 300 additions and 54 deletions

View File

@@ -1,5 +1,5 @@
class Domain::Fa::Job::UserAvatarJob < Domain::Fa::Job::Base
queue_as :fa_user_avatar
queue_as :static_file
def perform(args)
init_from_args!(args, build_user: false)

View File

@@ -34,10 +34,13 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
# - num_new_posts: the number of new posts in the page for the submission_json that was just processed (indicates to the caller that the next page should be fetched)
# - num_pages: the total number of pages in the submission_json that was just processed
# - rid: the RID for the submission_json that was just processed
def process!(submissions_json)
def process!(submissions_json, caused_by_entry: nil)
num_new_posts = 0
submissions_json["submissions"].each do |submission_json|
if upsert_post_from_submission_json!(submission_json)
if upsert_post_from_submission_json!(
submission_json,
caused_by_entry: caused_by_entry,
)
num_new_posts += 1
@total_new_posts += 1
end
@@ -103,7 +106,7 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
# - pagecount
# - rating_id
# - submission_type_id
def upsert_post_from_submission_json!(submission_json)
def upsert_post_from_submission_json!(submission_json, caused_by_entry: nil)
ib_post_id = submission_json["submission_id"]&.to_i
raise "ib_post_id is blank" if ib_post_id.blank?
if post = @shallow_posts_by_ib_post_id[ib_post_id]
@@ -117,8 +120,13 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
@shallow_posts_by_ib_post_id[ib_post_id] = post
is_new_post = post.new_record?
creator = upsert_user_from_submission_json!(submission_json)
if post.creator && post.creator.ib_user_id != creator.ib_user_id
creator =
upsert_user_from_submission_json!(
submission_json,
caused_by_entry: caused_by_entry,
)
if post.creator && post.creator.ib_user_id != creator&.ib_user_id
raise "post.creator.ib_user_id != creator.ib_user_id"
end
@@ -133,11 +141,16 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
post.submission_type = submission_json["submission_type_id"]&.to_i
post.ib_detail_raw["submission_json"] = submission_json
if post.changed? || post.files.count != post.num_files ||
post.creator.avatar_url_str.blank?
post_changed =
post.changed? || post.files.count != post.num_files ||
post.creator.avatar_url_str.blank?
if post_changed
post.shallow_update_log_entry = caused_by_entry
post.save!
@changed_posts << post
end
post.save! if post.changed?
is_new_post
end
@@ -147,7 +160,7 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
# Expected information from the endpoint (see fixture api_search.json):
# - username
# - user_id
def upsert_user_from_submission_json!(submission_json)
def upsert_user_from_submission_json!(submission_json, caused_by_entry: nil)
ib_user_id = submission_json["user_id"]&.to_i
if user = @users_by_ib_user_id[ib_user_id]
return user
@@ -156,7 +169,10 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
user = Domain::Inkbunny::User.find_or_initialize_by(ib_user_id: ib_user_id)
@users_by_ib_user_id[ib_user_id] = user
user.name = submission_json["username"]
user.save! if user.changed?
if user.changed?
user.shallow_update_log_entry = caused_by_entry
user.save!
end
user
end
end

View File

@@ -27,7 +27,11 @@ module Domain::Inkbunny::Job
fatal_error("api_search failed: #{response.status_code}")
end
result = processor.process!(JSON.parse(response.body))
result =
processor.process!(
JSON.parse(response.body),
caused_by_entry: response.log_entry,
)
num_new_posts = result[:num_new_posts]
logger.info(
[

View File

@@ -1,7 +1,7 @@
module Domain::Inkbunny::Job
class UpdatePostsJob < Base
def perform(args)
@caused_by_entry = args[:caused_by_entry]
caused_by_entry = args[:caused_by_entry]
ib_post_ids = args[:ib_post_ids]
if ib_post_ids.empty?
@@ -10,32 +10,34 @@ module Domain::Inkbunny::Job
end
ib_post_ids.each_slice(100) do |ib_post_ids_chunk|
process_ib_post_ids_chunk(ib_post_ids_chunk)
process_ib_post_ids_chunk(
ib_post_ids_chunk,
caused_by_entry: caused_by_entry,
)
end
end
def process_ib_post_ids_chunk(ib_post_ids_chunk)
def build_api_submissions_url(ib_post_ids_chunk)
ib_post_ids_list = ib_post_ids_chunk.join(",")
url =
"https://inkbunny.net/api_submissions.php?" +
"submission_ids=#{ib_post_ids_list}" +
"&show_description=yes&show_writing=yes&show_pools=yes"
@api_submissions_response =
http_client.get(url, caused_by_entry: @caused_by_entry)
@log_entry = @api_submissions_response.log_entry
if @api_submissions_response.status_code != 200
fatal_error(
"api_submissions failed: #{@api_submissions_response.status_code}",
)
"https://inkbunny.net/api_submissions.php?" +
"submission_ids=#{ib_post_ids_list}" +
"&show_description=yes&show_writing=yes&show_pools=yes"
end
def process_ib_post_ids_chunk(ib_post_ids_chunk, caused_by_entry:)
url = build_api_submissions_url(ib_post_ids_chunk)
response = http_client.get(url, caused_by_entry: caused_by_entry)
if response.status_code != 200
fatal_error("api_submissions failed: #{response.status_code}")
end
api_submissions_json = JSON.parse(@api_submissions_response.body)
api_submissions_json = JSON.parse(response.body)
submissions = api_submissions_json["submissions"]
logger.info("api_submissions page has #{submissions.size} posts")
submissions.each do |submission_json|
Domain::Inkbunny::Post.transaction do
deep_update_post_from_submission_json(
submission_json,
caused_by_entry: @log_entry,
caused_by_entry: response.log_entry,
)
end
end
@@ -54,13 +56,18 @@ module Domain::Inkbunny::Job
post.submission_type = submission_json["submission_type"]
post.num_views = submission_json["views"]
post.num_files = submission_json["pagecount"]
post.num_favs = submission_json["favorites_count"]&.to_i
post.num_comments = submission_json["comments_count"]&.to_i
post.last_file_updated_at =
Time.parse(submission_json["last_file_update_datetime"])
post.keywords = submission_json["keywords"]
post.deep_update_log_entry = caused_by_entry
if submission_json["user_icon_url_large"]
user = post.creator
user.avatar_url_str = submission_json["user_icon_url_large"]
if user.avatar_url_str_changed?
user.deep_update_log_entry = caused_by_entry
logger.info "avatar url changed, enqueuing download for user #{user.name}"
defer_job(
Domain::Inkbunny::Job::UserAvatarJob,

View File

@@ -40,7 +40,11 @@ module Domain::Inkbunny::Job
if response.status_code != 200
fatal_error("api_search failed: #{response.status_code}")
end
result = processor.process!(JSON.parse(response.body))
result =
processor.process!(
JSON.parse(response.body),
caused_by_entry: response.log_entry,
)
num_new_posts = result[:num_new_posts]
logger.info(
[

View File

@@ -10,10 +10,17 @@ module HasIndexedPost
autosave: true
before_create :ensure_indexed_post!
def ensure_indexed_post!
self.indexed_post ||=
IndexedPost.new(created_at: self.created_at, postable: self)
end
before_save :ensure_indexed_post_posted_at!
def ensure_indexed_post_posted_at!
if self.posted_at_changed?
self.ensure_indexed_post!
self.indexed_post.posted_at = self.posted_at
end
end
end
end

View File

@@ -6,6 +6,14 @@ class Domain::Inkbunny::Post < ReduxApplicationRecord
class_name: "::Domain::Inkbunny::User",
inverse_of: :posts
belongs_to :deep_update_log_entry,
class_name: "::HttpLogEntry",
optional: true
belongs_to :shallow_update_log_entry,
class_name: "::HttpLogEntry",
optional: true
has_many :files, class_name: "::Domain::Inkbunny::File", inverse_of: :post
enum :state, %i[ok error]

View File

@@ -17,6 +17,14 @@ class Domain::Inkbunny::User < ReduxApplicationRecord
foreign_key: :avatar_file_log_entry_id,
optional: true
belongs_to :deep_update_log_entry,
class_name: "::HttpLogEntry",
optional: true
belongs_to :shallow_update_log_entry,
class_name: "::HttpLogEntry",
optional: true
validates_presence_of :ib_user_id, :name
enum :state, %i[ok error]
enum :avatar_state, %i[ok not_found error], prefix: :avatar

View File

@@ -3,6 +3,12 @@ class IndexedPost < ReduxApplicationRecord
belongs_to :postable, polymorphic: true, inverse_of: :indexed_post
has_one :file, through: :postable
before_validation do
if self.attributes["posted_at"].nil?
self.attributes["posted_at"] = postable&.posted_at
self.posted_at = postable&.posted_at
end
end
def artist_name
case postable_type
@@ -12,6 +18,8 @@ class IndexedPost < ReduxApplicationRecord
array = postable&.tags_array
return unless array
array.is_a?(Hash) ? array["artist"].first : nil
when "Domain::Inkbunny::Post"
postable&.creator&.name
else
raise("Unsupported postable type: #{postable_type}")
end
@@ -27,6 +35,12 @@ class IndexedPost < ReduxApplicationRecord
end
when "Domain::E621::Post"
return nil
when "Domain::Inkbunny::Post"
if postable&.creator
Rails.application.routes.url_helpers.domain_inkbunny_user_path(
postable&.creator,
)
end
else
raise("Unsupported postable type: #{postable_type}")
end
@@ -46,16 +60,17 @@ class IndexedPost < ReduxApplicationRecord
end
def posted_at
case postable_type
when "Domain::Fa::Post"
postable&.posted_at
when "Domain::E621::Post"
postable&.posted_at
when "Domain::Inkbunny::Post"
postable&.posted_at
else
raise("Unsupported postable type: #{postable_type}")
end
super ||
case postable_type
when "Domain::Fa::Post"
postable&.posted_at
when "Domain::E621::Post"
postable&.posted_at
when "Domain::Inkbunny::Post"
postable&.posted_at
else
raise("Unsupported postable type: #{postable_type}")
end
end
def file_sha256
@@ -84,10 +99,12 @@ class IndexedPost < ReduxApplicationRecord
case postable_type
when "Domain::Fa::Post"
fa_id = postable&.fa_id
"FA #{fa_id}" if fa_id.present?
"FA ##{fa_id}" if fa_id.present?
when "Domain::E621::Post"
e621_id = postable&.e621_id
"E621 #{e621_id}" if e621_id.present?
"E621 ##{e621_id}" if e621_id.present?
when "Domain::Inkbunny::Post"
"IB ##{postable&.ib_post_id}" if postable&.ib_post_id.present?
else
raise("Unsupported postable type: #{postable_type}")
end
@@ -101,6 +118,9 @@ class IndexedPost < ReduxApplicationRecord
when "Domain::E621::Post"
e621_id = postable&.e621_id
"https://e621.net/posts/#{e621_id}" if e621_id.present?
when "Domain::Inkbunny::Post"
ib_post_id = postable&.ib_post_id
"https://inkbunny.net/s/#{ib_post_id}" if ib_post_id.present?
else
raise("Unsupported postable type: #{postable_type}")
end

View File

@@ -4,7 +4,11 @@
<div class="mx-auto">
<h2 class="mb-2 mt-4 text-xl">Posts (<%= @user.posts.count %>)</h2>
<div class="mx-2 flex-row">
<% @user.posts.each do |post| %>
<% @user
.posts
.order(posted_at: :desc)
.limit(10)
.each do |post| %>
<div class="border-stone-00 mb-4 rounded-md border-2 p-4">
<div class="mb-2 flex justify-between text-stone-800">
<div><%= link_to post.title, post, class: "hover:underline" %></div>
@@ -15,12 +19,16 @@
</div>
<div class="flex flex-row gap-2">
<% post.files.each do |file| %>
<% img_src_path =
blob_path(
HexUtil.bin2hex(file.blob_entry_sha256),
format: "jpg",
thumb: "small",
) %>
<% if file.blob_entry_sha256.present? %>
<% img_src_path =
blob_path(
HexUtil.bin2hex(file.blob_entry_sha256),
format: "jpg",
thumb: "small",
) %>
<% else %>
(no media file)
<% end %>
<div class="overflow-hidden rounded-md">
<img
class="h-32 p-2 first:pl-0 last:pr-0"

View File

@@ -1,8 +1,19 @@
<div
class="m-4 flex h-fit flex-col rounded-lg border border-slate-300 bg-slate-50 shadow-sm"
>
<div class="flex border-b border-slate-300 p-4">
<%= render partial: "inline_postable_domain_link", locals: { post: post } %>
<div class="flex justify-between border-b border-slate-300 p-4">
<div>
<%= render partial: "inline_postable_domain_link", locals: { post: post } %>
</div>
<div>
<% if post.artist_path.present? %>
<%= link_to post.artist_name,
post.artist_path,
class: "text-blue-600 hover:text-blue-800" %>
<% else %>
<%= post.artist_name %>
<% end %>
</div>
</div>
<div class="flex items-center justify-center p-4">

View File

@@ -0,0 +1,36 @@
class AddKeywordsToDomainInkbunnyPosts < ActiveRecord::Migration[7.2]
def change
change_table :domain_inkbunny_posts do |t|
t.references :deep_update_log_entry
t.references :shallow_update_log_entry
t.integer :num_favs
t.integer :num_comments
t.jsonb :keywords
end
change_table :domain_inkbunny_users do |t|
t.references :deep_update_log_entry
t.references :shallow_update_log_entry
end
add_foreign_key :domain_inkbunny_posts,
:http_log_entries,
column: :deep_update_log_entry_id,
primary_key: :id
add_foreign_key :domain_inkbunny_posts,
:http_log_entries,
column: :shallow_update_log_entry_id,
primary_key: :id
add_foreign_key :domain_inkbunny_users,
:http_log_entries,
column: :deep_update_log_entry_id,
primary_key: :id
add_foreign_key :domain_inkbunny_users,
:http_log_entries,
column: :shallow_update_log_entry_id,
primary_key: :id
end
end

View File

@@ -0,0 +1,9 @@
class AddPostedAtToIndexedPosts < ActiveRecord::Migration[7.2]
def change
change_table :indexed_posts do |t|
t.datetime :posted_at
end
add_index :indexed_posts, :posted_at
end
end

20
db/schema.rb generated
View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.2].define(version: 2024_12_31_061234) do
ActiveRecord::Schema[7.2].define(version: 2024_12_31_215756) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_prewarm"
enable_extension "pg_stat_statements"
@@ -1340,7 +1340,6 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_31_061234) do
t.boolean "removed", default: false, null: false
t.index ["post_id"], name: "index_domain_fa_favs_on_post_id"
t.index ["user_id", "post_id"], name: "index_domain_fa_favs_on_user_id_and_post_id", unique: true
t.index ["user_id"], name: "index_domain_fa_favs_on_user_id"
end
create_table "domain_fa_follows", id: false, force: :cascade do |t|
@@ -1518,7 +1517,14 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_31_061234) do
t.jsonb "ib_detail_raw"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.bigint "deep_update_log_entry_id"
t.bigint "shallow_update_log_entry_id"
t.integer "num_favs"
t.integer "num_comments"
t.jsonb "keywords"
t.index ["creator_id"], name: "index_domain_inkbunny_posts_on_creator_id"
t.index ["deep_update_log_entry_id"], name: "index_domain_inkbunny_posts_on_deep_update_log_entry_id"
t.index ["shallow_update_log_entry_id"], name: "index_domain_inkbunny_posts_on_shallow_update_log_entry_id"
end
create_table "domain_inkbunny_taggings", force: :cascade do |t|
@@ -1545,7 +1551,11 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_31_061234) do
t.integer "avatar_state"
t.jsonb "avatar_state_detail", default: {}, null: false
t.datetime "scanned_gallery_at"
t.bigint "deep_update_log_entry_id"
t.bigint "shallow_update_log_entry_id"
t.index ["deep_update_log_entry_id"], name: "index_domain_inkbunny_users_on_deep_update_log_entry_id"
t.index ["ib_user_id"], name: "index_domain_inkbunny_users_on_ib_user_id", unique: true
t.index ["shallow_update_log_entry_id"], name: "index_domain_inkbunny_users_on_shallow_update_log_entry_id"
end
create_table "domain_twitter_medias", id: false, force: :cascade do |t|
@@ -1746,8 +1756,10 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_31_061234) do
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.enum "postable_type", null: false, enum_type: "postable_type"
t.datetime "posted_at"
t.index ["created_at"], name: "index_indexed_posts_on_created_at"
t.index ["postable_id", "postable_type"], name: "index_indexed_posts_on_postable_id_and_postable_type", unique: true
t.index ["posted_at"], name: "index_indexed_posts_on_posted_at"
end
create_table "log_store_sst_entries", id: false, force: :cascade do |t|
@@ -1827,6 +1839,10 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_31_061234) do
add_foreign_key "domain_inkbunny_pool_joins", "domain_inkbunny_pools", column: "pool_id"
add_foreign_key "domain_inkbunny_pool_joins", "domain_inkbunny_posts", column: "post_id"
add_foreign_key "domain_inkbunny_posts", "domain_inkbunny_users", column: "creator_id"
add_foreign_key "domain_inkbunny_posts", "http_log_entries", column: "deep_update_log_entry_id"
add_foreign_key "domain_inkbunny_posts", "http_log_entries", column: "shallow_update_log_entry_id"
add_foreign_key "domain_inkbunny_users", "http_log_entries", column: "deep_update_log_entry_id"
add_foreign_key "domain_inkbunny_users", "http_log_entries", column: "shallow_update_log_entry_id"
add_foreign_key "domain_twitter_medias", "domain_twitter_tweets", column: "tweet_id"
add_foreign_key "domain_twitter_medias", "http_log_entries", column: "file_id"
add_foreign_key "domain_twitter_tweets", "domain_twitter_users", column: "author_id", primary_key: "tw_id", name: "on_author_id"

View File

@@ -50,6 +50,7 @@ module IndexedPostsRake
batch.each do |post|
post.ensure_indexed_post!
post.save!
post.indexed_post.save!
mutex.synchronize { progress.increment }
end
end

View File

@@ -53,6 +53,7 @@ describe Domain::Inkbunny::Job::LatestPostsJob do
expect(post_3104202.shallow_updated_at).to be_within(1.second).of(
Time.now,
)
expect(post_3104202.shallow_update_log_entry).to eq(log_entries[0])
expect(post_3104202.deep_updated_at).to be_nil
user_soulcentinel = Domain::Inkbunny::User.find_by!(ib_user_id: 349_747)

View File

@@ -93,6 +93,15 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
expect(post_3104200.description).to match(/Some requests from my Patreon/)
expect(post_3104200.writing).to eq("")
expect(post_3104200.num_views).to eq(690)
expect(post_3104200.num_favs).to eq(139)
expect(post_3104200.num_comments).to eq(2)
expect(post_3104200.keywords).to match(
array_including(
including("keyword_id" => "128", "keyword_name" => "husky"),
including("keyword_id" => "32715", "keyword_name" => "direwolf"),
),
)
expect(post_3104200.deep_update_log_entry).to eq(log_entries[0])
expect(post_3104200.deep_updated_at).to be_within(1.second).of(Time.now)
# Check user details were updated

View File

@@ -17,7 +17,7 @@ RSpec.describe Domain::Inkbunny::Job::UserGalleryJob do
end
context "when fetching posts" do
before do
let!(:log_entries) do
SpecUtil.init_http_client_mock(
http_client_mock,
[
@@ -61,11 +61,13 @@ RSpec.describe Domain::Inkbunny::Job::UserGalleryJob do
expect(user.name).to eq("Zaush")
expect(user.scanned_gallery_at).to be_present
expect(user.posts.count).to eq(4)
expect(user.shallow_update_log_entry).to eq(log_entries[0])
post_3507105 = user.posts.find_by(ib_post_id: 350_7105)
expect(post_3507105).to be_present
expect(post_3507105.num_files).to eq(5)
expect(post_3507105.files.count).to eq(0)
expect(post_3507105.shallow_update_log_entry).to eq(log_entries[0])
end
end
end

View File

@@ -166,4 +166,83 @@ RSpec.describe Domain::Fa::Post do
expect(post.indexed_post.postable_type).to eq("Domain::Fa::Post")
expect(post.indexed_post.postable).to eq(post)
end
describe "posted_at synchronization" do
let(:post) { create(:domain_fa_post) }
it "updates indexed_post.posted_at when post.posted_at changes from nil" do
expect(post.indexed_post.posted_at).to be_nil
new_time = Time.current
post.posted_at = new_time
post.save!
expect(post.indexed_post.posted_at).to be_within(1.second).of(new_time)
end
it "updates indexed_post.posted_at when post.posted_at changes to a different value" do
initial_time = 1.day.ago
post.update!(posted_at: initial_time)
expect(post.indexed_post.posted_at).to be_within(1.second).of(
initial_time,
)
new_time = Time.current
post.posted_at = new_time
post.save!
expect(post.indexed_post.posted_at).to be_within(1.second).of(new_time)
end
it "updates indexed_post.posted_at when post.posted_at changes to nil" do
initial_time = Time.current
post.update!(posted_at: initial_time)
expect(post.indexed_post.posted_at).to be_within(1.second).of(
initial_time,
)
post.posted_at = nil
post.save!
expect(post.indexed_post.posted_at).to be_nil
end
it "indexed_post uses posted_at of postable when its own field is nil" do
initial_time = 1.day.ago
post.update!(posted_at: initial_time)
post.indexed_post.update!(posted_at: nil)
post.reload
expect(post.indexed_post.posted_at).to be_within(1.second).of(
initial_time,
)
end
it "indexed_post uses posted_at if it is set, even if out of sync" do
initial_time = 1.day.ago
post.update!(posted_at: nil)
post.indexed_post.update!(posted_at: initial_time)
post.reload
expect(post.indexed_post.posted_at).to be_within(1.second).of(
initial_time,
)
end
it "gets the posted_at from the postable if its nil before save" do
initial_time = 1.day.ago
post.update_column(:posted_at, initial_time)
post.indexed_post.update_column(:posted_at, nil)
post.reload
# attribute is nil, but posted_at is via postable
expect(post.indexed_post.attributes["posted_at"]).to be_nil
expect(post.indexed_post.posted_at).to be_within(1.second).of(
initial_time,
)
post.indexed_post.save!
expect(post.indexed_post.attributes["posted_at"]).to be_within(
1.second,
).of(initial_time)
end
end
end