dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate e621 jobs to unified domain models

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-12 18:35:40 +00:00
parent 512119ebb4
commit c2cbe78fd1
44 changed files with 940 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@@ -35,6 +35,7 @@ RUN \
# preinstall gems that take a long time to install
RUN MAKE="make -j12" gem install bundler -v '2.5.6' --verbose
RUN MAKE="make -j12" gem install rice -v '4.3.3' --verbose
RUN MAKE="make -j12" gem install faiss -v '0.3.2' --verbose
RUN MAKE="make -j12" gem install rails_live_reload -v '0.3.6' --verbose
RUN bundle config --global frozen 1

61
Rakefile
View File

@@ -110,6 +110,7 @@ end
task infer_last_submission_log_entries: :environment do
only_fa_id = ENV["only_fa_id"]
start = ENV["start_at"]&.to_i || nil
if only_fa_id
relation = Domain::Fa::Post.where(fa_id: only_fa_id)
@@ -121,33 +122,43 @@ task infer_last_submission_log_entries: :environment do
.or(Domain::Fa::Post.where(state: :ok).where(posted_at: nil))
end
relation.find_each do |post|
relation.find_each(batch_size: 10, start:) do |post|
parts = ["[id: #{post.id}]", "[fa_id: #{post.fa_id}]"]
log_entry = post.guess_last_submission_page
if log_entry
contents = log_entry.response&.contents
if contents
parser = Domain::Fa::Parser::Page.new(contents)
if parser.submission_not_found?
parts << "[removed]"
post.state = :removed
else
posted_at = parser.submission.posted_date
parts << "[posted_at_parser: #{posted_at}]" if posted_at
end
end
if post.last_submission_page_id.present? &&
log_entry.id != post.last_submission_page_id
parts << "[overwrite]"
end
post.last_submission_page_id = log_entry.id
posted_at = post.posted_at ||= post.guess_posted_at
parts << "[posted_at_attr: #{posted_at}]" if posted_at
parts << "[submission log entry: #{log_entry.id}]"
parts << "[uri: #{log_entry.uri.to_s}]"
puts parts.join(" ")
post.save!
log_entry = post.guess_last_submission_page
unless log_entry
parts << "[no log entry]"
next
end
contents = log_entry.response&.contents
unless contents
parts << "[no contents]"
next
end
parser = Domain::Fa::Parser::Page.new(contents)
if parser.submission_not_found?
parts << "[removed]"
post.state = :removed
else
posted_at = parser.submission.posted_date
post.posted_at ||= posted_at
parts << "[posted at: #{posted_at}]"
end
if post.last_submission_page_id.present? &&
log_entry.id != post.last_submission_page_id
parts << "[overwrite]"
end
post.last_submission_page_id = log_entry.id
parts << "[log entry: #{log_entry.id}]"
parts << "[uri: #{log_entry.uri.to_s}]"
post.save!
rescue => e
parts << "[error: #{e.message}]"
ensure
puts parts.join(" ")
end
end

6
app/jobs/domain/e621/job/base.rb
View File

@@ -7,13 +7,13 @@ class Domain::E621::Job::Base < Scraper::JobBase
:get_e621_http_client
end
sig { returns(Domain::E621::User) }
sig { returns(Domain::User::E621User) }
def user_from_args!
T.must(user_from_args)
end
sig { returns(T.nilable(Domain::E621::User)) }
sig { returns(T.nilable(Domain::User::E621User)) }
def user_from_args
T.cast(arguments[0][:user], T.nilable(Domain::E621::User))
T.cast(arguments[0][:user], T.nilable(Domain::User::E621User))
end
end

2
app/jobs/domain/e621/job/posts_index_job.rb
View File

@@ -2,7 +2,7 @@
class Domain::E621::Job::PostsIndexJob < Domain::E621::Job::Base
queue_as :e621
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
response = http_client.get("https://e621.net/posts.json")

22
app/jobs/domain/e621/job/scan_post_favs_job.rb
View File

@@ -10,9 +10,9 @@ class Domain::E621::Job::ScanPostFavsJob < Domain::E621::Job::Base
const :num_other_favs, Integer
end
sig { override.params(args: T.untyped).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
post = T.cast(args[:post], Domain::E621::Post)
post = T.cast(args[:post], Domain::Post::E621Post)
page = 1
breaker = 0
total_created_users = 0
@@ -32,9 +32,9 @@ class Domain::E621::Job::ScanPostFavsJob < Domain::E621::Job::Base
rows = html.css("tbody tr")
rows.each do |row_elem|
user_member_elem = row_elem.css("td:first-child a")&.first
e621_user_id = user_member_elem["href"].split("/").last.to_i
e621_id_to_user_row[e621_user_id] = UserRow.new(
e621_id: e621_user_id,
e621_id = user_member_elem["href"].split("/").last.to_i
e621_id_to_user_row[e621_id] = UserRow.new(
e621_id: e621_id,
name: user_member_elem.text,
num_other_favs: row_elem.css("td:last-child").text.to_i,
)
@@ -43,16 +43,16 @@ class Domain::E621::Job::ScanPostFavsJob < Domain::E621::Job::Base
ReduxApplicationRecord.transaction do
e621_id_to_user =
T.cast(
Domain::E621::User.where(
e621_user_id: e621_id_to_user_row.keys,
).index_by(&:e621_user_id),
T::Hash[Integer, Domain::E621::User],
Domain::User::E621User.where(
e621_id: e621_id_to_user_row.keys,
).index_by(&:e621_id),
T::Hash[Integer, Domain::User::E621User],
)
e621_id_to_user_row.values.each do |user_row|
user =
e621_id_to_user[user_row.e621_id] ||
Domain::E621::User.new(
e621_user_id: user_row.e621_id,
Domain::User::E621User.new(
e621_id: user_row.e621_id,
name: user_row.name,
)
user.num_other_favs_cached = user_row.num_other_favs

19
app/jobs/domain/e621/job/scan_post_job.rb
View File

@@ -2,29 +2,25 @@
class Domain::E621::Job::ScanPostJob < Domain::E621::Job::Base
queue_as :e621
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
post = T.let(args[:post] || raise("no post provided"), Domain::E621::Post)
post =
T.let(args[:post] || raise("no post provided"), Domain::Post::E621Post)
logger.prefix =
proc { "[e621_id #{post.e621_id.to_s.bold} / #{post.state&.bold}]" }
if post.file.present?
file = post.file
if file.present? && file.state == "ok" && file.log_entry_id.present?
logger.warn("Post #{post.e621_id} already has a file")
return
end
if post.file_url_str.present?
logger.error("Post #{post.e621_id} already has a file URL")
return
end
logger.info("Scanning post #{post.e621_id}")
response = http_client.get("https://e621.net/posts/#{post.e621_id}.json")
log_entry = response.log_entry
post.scan_log_entry = response.log_entry
if response.status_code != 200
post.scan_log_entry_id = log_entry.id
post.state = :scan_error
post.state = "scan_error"
post.scan_error =
"Error scanning post #{post.e621_id}: #{response.status_code}"
post.save!
@@ -40,5 +36,6 @@ class Domain::E621::Job::ScanPostJob < Domain::E621::Job::Base
caused_by_entry: causing_log_entry,
)
post.save!
post.file&.save!
end
end

12
app/jobs/domain/e621/job/scan_user_favs_job.rb
View File

@@ -4,11 +4,11 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
MAX_PER_PAGE = T.let(Rails.env.test? ? 4 : 320, Integer)
include HasMeasureDuration
sig { override.params(args: T.untyped).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
user = user_from_args!
if user.scanned_favs_status == "error" && !args[:force]
logger.info("[user #{user.e621_user_id} has error status, skipping]")
logger.info("[user #{user.e621_id} has error status, skipping]")
return
end
@@ -18,7 +18,7 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
total_new_posts = 0
prefix = [
"[e621 user id: #{user.e621_user_id&.to_s&.bold}]",
"[e621 user id: #{user.e621_id&.to_s&.bold}]",
"[username: #{user.name&.bold}]",
].join(" ")
@@ -134,9 +134,9 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
measure("#{prefix} [upserting favs: #{post_ids.size}]") do
post_ids.each_slice(1000) do |slice|
ReduxApplicationRecord.transaction do
Domain::E621::Fav.upsert_all(
Domain::UserPostFav.upsert_all(
slice.map { |post_id| { user_id: user.id, post_id: post_id } },
unique_by: :index_domain_e621_favs_on_user_id_and_post_id,
unique_by: :index_domain_user_post_favs_on_user_id_and_post_id,
)
end
end
@@ -156,7 +156,7 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
user.scanned_favs_at = DateTime.current
user.save!
rescue StandardError
logger.error("error scanning user favs: #{user&.e621_user_id}")
logger.error("error scanning user favs: #{user&.e621_id}")
user = user_from_args
if user
user.scanned_favs_status = "error"

6
app/jobs/domain/e621/job/scan_users_job.rb
View File

@@ -1,6 +1,6 @@
# typed: strict
class Domain::E621::Job::ScanUsersJob < Domain::E621::Job::Base
sig { override.params(args: T.untyped).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
after = T.let(args[:after_e621_id], T.nilable(String))
breaker = 0
@@ -27,8 +27,8 @@ class Domain::E621::Job::ScanUsersJob < Domain::E621::Job::Base
ReduxApplicationRecord.transaction do
users_json.each do |user_json|
user =
Domain::E621::User.find_or_initialize_by(
e621_user_id: user_json["id"],
Domain::User::E621User.find_or_initialize_by(
e621_id: user_json["id"],
) { |user| user.name = user_json["name"] }
is_new = user.new_record?
num_new_users += 1 if is_new

65
app/jobs/domain/e621/job/static_file_job.rb
View File

@@ -2,50 +2,55 @@
class Domain::E621::Job::StaticFileJob < Domain::E621::Job::Base
queue_as :static_file
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
post =
T.let(args[:post] || fatal_error("post is required"), Domain::E621::Post)
post_file =
T.let(
args[:post_file] || fatal_error("post_file is required"),
Domain::PostFile,
)
post = T.cast(post_file.post, Domain::Post::E621Post)
logger.prefix = proc { "[e621_id #{post.e621_id.to_s.bold}]" }
file_url_str = post.file_url_str
if file_url_str.blank?
logger.warn("post has no file_url_str, enqueueing for scan")
defer_job(Domain::E621::Job::ScanPostJob, { post: post })
if post_file.state == "terminal_error"
logger.error("post file is in a terminal error state, skipping")
return
end
if post.state == "file_error"
retry_count = post.file_error&.retry_count || 0
if retry_count >= 3
logger.error("file has been retried 3 times, giving up")
return
end
if post_file.state == "ok" && post_file.log_entry_id.present?
logger.warn("post file has already been downloaded, skipping")
return
end
file_url_str = post_file.url_str
if file_url_str.blank?
logger.warn("post file has no url, skipping")
return
end
response = http_client.get(file_url_str)
post_file.log_entry = response.log_entry
post_file.last_status_code = response.status_code
if response.status_code != 200
post.state = :file_error
fe = (post.file_error ||= Domain::E621::Post::FileError.new)
fe.status_code = response.status_code
fe.log_entry_id = response.log_entry.id
fe.retry_count = (fe.retry_count || 0) + 1
post.save!
if response.status_code == 404
logger.error("#{response.status_code}, not retrying download")
if response.status_code == 200
post_file.state = "ok"
logger.info "downloaded file"
elsif response.status_code == 404
post_file.state = "terminal_error"
post_file.retry_count += 1
logger.error("#{response.status_code}, not retrying download")
else
post_file.retry_count += 1
if post_file.retry_count >= 3
post_file.state = "terminal_error"
logger.error("file has been retried 3 times, giving up")
else
post_file.state = "retryable_error"
fatal_error("#{response.status_code}, will retry later")
end
return
end
post.state = :ok
post.file = response.log_entry
post.save!
logger.info "downloaded file"
ensure
post_file.save! if post_file
end
end

2
app/jobs/domain/fa/job/browse_page_job.rb
View File

@@ -10,7 +10,7 @@ class Domain::Fa::Job::BrowsePageJob < Domain::Fa::Job::Base
@total_num_posts_seen = T.let(0, Integer)
end
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
while true
break unless scan_browse_page

2
app/jobs/domain/fa/job/favs_job.rb
View File

@@ -19,7 +19,7 @@ class Domain::Fa::Job::FavsJob < Domain::Fa::Job::Base
@use_http_cache = T.let(false, T::Boolean)
end
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
@first_job_entry = nil
user = init_from_args!(args, build_user: false)

2
app/jobs/domain/fa/job/scan_file_job.rb
View File

@@ -2,7 +2,7 @@
class Domain::Fa::Job::ScanFileJob < Domain::Fa::Job::Base
queue_as :static_file
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
file =
T.cast(args[:post_file], T.nilable(Domain::PostFile)) ||

2
app/jobs/domain/fa/job/scan_post_job.rb
View File

@@ -2,7 +2,7 @@
class Domain::Fa::Job::ScanPostJob < Domain::Fa::Job::Base
queue_as :fa_post
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
post =
T.cast(

2
app/jobs/domain/fa/job/user_avatar_job.rb
View File

@@ -3,7 +3,7 @@ class Domain::Fa::Job::UserAvatarJob < Domain::Fa::Job::Base
extend T::Sig
queue_as :static_file
sig { override.params(args: T.untyped).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
init_from_args!(args, build_user: false, set_user: false)
avatar =

2
app/jobs/domain/fa/job/user_follows_job.rb
View File

@@ -19,7 +19,7 @@ class Domain::Fa::Job::UserFollowsJob < Domain::Fa::Job::Base
@scanned_followed_ids = T.let(Set.new, T::Set[Integer])
end
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
init_from_args!(args)
user = T.must(@user)

2
app/jobs/domain/fa/job/user_gallery_job.rb
View File

@@ -18,7 +18,7 @@ class Domain::Fa::Job::UserGalleryJob < Domain::Fa::Job::Base
@folders = T.let(Set.new, T::Set[Folder])
end
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
init_from_args!(args)
user = T.must(@user)

2
app/jobs/domain/fa/job/user_incremental_job.rb
View File

@@ -3,7 +3,7 @@ module Domain::Fa::Job
class UserIncrementalJob < Base
queue_as :fa_user_page
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
init_from_args!(args)
user = T.must(@user)

2
app/jobs/domain/fa/job/user_page_job.rb
View File

@@ -3,7 +3,7 @@ class Domain::Fa::Job::UserPageJob < Domain::Fa::Job::Base
ScanUserUtils = Domain::Fa::Job::ScanUserUtils
queue_as :fa_user_page
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
init_from_args!(args)
user = T.must(@user)

2
app/jobs/domain/inkbunny/job/file_job.rb
View File

@@ -3,7 +3,7 @@ module Domain::Inkbunny::Job
class FileJob < Base
queue_as :static_file
sig { override.params(args: T.untyped).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
file = args[:file] || fatal_error("file is required")
logger.prefix =

2
app/jobs/domain/inkbunny/job/latest_posts_job.rb
View File

@@ -1,7 +1,7 @@
# typed: strict
module Domain::Inkbunny::Job
class LatestPostsJob < Base
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
processor = ApiSearchPageProcessor.new
first_log_entry = nil

2
app/jobs/domain/inkbunny/job/update_pool_job.rb
View File

@@ -1,7 +1,7 @@
# typed: strict
module Domain::Inkbunny::Job
class UpdatePoolJob < Base
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
pool = T.let(args[:pool], Domain::Inkbunny::Pool)
logger.prefix = "[pool #{pool.ib_pool_id.to_s.bold}]"

2
app/jobs/domain/inkbunny/job/update_posts_job.rb
View File

@@ -1,6 +1,6 @@
# typed: strict
class Domain::Inkbunny::Job::UpdatePostsJob < Domain::Inkbunny::Job::Base
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
ib_post_ids = args[:ib_post_ids]

2
app/jobs/domain/inkbunny/job/user_avatar_job.rb
View File

@@ -8,7 +8,7 @@ module Domain::Inkbunny::Job
super(*T.unsafe(args))
end
sig { override.params(args: T.untyped).void }
sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
user = user_from_args!

2
app/jobs/scraper/job_base.rb
View File

@@ -90,7 +90,7 @@ class Scraper::JobBase < ApplicationJob
@gallery_dl_client ||= Scraper::ClientFactory.get_gallery_dl_client
end
sig { abstract.params(args: T::Hash[Symbol, T.untyped]).void }
sig { abstract.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
def perform(args)
end

31
app/lib/domain/e621/tag_util.rb
View File

@@ -9,12 +9,12 @@ class Domain::E621::TagUtil
params(
post_json: T::Hash[String, T.untyped],
caused_by_entry: T.nilable(ReduxApplicationRecord),
).returns(Domain::E621::Post)
).returns(Domain::Post::E621Post)
end
def self.initialize_or_update_post(post_json:, caused_by_entry: nil)
# create all posts that don't already exist
e621_id = T.cast(post_json["id"], Integer)
e621_post = Domain::E621::Post.find_or_initialize_by(e621_id: e621_id)
e621_post = Domain::Post::E621Post.find_or_initialize_by(e621_id: e621_id)
e621_updated_at = post_json["updated_at"]
return e621_post if e621_post.e621_updated_at == e621_updated_at
@@ -29,20 +29,26 @@ class Domain::E621::TagUtil
"md5 changed for post: #{e621_post.md5.to_s.bold} => #{e621_md5.to_s.bold}",
)
e621_post.prev_md5s ||= []
e621_post.prev_md5s << {
"md5" => e621_post.md5,
"file_id" => e621_post.file_id,
}
e621_post.prev_md5s << e621_post.md5
e621_post.file = nil
end
e621_post.md5 = e621_md5
e621_post.file_url_str =
file =
e621_post.file ||
e621_post.build_file do |new_file|
new_file = T.cast(new_file, Domain::PostFile)
new_file.enqueue_job_after_save(
Domain::E621::Job::StaticFileJob,
{ post_file: new_file, caused_by_entry: },
)
end
file.url_str =
begin
file = post_json["file"]
file["url"] ||
"https://static1.e621.net/data/#{e621_md5[0...2]}/#{e621_md5[2...4]}/#{e621_md5}.#{file["ext"]}"
end
e621_post.md5 = e621_md5
e621_post.posted_at = post_json["created_at"]
e621_post.description = post_json["description"]
e621_post.rating = post_json["rating"]
@@ -52,7 +58,7 @@ class Domain::E621::TagUtil
e621_post.num_favorites = post_json["fav_count"]
e621_post.num_comments = post_json["comment_count"]
e621_post.change_seq = post_json["change_seq"]
e621_post.parent_e621_id = post_json["relationships"]["parent_id"]
e621_post.parent_post_e621_id = post_json["relationships"]["parent_id"]
e621_post.flags_array =
post_json["flags"].to_a.select(&:second).map(&:first)
@@ -60,13 +66,6 @@ class Domain::E621::TagUtil
e621_post.sources_array = post_json["sources"]
e621_post.tags_array = post_json["tags"]
if e621_post.md5_changed? && e621_post.md5.present?
e621_post.enqueue_job_after_save(
Domain::E621::Job::StaticFileJob,
{ post: e621_post, caused_by_entry: },
)
end
e621_post
end
end

16
app/lib/domain/migrate_to_domain.rb
View File

@@ -577,19 +577,29 @@ class Domain::MigrateToDomain
new_post.caused_by_entry_id = old_post.caused_by_entry_id
new_post.scan_log_entry_id = old_post.scan_log_entry_id
new_post.index_page_ids = old_post.index_page_ids
new_post.md5 = old_post.md5
new_post.prev_md5s = old_post.prev_md5s
new_post.scan_error = old_post.scan_error
new_post.file_error = old_post.file_error
new_post.created_at = old_post.created_at
new_post.parent_post_e621_id = old_post.parent_e621_id
new_post.description = old_post.description
new_post.rating = old_post.rating
new_post.score = old_post.score
new_post.score_up = old_post.score_up
new_post.score_down = old_post.score_down
new_post.num_favorites = old_post.num_favorites
new_post.num_comments = old_post.num_comments
new_post.change_seq = old_post.change_seq
old_file = old_post.file
file_url_str = old_post.file_url_str
if old_file || file_url_str
new_file = Domain::PostFile.new
new_file.url_str = file_url_str
new_file.log_entry = old_file
new_file.last_status_code = old_file&.status_code
new_file.log_entry_id = old_file&.id || old_post.file_error&.log_entry_id
new_file.last_status_code =
old_file&.status_code || old_post.file_error&.status_code
new_file.retry_count = old_post.file_error&.retry_count
if old_file.present? && old_file.status_code == 200
new_file.state = "ok"
elsif old_file.present?

25
app/models/domain/post/e621_post.rb
View File

@@ -1,12 +1,5 @@
# typed: strict
class Domain::Post::E621Post < Domain::Post
class FileError
include AttrJson::Model
attr_json :retry_count, :integer
attr_json :status_code, :integer
attr_json :log_entry_id, :integer
end
attr_json :state, :string
attr_json :e621_id, :integer
# When was the post's /posts/<post_id>/favorites pages scanned?
@@ -26,9 +19,16 @@ class Domain::Post::E621Post < Domain::Post
attr_json :caused_by_entry_id, :integer
attr_json :scan_log_entry_id, :integer
attr_json :index_page_ids, :integer, array: true
attr_json :description, :string
attr_json :score, :integer
attr_json :score_up, :integer
attr_json :score_down, :integer
attr_json :num_favorites, :integer
attr_json :num_comments, :integer
attr_json :change_seq, :integer
attr_json :md5, :string
attr_json :prev_md5s, :string, array: true
attr_json :scan_error, :string
attr_json :file_error, FileError.to_type
attr_json :uploader_user_id, :integer
has_single_file!
@@ -49,9 +49,14 @@ class Domain::Post::E621Post < Domain::Post
optional: true
belongs_to :last_index_page, class_name: "HttpLogEntry", optional: true
belongs_to :scan_log_entry, class_name: "HttpLogEntry", optional: true
validates :state, inclusion: { in: %w[ok removed scan_error file_error] }
validates :rating, inclusion: { in: %w[s q e] }
validates :rating,
inclusion: {
in: %w[s q e],
},
if: ->(p) { p.rating.present? }
validates :e621_id, presence: true
after_initialize do

7
app/models/domain/user/e621_user.rb
View File

@@ -52,4 +52,11 @@ class Domain::User::E621User < Domain::User
def names_for_search
[name].compact
end
sig { returns(T.nilable(::String)) }
def url_name
if name = self.name
name.gsub(" ", "_")
end
end
end

543
sorbet/rbi/dsl/domain/post/e621_post.rbi generated
View File

@@ -11,9 +11,6 @@ class Domain::Post::E621Post
extend CommonRelationMethods
extend GeneratedRelationMethods
sig { returns(T.nilable(Domain::Post::E621Post::FileError)) }
def file_error; end
sig { returns(ColorLogger) }
def logger; end
@@ -458,6 +455,9 @@ class Domain::Post::E621Post
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Post::E621Post) }
def build_parent_post(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
def build_scan_log_entry(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::User::E621User) }
def build_uploader_user(*args, &blk); end
@@ -485,6 +485,12 @@ class Domain::Post::E621Post
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Post::E621Post) }
def create_parent_post!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
def create_scan_log_entry(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
def create_scan_log_entry!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::User::E621User) }
def create_uploader_user(*args, &blk); end
@@ -595,6 +601,9 @@ class Domain::Post::E621Post
sig { returns(T.nilable(::Domain::Post::E621Post)) }
def reload_parent_post; end
sig { returns(T.nilable(::HttpLogEntry)) }
def reload_scan_log_entry; end
sig { returns(T.nilable(::Domain::User::E621User)) }
def reload_uploader_user; end
@@ -610,9 +619,24 @@ class Domain::Post::E621Post
sig { void }
def reset_parent_post; end
sig { void }
def reset_scan_log_entry; end
sig { void }
def reset_uploader_user; end
sig { returns(T.nilable(::HttpLogEntry)) }
def scan_log_entry; end
sig { params(value: T.nilable(::HttpLogEntry)).void }
def scan_log_entry=(value); end
sig { returns(T::Boolean) }
def scan_log_entry_changed?; end
sig { returns(T::Boolean) }
def scan_log_entry_previously_changed?; end
sig { returns(T.nilable(::Domain::User::E621User)) }
def uploader_user; end
@@ -889,6 +913,51 @@ class Domain::Post::E621Post
sig { void }
def caused_by_entry_id_will_change!; end
sig { returns(T.nilable(::Integer)) }
def change_seq; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def change_seq=(value); end
sig { returns(T::Boolean) }
def change_seq?; end
sig { returns(T.nilable(::Integer)) }
def change_seq_before_last_save; end
sig { returns(T.untyped) }
def change_seq_before_type_cast; end
sig { returns(T::Boolean) }
def change_seq_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def change_seq_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def change_seq_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def change_seq_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def change_seq_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def change_seq_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def change_seq_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def change_seq_previously_was; end
sig { returns(T.nilable(::Integer)) }
def change_seq_was; end
sig { void }
def change_seq_will_change!; end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def created_at; end
@@ -944,6 +1013,51 @@ class Domain::Post::E621Post
sig { void }
def created_at_will_change!; end
sig { returns(T.nilable(::String)) }
def description; end
sig { params(value: T.nilable(::String)).returns(T.nilable(::String)) }
def description=(value); end
sig { returns(T::Boolean) }
def description?; end
sig { returns(T.nilable(::String)) }
def description_before_last_save; end
sig { returns(T.untyped) }
def description_before_type_cast; end
sig { returns(T::Boolean) }
def description_came_from_user?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def description_change; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def description_change_to_be_saved; end
sig { params(from: T.nilable(::String), to: T.nilable(::String)).returns(T::Boolean) }
def description_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::String)) }
def description_in_database; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def description_previous_change; end
sig { params(from: T.nilable(::String), to: T.nilable(::String)).returns(T::Boolean) }
def description_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::String)) }
def description_previously_was; end
sig { returns(T.nilable(::String)) }
def description_was; end
sig { void }
def description_will_change!; end
sig { returns(T.nilable(::Integer)) }
def e621_id; end
@@ -1044,51 +1158,6 @@ class Domain::Post::E621Post
sig { void }
def e621_updated_at_will_change!; end
sig { returns(T.untyped) }
def file_error; end
sig { params(value: T.untyped).returns(T.untyped) }
def file_error=(value); end
sig { returns(T::Boolean) }
def file_error?; end
sig { returns(T.untyped) }
def file_error_before_last_save; end
sig { returns(T.untyped) }
def file_error_before_type_cast; end
sig { returns(T::Boolean) }
def file_error_came_from_user?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def file_error_change; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def file_error_change_to_be_saved; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def file_error_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def file_error_in_database; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def file_error_previous_change; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def file_error_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def file_error_previously_was; end
sig { returns(T.untyped) }
def file_error_was; end
sig { void }
def file_error_will_change!; end
sig { returns(T.untyped) }
def flags_array; end
@@ -1404,6 +1473,141 @@ class Domain::Post::E621Post
sig { void }
def last_submission_log_entry_id_will_change!; end
sig { returns(T.nilable(::String)) }
def md5; end
sig { params(value: T.nilable(::String)).returns(T.nilable(::String)) }
def md5=(value); end
sig { returns(T::Boolean) }
def md5?; end
sig { returns(T.nilable(::String)) }
def md5_before_last_save; end
sig { returns(T.untyped) }
def md5_before_type_cast; end
sig { returns(T::Boolean) }
def md5_came_from_user?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def md5_change; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def md5_change_to_be_saved; end
sig { params(from: T.nilable(::String), to: T.nilable(::String)).returns(T::Boolean) }
def md5_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::String)) }
def md5_in_database; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def md5_previous_change; end
sig { params(from: T.nilable(::String), to: T.nilable(::String)).returns(T::Boolean) }
def md5_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::String)) }
def md5_previously_was; end
sig { returns(T.nilable(::String)) }
def md5_was; end
sig { void }
def md5_will_change!; end
sig { returns(T.nilable(::Integer)) }
def num_comments; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def num_comments=(value); end
sig { returns(T::Boolean) }
def num_comments?; end
sig { returns(T.nilable(::Integer)) }
def num_comments_before_last_save; end
sig { returns(T.untyped) }
def num_comments_before_type_cast; end
sig { returns(T::Boolean) }
def num_comments_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def num_comments_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def num_comments_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def num_comments_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def num_comments_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def num_comments_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def num_comments_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def num_comments_previously_was; end
sig { returns(T.nilable(::Integer)) }
def num_comments_was; end
sig { void }
def num_comments_will_change!; end
sig { returns(T.nilable(::Integer)) }
def num_favorites; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def num_favorites=(value); end
sig { returns(T::Boolean) }
def num_favorites?; end
sig { returns(T.nilable(::Integer)) }
def num_favorites_before_last_save; end
sig { returns(T.untyped) }
def num_favorites_before_type_cast; end
sig { returns(T::Boolean) }
def num_favorites_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def num_favorites_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def num_favorites_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def num_favorites_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def num_favorites_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def num_favorites_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def num_favorites_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def num_favorites_previously_was; end
sig { returns(T.nilable(::Integer)) }
def num_favorites_was; end
sig { void }
def num_favorites_will_change!; end
sig { returns(T.nilable(::Integer)) }
def parent_post_e621_id; end
@@ -1645,18 +1849,21 @@ class Domain::Post::E621Post
sig { void }
def restore_caused_by_entry_id!; end
sig { void }
def restore_change_seq!; end
sig { void }
def restore_created_at!; end
sig { void }
def restore_description!; end
sig { void }
def restore_e621_id!; end
sig { void }
def restore_e621_updated_at!; end
sig { void }
def restore_file_error!; end
sig { void }
def restore_flags_array!; end
@@ -1678,6 +1885,15 @@ class Domain::Post::E621Post
sig { void }
def restore_last_submission_log_entry_id!; end
sig { void }
def restore_md5!; end
sig { void }
def restore_num_comments!; end
sig { void }
def restore_num_favorites!; end
sig { void }
def restore_parent_post_e621_id!; end
@@ -1702,6 +1918,15 @@ class Domain::Post::E621Post
sig { void }
def restore_scanned_post_favs_at!; end
sig { void }
def restore_score!; end
sig { void }
def restore_score_down!; end
sig { void }
def restore_score_up!; end
sig { void }
def restore_sources_array!; end
@@ -1732,12 +1957,24 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def saved_change_to_caused_by_entry_id?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_change_seq; end
sig { returns(T::Boolean) }
def saved_change_to_change_seq?; end
sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
def saved_change_to_created_at; end
sig { returns(T::Boolean) }
def saved_change_to_created_at?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def saved_change_to_description; end
sig { returns(T::Boolean) }
def saved_change_to_description?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_e621_id; end
@@ -1750,12 +1987,6 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def saved_change_to_e621_updated_at?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_file_error; end
sig { returns(T::Boolean) }
def saved_change_to_file_error?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_flags_array; end
@@ -1798,6 +2029,24 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def saved_change_to_last_submission_log_entry_id?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def saved_change_to_md5; end
sig { returns(T::Boolean) }
def saved_change_to_md5?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_num_comments; end
sig { returns(T::Boolean) }
def saved_change_to_num_comments?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_num_favorites; end
sig { returns(T::Boolean) }
def saved_change_to_num_favorites?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_parent_post_e621_id; end
@@ -1846,6 +2095,24 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def saved_change_to_scanned_post_favs_at?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_score; end
sig { returns(T::Boolean) }
def saved_change_to_score?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_score_down; end
sig { returns(T::Boolean) }
def saved_change_to_score_down?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_score_up; end
sig { returns(T::Boolean) }
def saved_change_to_score_up?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_sources_array; end
@@ -2027,6 +2294,141 @@ class Domain::Post::E621Post
sig { void }
def scanned_post_favs_at_will_change!; end
sig { returns(T.nilable(::Integer)) }
def score; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def score=(value); end
sig { returns(T::Boolean) }
def score?; end
sig { returns(T.nilable(::Integer)) }
def score_before_last_save; end
sig { returns(T.untyped) }
def score_before_type_cast; end
sig { returns(T::Boolean) }
def score_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def score_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def score_down; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def score_down=(value); end
sig { returns(T::Boolean) }
def score_down?; end
sig { returns(T.nilable(::Integer)) }
def score_down_before_last_save; end
sig { returns(T.untyped) }
def score_down_before_type_cast; end
sig { returns(T::Boolean) }
def score_down_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_down_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_down_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def score_down_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def score_down_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_down_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def score_down_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def score_down_previously_was; end
sig { returns(T.nilable(::Integer)) }
def score_down_was; end
sig { void }
def score_down_will_change!; end
sig { returns(T.nilable(::Integer)) }
def score_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def score_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def score_previously_was; end
sig { returns(T.nilable(::Integer)) }
def score_up; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def score_up=(value); end
sig { returns(T::Boolean) }
def score_up?; end
sig { returns(T.nilable(::Integer)) }
def score_up_before_last_save; end
sig { returns(T.untyped) }
def score_up_before_type_cast; end
sig { returns(T::Boolean) }
def score_up_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_up_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_up_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def score_up_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def score_up_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def score_up_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def score_up_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def score_up_previously_was; end
sig { returns(T.nilable(::Integer)) }
def score_up_was; end
sig { void }
def score_up_will_change!; end
sig { returns(T.nilable(::Integer)) }
def score_was; end
sig { void }
def score_will_change!; end
sig { returns(T.untyped) }
def sources_array; end
@@ -2313,18 +2715,21 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def will_save_change_to_caused_by_entry_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_change_seq?; end
sig { returns(T::Boolean) }
def will_save_change_to_created_at?; end
sig { returns(T::Boolean) }
def will_save_change_to_description?; end
sig { returns(T::Boolean) }
def will_save_change_to_e621_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_e621_updated_at?; end
sig { returns(T::Boolean) }
def will_save_change_to_file_error?; end
sig { returns(T::Boolean) }
def will_save_change_to_flags_array?; end
@@ -2346,6 +2751,15 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def will_save_change_to_last_submission_log_entry_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_md5?; end
sig { returns(T::Boolean) }
def will_save_change_to_num_comments?; end
sig { returns(T::Boolean) }
def will_save_change_to_num_favorites?; end
sig { returns(T::Boolean) }
def will_save_change_to_parent_post_e621_id?; end
@@ -2370,6 +2784,15 @@ class Domain::Post::E621Post
sig { returns(T::Boolean) }
def will_save_change_to_scanned_post_favs_at?; end
sig { returns(T::Boolean) }
def will_save_change_to_score?; end
sig { returns(T::Boolean) }
def will_save_change_to_score_down?; end
sig { returns(T::Boolean) }
def will_save_change_to_score_up?; end
sig { returns(T::Boolean) }
def will_save_change_to_sources_array?; end

26
sorbet/rbi/dsl/domain/post/e621_post/file_error.rbi generated
View File

@@ -1,26 +0,0 @@
# typed: true
# DO NOT EDIT MANUALLY
# This is an autogenerated file for dynamic methods in `Domain::Post::E621Post::FileError`.
# Please instead update this file by running `bin/tapioca dsl Domain::Post::E621Post::FileError`.
class Domain::Post::E621Post::FileError
sig { returns(T.nilable(::Integer)) }
def log_entry_id; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def log_entry_id=(value); end
sig { returns(T.nilable(::Integer)) }
def retry_count; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def retry_count=(value); end
sig { returns(T.nilable(::Integer)) }
def status_code; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def status_code=(value); end
end

53
sorbet/rbi/dsl/generated_path_helpers_module.rbi generated
View File

@@ -60,30 +60,6 @@ module GeneratedPathHelpersModule
sig { params(args: T.untyped).returns(String) }
def domain_inkbunny_user_posts_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_post_e621_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_post_fa_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_post_inkbunny_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_posts_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_user_e621_user_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_user_fa_user_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_user_inkbunny_user_path(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_users_path(*args); end
sig { params(args: T.untyped).returns(String) }
def edit_global_state_path(*args); end
@@ -100,17 +76,14 @@ module GeneratedPathHelpersModule
def fa_cookies_global_states_path(*args); end
sig { params(args: T.untyped).returns(String) }
def faved_by_domain_post_e621_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def faved_by_domain_post_fa_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def faved_by_domain_post_inkbunny_post_path(*args); end
def faved_by_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def favorites_domain_fa_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def favorites_user_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def global_state_path(*args); end
@@ -159,6 +132,12 @@ module GeneratedPathHelpersModule
sig { params(args: T.untyped).returns(String) }
def pg_hero_path(*args); end
sig { params(args: T.untyped).returns(String) }
def post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def posts_path(*args); end
sig { params(args: T.untyped).returns(String) }
def prometheus_path(*args); end
@@ -243,6 +222,9 @@ module GeneratedPathHelpersModule
sig { params(args: T.untyped).returns(String) }
def scan_post_domain_fa_post_path(*args); end
sig { params(args: T.untyped).returns(String) }
def search_by_name_users_path(*args); end
sig { params(args: T.untyped).returns(String) }
def stats_log_entries_path(*args); end
@@ -261,9 +243,18 @@ module GeneratedPathHelpersModule
sig { params(args: T.untyped).returns(String) }
def user_password_path(*args); end
sig { params(args: T.untyped).returns(String) }
def user_path(*args); end
sig { params(args: T.untyped).returns(String) }
def user_posts_path(*args); end
sig { params(args: T.untyped).returns(String) }
def user_registration_path(*args); end
sig { params(args: T.untyped).returns(String) }
def user_session_path(*args); end
sig { params(args: T.untyped).returns(String) }
def users_path(*args); end
end

53
sorbet/rbi/dsl/generated_url_helpers_module.rbi generated
View File

@@ -60,30 +60,6 @@ module GeneratedUrlHelpersModule
sig { params(args: T.untyped).returns(String) }
def domain_inkbunny_user_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_post_e621_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_post_fa_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_post_inkbunny_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_posts_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_user_e621_user_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_user_fa_user_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_user_inkbunny_user_url(*args); end
sig { params(args: T.untyped).returns(String) }
def domain_users_url(*args); end
sig { params(args: T.untyped).returns(String) }
def edit_global_state_url(*args); end
@@ -100,17 +76,14 @@ module GeneratedUrlHelpersModule
def fa_cookies_global_states_url(*args); end
sig { params(args: T.untyped).returns(String) }
def faved_by_domain_post_e621_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def faved_by_domain_post_fa_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def faved_by_domain_post_inkbunny_post_url(*args); end
def faved_by_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def favorites_domain_fa_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def favorites_user_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def global_state_url(*args); end
@@ -159,6 +132,12 @@ module GeneratedUrlHelpersModule
sig { params(args: T.untyped).returns(String) }
def pg_hero_url(*args); end
sig { params(args: T.untyped).returns(String) }
def post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def posts_url(*args); end
sig { params(args: T.untyped).returns(String) }
def prometheus_url(*args); end
@@ -243,6 +222,9 @@ module GeneratedUrlHelpersModule
sig { params(args: T.untyped).returns(String) }
def scan_post_domain_fa_post_url(*args); end
sig { params(args: T.untyped).returns(String) }
def search_by_name_users_url(*args); end
sig { params(args: T.untyped).returns(String) }
def stats_log_entries_url(*args); end
@@ -261,9 +243,18 @@ module GeneratedUrlHelpersModule
sig { params(args: T.untyped).returns(String) }
def user_password_url(*args); end
sig { params(args: T.untyped).returns(String) }
def user_posts_url(*args); end
sig { params(args: T.untyped).returns(String) }
def user_registration_url(*args); end
sig { params(args: T.untyped).returns(String) }
def user_session_url(*args); end
sig { params(args: T.untyped).returns(String) }
def user_url(*args); end
sig { params(args: T.untyped).returns(String) }
def users_url(*args); end
end

52
spec/factories/domain/post_file.rb
View File

@@ -1,9 +1,53 @@
# typed: false
FactoryBot.define do
factory :domain_post_file, class: "Domain::PostFile" do
post { create(:domain_post_fa_post) }
log_entry { create(:http_log_entry) }
url_str { "https://example.com/image.jpg" }
state { "ok" }
state { "pending" }
association :post, factory: :domain_post_fa_post
trait :has_url do
url_str { "https://example.com/image.jpg" }
end
trait :has_file do
state { "ok" }
url_str { "https://example.com/image.jpg" }
last_status_code { 200 }
before(:create) do
self.log_entry =
create(
:http_log_entry,
url_str: self.url_str,
status_code: self.last_status_code,
)
end
end
trait :terminal_error do
state { "terminal_error" }
url_str { "https://example.com/image.jpg" }
last_status_code { 404 }
before(:create) do
self.log_entry =
create(
:http_log_entry,
url_str: self.url_str,
status_code: self.last_status_code,
)
end
end
trait :retryable_error do
state { "retryable_error" }
url_str { "https://example.com/image.jpg" }
last_status_code { 500 }
before(:create) do
self.log_entry =
create(
:http_log_entry,
url_str: self.url_str,
status_code: self.last_status_code,
)
end
end
end
end

3
spec/factories/domain/user/e621_user.rb
View File

@@ -3,9 +3,6 @@ FactoryBot.define do
factory :domain_user_e621_user, class: "Domain::User::E621User" do
sequence(:e621_id) { |n| n }
sequence(:name) { |n| "user#{n}" }
favs_are_hidden { false }
num_other_favs_cached { 0 }
scanned_favs_status { "ok" }
scanned_favs_at { nil }
end
end

8
spec/jobs/domain/e621/job/posts_index_job_spec.rb
View File

@@ -25,9 +25,9 @@ describe Domain::E621::Job::PostsIndexJob do
described_class.perform_now({ caused_by_entry: file })
expect(Domain::E621::Post.count).to eq(5)
post = Domain::E621::Post.find_by(e621_id: 4_247_443)
expect(post.file_url_str).to eq(
expect(Domain::Post::E621Post.count).to eq(5)
post = Domain::Post::E621Post.find_by(e621_id: 4_247_443)
expect(post.file.url_str).to eq(
"https://static1.e621.net/data/1c/61/1c6169aa51668681e9697a48144d7c78.jpg",
)
expect(post.md5).to eq("1c6169aa51668681e9697a48144d7c78")
@@ -46,7 +46,7 @@ describe Domain::E621::Job::PostsIndexJob do
SpecUtil.enqueued_job_args(Domain::E621::Job::StaticFileJob)[0],
).to eq(
{
post: Domain::E621::Post.find_by(e621_id: 4_247_444),
post_file: Domain::Post::E621Post.find_by(e621_id: 4_247_444).file,
caused_by_entry: log_entries[0],
},
)

70
spec/jobs/domain/e621/job/scan_post_favs_job_spec.rb
View File

@@ -16,13 +16,13 @@ describe Domain::E621::Job::ScanPostFavsJob do
1_775_383,
954_593,
]
e621_user_ids.each do |e621_user_id|
create(:domain_e621_user, e621_user_id: e621_user_id)
e621_user_ids.each do |e621_id|
create(:domain_user_e621_user, e621_id: e621_id)
end
end
it "scans users who favorited the post" do
post = create(:domain_e621_post, e621_id: 4_005_902)
post = create(:domain_post_e621_post, e621_id: 4_005_902)
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
[
@@ -56,52 +56,52 @@ describe Domain::E621::Job::ScanPostFavsJob do
# Verify users were created and updated with correct fav counts
# First page users
users_page1 =
Domain::E621::User.where(
e621_user_id: [454_589, 1_535_298, 956_950, 413_725, 372_696],
Domain::User::E621User.where(
e621_id: [454_589, 1_535_298, 956_950, 413_725, 372_696],
)
expect(users_page1.count).to eq(5)
# Check specific user fav counts from first page
expect(
users_page1.find_by(e621_user_id: 454_589).num_other_favs_cached,
).to eq(765)
expect(
users_page1.find_by(e621_user_id: 1_535_298).num_other_favs_cached,
).to eq(330)
expect(
users_page1.find_by(e621_user_id: 956_950).num_other_favs_cached,
).to eq(24)
expect(
users_page1.find_by(e621_user_id: 413_725).num_other_favs_cached,
).to eq(2529)
expect(
users_page1.find_by(e621_user_id: 372_696).num_other_favs_cached,
).to eq(88)
expect(users_page1.find_by(e621_id: 454_589).num_other_favs_cached).to eq(
765,
)
expect(users_page1.find_by(e621_id: 1_535_298).num_other_favs_cached).to eq(
330,
)
expect(users_page1.find_by(e621_id: 956_950).num_other_favs_cached).to eq(
24,
)
expect(users_page1.find_by(e621_id: 413_725).num_other_favs_cached).to eq(
2529,
)
expect(users_page1.find_by(e621_id: 372_696).num_other_favs_cached).to eq(
88,
)
# Second page users
users_page2 =
Domain::E621::User.where(
e621_user_id: [940_693, 2_055_406, 1_775_383, 954_593],
Domain::User::E621User.where(
e621_id: [940_693, 2_055_406, 1_775_383, 954_593],
)
expect(users_page2.count).to eq(4)
# Check specific user fav counts from second page
expect(
users_page2.find_by(e621_user_id: 940_693).num_other_favs_cached,
).to eq(25_685)
expect(
users_page2.find_by(e621_user_id: 2_055_406).num_other_favs_cached,
).to eq(37)
expect(
users_page2.find_by(e621_user_id: 1_775_383).num_other_favs_cached,
).to eq(497)
expect(
users_page2.find_by(e621_user_id: 954_593).num_other_favs_cached,
).to eq(70)
expect(users_page2.find_by(e621_id: 940_693).num_other_favs_cached).to eq(
25_685,
)
expect(users_page2.find_by(e621_id: 2_055_406).num_other_favs_cached).to eq(
37,
)
expect(users_page2.find_by(e621_id: 1_775_383).num_other_favs_cached).to eq(
497,
)
expect(users_page2.find_by(e621_id: 954_593).num_other_favs_cached).to eq(
70,
)
end
it "handles error responses" do
post = create(:domain_e621_post, e621_id: 4_005_902)
post = create(:domain_post_e621_post, e621_id: 4_005_902)
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
[

20
spec/jobs/domain/e621/job/scan_post_job_spec.rb
View File

@@ -6,7 +6,7 @@ describe Domain::E621::Job::ScanPostJob do
before { Scraper::ClientFactory.http_client_mock = http_client_mock }
it "scans the post" do
post = create(:domain_e621_post, e621_id: 2_227_914)
post = create(:domain_post_e621_post, e621_id: 2_227_914)
caused_by_entry = create(:http_log_entry)
log_entries =
HttpClientMockHelpers.init_http_client_mock(
@@ -28,9 +28,11 @@ describe Domain::E621::Job::ScanPostJob do
{ post: post, caused_by_entry: caused_by_entry },
)
post.reload
expect(post.file).to be_nil
expect(post.state).to eq("ok")
expect(post.file_url_str).to eq(
expect(post.file).not_to be_nil
expect(post.file.state).to eq("pending")
expect(post.file.log_entry).to be_nil
expect(post.file.url_str).to eq(
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
)
expect(post.md5).to eq("c0fa5293f1d1440c2d3f2c3e027d3c36")
@@ -39,12 +41,12 @@ describe Domain::E621::Job::ScanPostJob do
)
expect(SpecUtil.enqueued_job_args(Domain::E621::Job::StaticFileJob)).to eq(
[{ post: post, caused_by_entry: log_entries[0] }],
[{ post_file: post.file, caused_by_entry: log_entries[0] }],
)
end
it "handles a post with no file url" do
post = create(:domain_e621_post, e621_id: 5_270_136)
post = create(:domain_post_e621_post, e621_id: 5_270_136)
log_entries =
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
@@ -59,20 +61,22 @@ describe Domain::E621::Job::ScanPostJob do
],
)
expect(post.file_url_str).to be_nil
expect(post.file).to be_nil
described_class.perform_now({ post: post })
post.reload
expect(post.file_url_str).to eq(
expect(post.file.url_str).to eq(
"https://static1.e621.net/data/d7/72/d7720676ae1dc4c0ff53c1c34ae5c5b0.png",
)
expect(post.file.state).to eq("pending")
expect(post.file.log_entry).to be_nil
expect(post.md5).to eq("d7720676ae1dc4c0ff53c1c34ae5c5b0")
expect(post.tags_array).to match(
hash_including("general" => array_including("anthro", "duo")),
)
expect(SpecUtil.enqueued_job_args(Domain::E621::Job::StaticFileJob)).to eq(
[{ post: post, caused_by_entry: log_entries[0] }],
[{ post_file: post.file, caused_by_entry: log_entries[0] }],
)
end
end

45
spec/jobs/domain/e621/job/scan_user_favs_job_spec.rb
View File

@@ -4,7 +4,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
before { Scraper::ClientFactory.http_client_mock = http_client_mock }
let(:user) { create(:domain_e621_user, e621_user_id: 123_456) }
let(:user) { create(:domain_user_e621_user, e621_id: 123_456) }
let(:job) { described_class.new }
let!(:log_entries) do
@@ -36,26 +36,29 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
}.from(nil).to("ok")
# Verify the posts were created
expect(Domain::E621::Post.count).to eq(5)
expect(Domain::E621::Fav.count).to eq(5)
expect(Domain::Post::E621Post.pluck(:e621_id)).to eq(
[5_212_363, 5_214_461, 5_306_537, 2_518_409, 5_129_881],
)
expect(Domain::UserPostFav.count).to eq(5)
# Verify StaticFileJob was enqueued for each new post
static_file_jobs =
SpecUtil.enqueued_job_args(Domain::E621::Job::StaticFileJob)
expect(static_file_jobs.size).to eq(5)
expect(
static_file_jobs.map { |args| args[:post].e621_id },
).to match_array(Domain::E621::Post.pluck(:e621_id))
static_file_jobs.map { |args| args[:post_file].post.e621_id },
).to match_array(Domain::Post::E621Post.pluck(:e621_id))
# Verify specific post details from fixture
post = Domain::E621::Post.find_by(e621_id: 5_212_363)
post = Domain::Post::E621Post.find_by(e621_id: 5_212_363)
expect(post).to be_present
expect(post.file_url_str).to eq(
expect(post.file).to be_present
expect(post.file.url_str).to eq(
"https://static1.e621.net/data/87/18/8718995a7dd49f24dfae9ffc042c9578.png",
)
# Verify fav relationship
fav = Domain::E621::Fav.find_by(user: user, post: post)
fav = Domain::UserPostFav.find_by(user: user, post: post)
expect(fav).to be_present
end
@@ -112,14 +115,26 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
it "handles hidden favorites appropriately" do
expect { perform_now({ user: user }) }.to change {
user.reload.favs_are_hidden
}.from(nil).to(true).and change { user.reload.scanned_favs_at }.from(
nil,
).to(be_within(1.second).of(Time.current)).and change {
user.reload.scanned_favs_status
}.from(nil).to("ok")
}.from(nil).to(true)
end
# Should not create any favs
expect(Domain::E621::Fav.count).to eq(0)
it "updates scanned_favs_at timestamp" do
expect { perform_now({ user: user }) }.to change {
user.reload.scanned_favs_at
}.from(nil).to(be_within(1.second).of(Time.current))
end
it "sets scanned_favs_status to ok" do
expect { perform_now({ user: user }) }.to change {
user.reload.scanned_favs_status
}.from(nil).to("ok")
end
it "does not create any favs" do
expect { perform_now({ user: user }) }.not_to change(
Domain::UserPostFav,
:count,
)
end
end
end

30
spec/jobs/domain/e621/job/scan_users_job_spec.rb
View File

@@ -29,13 +29,13 @@ describe Domain::E621::Job::ScanUsersJob do
end
it "creates users from the API response" do
expect { perform_now({}) }.to change(Domain::E621::User, :count).by(12)
expect { perform_now({}) }.to change(Domain::User::E621User, :count).by(12)
user = Domain::E621::User.find_by(e621_user_id: 2_089_238)
user = Domain::User::E621User.find_by(e621_id: 2_089_238)
expect(user).not_to be_nil
expect(user.name).to eq("chongzi")
user = Domain::E621::User.find_by(e621_user_id: 2_089_235)
user = Domain::User::E621User.find_by(e621_id: 2_089_235)
expect(user).not_to be_nil
expect(user.name).to eq("dhqobc")
end
@@ -43,8 +43,8 @@ describe Domain::E621::Job::ScanUsersJob do
# it "enqueues scan user favs jobs for new users" do
# perform_now({})
# user1 = Domain::E621::User.find_by(e621_user_id: 2_089_238)
# user2 = Domain::E621::User.find_by(e621_user_id: 2_089_237)
# user1 = Domain::User::E621User.find_by(e621_id: 2_089_238)
# user2 = Domain::User::E621User.find_by(e621_id: 2_089_237)
# expect(SpecUtil.enqueued_jobs(Domain::E621::Job::ScanUserFavsJob)).to match(
# array_including(
@@ -56,17 +56,19 @@ describe Domain::E621::Job::ScanUsersJob do
context "when user already exists" do
let!(:existing_user) do
Domain::E621::User.create!(e621_user_id: 2_089_238, name: "chongzi")
Domain::User::E621User.create!(e621_id: 2_089_238, name: "chongzi")
end
it "does not create duplicate users" do
expect { perform_now({}) }.to change(Domain::E621::User, :count).by(11)
expect { perform_now({}) }.to change(Domain::User::E621User, :count).by(
11,
)
end
# it "does not enqueue scan favs job for existing users" do
# perform_now({})
# new_user = Domain::E621::User.find_by(e621_user_id: 2_089_237)
# new_user = Domain::User::E621User.find_by(e621_id: 2_089_237)
# expect(
# SpecUtil.enqueued_jobs(Domain::E621::Job::ScanUserFavsJob),
@@ -119,15 +121,17 @@ describe Domain::E621::Job::ScanUsersJob do
end
it "follows pagination and creates all users" do
expect { perform_now({}) }.to change(Domain::E621::User, :count).by(23)
expect { perform_now({}) }.to change(Domain::User::E621User, :count).by(
23,
)
# First page
expect(Domain::E621::User.exists?(e621_user_id: 2_089_238)).to be true
expect(Domain::E621::User.exists?(e621_user_id: 2_089_235)).to be true
expect(Domain::User::E621User.exists?(e621_id: 2_089_238)).to be true
expect(Domain::User::E621User.exists?(e621_id: 2_089_235)).to be true
# Second page
expect(Domain::E621::User.exists?(e621_user_id: 2_089_163)).to be true
expect(Domain::E621::User.exists?(e621_user_id: 2_089_091)).to be true
expect(Domain::User::E621User.exists?(e621_id: 2_089_163)).to be true
expect(Domain::User::E621User.exists?(e621_id: 2_089_091)).to be true
end
end

114
spec/jobs/domain/e621/job/static_file_job_spec.rb
View File

@@ -5,14 +5,20 @@ describe Domain::E621::Job::StaticFileJob do
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
before { Scraper::ClientFactory.http_client_mock = http_client_mock }
let (:post) do
create(:domain_post_e621_post)
end
let(:post_file) do
create(
:domain_post_file,
post: post,
url_str:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
)
end
it "downloads the file" do
post =
create(
:domain_e621_post,
file_url_str:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
)
hle = create(:http_log_entry)
mock_log_entries =
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
@@ -23,24 +29,17 @@ describe Domain::E621::Job::StaticFileJob do
status_code: 200,
content_type: "image/jpeg",
contents: "test",
caused_by_entry: hle,
},
],
)
perform_now({ post: post, caused_by_entry: hle })
post.reload
expect(post.file).to eq(mock_log_entries[0])
perform_now({ post_file: post_file })
post_file.reload
expect(post_file.state).to eq("ok")
expect(post_file.log_entry).to eq(mock_log_entries[0])
end
it "handles a 404" do
post =
create(
:domain_e621_post,
file_url_str:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
)
hle = create(:http_log_entry)
mock_log_entries =
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
@@ -51,20 +50,77 @@ describe Domain::E621::Job::StaticFileJob do
status_code: 404,
content_type: "text/html",
contents: "test",
caused_by_entry: hle,
},
],
)
described_class.perform_now({ post: post, caused_by_entry: hle })
post.reload
expect(post.state).to eq("file_error")
expect(post.file_error).to eq(
Domain::E621::Post::FileError.new(
status_code: 404,
log_entry_id: mock_log_entries[0].id,
retry_count: 1,
),
)
perform_now({ post_file: post_file })
post_file.reload
expect(post_file.state).to eq("terminal_error")
expect(post_file.log_entry).to eq(mock_log_entries[0])
expect(post_file.last_status_code).to eq(404)
expect(post_file.retry_count).to eq(1)
end
it "retries with a retryable error" do
mock_log_entries =
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
[
{
uri:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
status_code: 500,
content_type: "text/html",
contents: "test",
},
],
)
perform_now({ post_file: post_file }, should_raise: /will retry later/)
post_file.reload
expect(post_file.state).to eq("retryable_error")
expect(post_file.log_entry).to eq(mock_log_entries[0])
expect(post_file.last_status_code).to eq(500)
expect(post_file.retry_count).to eq(1)
end
it "gives up after 3 retries" do
mock_log_entries =
HttpClientMockHelpers.init_http_client_mock(
http_client_mock,
[
{
uri:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
status_code: 500,
content_type: "text/html",
contents: "test",
},
{
uri:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
status_code: 500,
content_type: "text/html",
contents: "test",
},
{
uri:
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
status_code: 500,
content_type: "text/html",
contents: "test",
},
],
)
perform_now({ post_file: post_file }, should_raise: /will retry later/)
perform_now({ post_file: post_file }, should_raise: /will retry later/)
perform_now({ post_file: post_file })
post_file.reload
expect(post_file.state).to eq("terminal_error")
expect(post_file.log_entry).to eq(mock_log_entries[2])
expect(post_file.last_status_code).to eq(500)
expect(post_file.retry_count).to eq(3)
end
end

7
spec/jobs/domain/fa/job/scan_file_job_spec.rb
View File

@@ -5,7 +5,7 @@ RSpec.describe Domain::Fa::Job::ScanFileJob do
include PerformJobHelpers
let(:fa_post) { create(:domain_post_fa_post) }
let(:post_file) { create(:domain_post_file) }
let(:post_file) { create(:domain_post_file, :has_url) }
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
before do
@@ -122,10 +122,7 @@ RSpec.describe Domain::Fa::Job::ScanFileJob do
end
context "with post arg" do
let(:post) { create(:domain_post_fa_post) }
context "with valid post file" do
let(:post_file) { create(:domain_post_file, post: post) }
let(:client_mock_config) do
[
{
@@ -139,7 +136,7 @@ RSpec.describe Domain::Fa::Job::ScanFileJob do
it "processes using the post's file" do
post_file.update!(state: "pending")
perform_now({ post: post })
perform_now({ post: post_file.post })
post_file.reload
expect(post_file.state).to eq("ok")

9
spec/lib/domain/migrate_to_domain/migrate_e621_to_domain_spec.rb
View File

@@ -35,10 +35,17 @@ RSpec.describe Domain::MigrateToDomain do
index_page_ids: old_post.index_page_ids,
prev_md5s: old_post.prev_md5s,
scan_error: old_post.scan_error,
file_error: old_post.file_error,
created_at: be_within(1.second).of(old_post.created_at),
parent_post_e621_id: old_post.parent_e621_id,
)
if old_file_error = old_post.file_error
expect(new_post.file).to have_attributes(
log_entry_id: old_file_error.log_entry_id,
last_status_code: old_file_error.status_code,
retry_count: old_file_error.retry_count,
)
end
end
describe "#migrate_e621_users" do

22
spec/models/domain/post/e621_post_spec.rb
View File

@@ -120,27 +120,7 @@ RSpec.describe Domain::Post::E621Post, type: :model do
end
it "returns formatted string when e621_id is present" do
expect(post.to_param).to eq("12345")
end
end
describe "file error handling" do
let(:post) { create(:domain_post_e621_post) }
it "can store file error details" do
error =
Domain::Post::E621Post::FileError.new(
retry_count: 3,
status_code: 404,
log_entry_id: 123,
)
post.file_error = error
post.save!
post.reload
expect(post.file_error.retry_count).to eq(3)
expect(post.file_error.status_code).to eq(404)
expect(post.file_error.log_entry_id).to eq(123)
expect(post.to_param).to eq("e621/12345")
end
end
end

4
spec/models/domain/user/fa_user_spec.rb
View File

@@ -95,11 +95,11 @@ RSpec.describe Domain::User::FaUser, type: :model do
expect(user.to_param).to be_nil
user.url_name = ""
expect(user.to_param).to eq("")
expect(user.to_param).to be_nil
end
it "returns fa/url_name when url_name is present" do
expect(user.to_param).to eq("artist123")
expect(user.to_param).to eq("fa/artist123")
end
end