dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate out of e621 post state_detail

Browse Source
This commit is contained in:
Dylan Knutson
2025-01-29 20:17:14 +00:00
parent 4d5784b630
commit 5c14d26f5f
16 changed files with 747 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/jobs/domain/e621/job/scan_post_job.rb
View File

@@ -23,11 +23,10 @@ class Domain::E621::Job::ScanPostJob < Domain::E621::Job::Base
response = http_client.get("https://e621.net/posts/#{post.e621_id}.json")
log_entry = response.log_entry
if response.status_code != 200
post.state_detail["scan_log_entry_id"] = log_entry.id
post.scan_log_entry_id = log_entry.id
post.state = :scan_error
post.state_detail[
"scan_error"
] = "Error scanning post #{post.e621_id}: #{response.status_code}"
post.scan_error =
"Error scanning post #{post.e621_id}: #{response.status_code}"
post.save!
fatal_error(
"Error scanning post #{post.e621_id}: #{response.status_code}",

11
app/jobs/domain/e621/job/static_file_job.rb
View File

@@ -17,7 +17,7 @@ class Domain::E621::Job::StaticFileJob < Domain::E621::Job::Base
end
if post.state == "file_error"
retry_count = post.state_detail&.[]("file_error")&.[]("retry_count") || 0
retry_count = post.file_error&.retry_count || 0
if retry_count >= 3
logger.error("file has been retried 3 times, giving up")
return
@@ -28,11 +28,10 @@ class Domain::E621::Job::StaticFileJob < Domain::E621::Job::Base
if response.status_code != 200
post.state = :file_error
fe = (post.state_detail["file_error"] ||= {})
fe["status_code"] = response.status_code
fe["log_entry_id"] = response.log_entry.id
fe["retry_count"] ||= 0
fe["retry_count"] += 1
fe = (post.file_error ||= Domain::E621::Post::FileError.new)
fe.status_code = response.status_code
fe.log_entry_id = response.log_entry.id
fe.retry_count = (fe.retry_count || 0) + 1
post.save!
if response.status_code == 404

16
app/lib/domain/e621/tag_util.rb
View File

@@ -17,23 +17,19 @@ class Domain::E621::TagUtil
e621_post = Domain::E621::Post.find_or_initialize_by(e621_id: e621_id)
e621_updated_at = post_json["updated_at"]
if e621_post.state_detail["e621_updated_at"] == e621_updated_at
return e621_post
end
return e621_post if e621_post.e621_updated_at == e621_updated_at
e621_post.state_detail["e621_updated_at"] = post_json["updated_at"]
e621_post.state_detail["index_page_ids"] ||= []
e621_post.state_detail[
"caused_by_entry_id"
] = caused_by_entry.id if caused_by_entry
e621_post.e621_updated_at = post_json["updated_at"]
e621_post.index_page_ids ||= []
e621_post.caused_by_entry_id = caused_by_entry.id if caused_by_entry
e621_md5 = T.cast(post_json["file"]["md5"], String)
if e621_post.md5 && e621_post.md5 != e621_md5
logger.warn(
"md5 changed for post: #{e621_post.md5.to_s.bold} => #{e621_md5.to_s.bold}",
)
e621_post.state_detail["prev_md5s"] ||= []
e621_post.state_detail["prev_md5s"] << {
e621_post.prev_md5s ||= []
e621_post.prev_md5s << {
"md5" => e621_post.md5,
"file_id" => e621_post.file_id,
}

57
app/models/domain/e621/post.rb
View File

@@ -1,10 +1,17 @@
# typed: strict
class Domain::E621::Post < ReduxApplicationRecord
include AttrJson::Record
self.table_name = "domain_e621_posts"
include HasIndexedPost
include Discard::Model
class FileError
include AttrJson::Model
attr_json :retry_count, :integer
attr_json :status_code, :integer
attr_json :log_entry_id, :integer
end
self.table_name = "domain_e621_posts"
self.discard_column = :deleted_at
default_scope -> { kept }
@@ -48,6 +55,21 @@ class Domain::E621::Post < ReduxApplicationRecord
# their `num_other_favs_cached` attribute
attr_json :scanned_post_favs_at, :datetime
attr_json :tags_array, ActiveModel::Type::Value.new
attr_json :flags_array, :string, array: true
attr_json :pools_array, :string, array: true
attr_json :sources_array, :string, array: true
attr_json :artists_array, :string, array: true
attr_json :e621_updated_at, :datetime
attr_json :last_index_page_id, :integer
attr_json :caused_by_entry_id, :integer
attr_json :scan_log_entry_id, :integer
attr_json :index_page_ids, :integer, array: true
attr_json :prev_md5s, :string, array: true
attr_json :scan_error, :string
attr_json :file_error, FileError.to_type
sig { returns(String) }
def to_param
self.e621_id.to_s
@@ -60,11 +82,16 @@ class Domain::E621::Post < ReduxApplicationRecord
ta.is_a?(Hash) ? ta : { "general" => ta }
end
# sig { returns(T.nilable(HttpLogEntry)) }
# def index_page_http_log_entry
# if state_detail["last_index_page_id"].present?
# HttpLogEntry.find_by(id: state_detail["last_index_page_id"])
# end
# end
sig { returns(T.nilable(HttpLogEntry)) }
def index_page_http_log_entry
if state_detail["last_index_page_id"].present?
HttpLogEntry.find_by(id: state_detail["last_index_page_id"])
end
HttpLogEntry.find_by(id: last_index_page_id) if last_index_page_id.present?
end
sig { returns(T.nilable(Addressable::URI)) }
@@ -72,15 +99,15 @@ class Domain::E621::Post < ReduxApplicationRecord
Addressable::URI.parse(self.file_url_str) if self.file_url_str.present?
end
sig { returns(T.nilable(Time)) }
def e621_updated_at
str = state_detail["e621_updated_at"]
Time.parse(str) if str
end
# sig { returns(T.nilable(Time)) }
# def e621_updated_at
# str = state_detail["e621_updated_at"]
# Time.parse(str) if str
# end
sig { params(time: T.any(Time, String)).void }
def e621_updated_at=(time)
time = Time.parse(time) if time.is_a?(String)
state_detail["e621_updated_at"] = time.iso8601
end
# sig { params(time: T.any(Time, String)).void }
# def e621_updated_at=(time)
# time = Time.parse(time) if time.is_a?(String)
# state_detail["e621_updated_at"] = time.iso8601
# end
end

19
app/models/redux_application_record.rb
View File

@@ -18,6 +18,25 @@ class ReduxApplicationRecord < ActiveRecord::Base
after_update { observe(:update) }
after_destroy { observe(:destroy) }
TRUNCATE_FIELDS = %w[
json_attributes
tags_array
flags_array
pools_array
sources_array
artists_array
].freeze
# clean up the json_attributes field in console output
sig { params(attr_name: T.any(Symbol, String)).returns(T.untyped) }
def attribute_for_inspect(attr_name)
if TRUNCATE_FIELDS.include?(attr_name.to_s)
str_value = read_attribute(attr_name).inspect
str_value = "#{str_value[0, 50]}..." if str_value.length > 50
return str_value
end
super
end
sig { params(attr_name: Symbol).void }
def self.json_attributes_scope(attr_name)
scope :"where_#{attr_name}",

12
db/migrate/20250129173329_add_specific_indexes_to_e621_json_attrs.rb Normal file
View File

@@ -0,0 +1,12 @@
# typed: strict
class AddSpecificIndexesToE621JsonAttrs < ActiveRecord::Migration[7.2]
extend T::Sig
sig { void }
def change
remove_index :domain_e621_posts, :json_attributes, using: :gin
add_index :domain_e621_posts,
"(json_attributes->'scanned_post_favs_at')",
name: "index_e621_posts_on_scanned_post_favs_at"
end
end

75
db/migrate/20250129174128_migrate_e621_columns_to_json_attrs.rb Normal file
View File

@@ -0,0 +1,75 @@
# typed: strict
class MigrateE621ColumnsToJsonAttrs < ActiveRecord::Migration[7.2]
extend T::Sig
sig { void }
def change
migrate_column("tags_array", :jsonb, :jsonb)
migrate_column("flags_array", :jsonb, :jsonb)
migrate_column("pools_array", :jsonb, :jsonb)
migrate_column("sources_array", :jsonb, :jsonb)
migrate_column("artists_array", :jsonb, :jsonb)
migrate_state_detail("e621_updated_at")
migrate_state_detail("last_index_page_id")
migrate_state_detail("index_page_ids")
migrate_state_detail("prev_md5s")
migrate_state_detail("caused_by_entry_id")
migrate_state_detail("scan_log_entry_id")
migrate_state_detail("scan_error")
migrate_state_detail("file_error")
end
sig { params(column_name: String).void }
def migrate_state_detail(column_name)
reversible do |dir|
dir.up do
execute <<~SQL
UPDATE domain_e621_posts
SET
json_attributes = jsonb_set(json_attributes, '{#{column_name}}', state_detail->'#{column_name}')
-- , state_detail = jsonb_set(state_detail, '{#{column_name}}', 'null'::jsonb)
WHERE state_detail->'#{column_name}' IS NOT NULL
SQL
# remove_column :domain_e621_posts, :state_detail
end
dir.down do
# add_column :domain_e621_posts, column_name, rails_column_type
execute <<~SQL
UPDATE domain_e621_posts
SET
state_detail = jsonb_set(state_detail, '{#{column_name}}', json_attributes->'#{column_name}')
-- , json_attributes = jsonb_set(json_attributes, '{#{column_name}}', 'null'::jsonb)
WHERE json_attributes->>'#{column_name}' IS NOT NULL
SQL
end
end
end
sig do
params(
column_name: String,
rails_column_type: Symbol,
db_column_type: Symbol,
).void
end
def migrate_column(column_name, rails_column_type, db_column_type)
reversible do |dir|
dir.up do
execute <<~SQL
UPDATE domain_e621_posts
SET json_attributes = jsonb_set(json_attributes, '{#{column_name}}', to_json(#{column_name})::jsonb)
WHERE #{column_name} IS NOT NULL
SQL
remove_column :domain_e621_posts, column_name
end
dir.down do
add_column :domain_e621_posts, column_name, rails_column_type
execute <<~SQL
UPDATE domain_e621_posts
SET #{column_name} = (json_attributes->>'#{column_name}')::text::#{db_column_type}
WHERE json_attributes->>'#{column_name}' IS NOT NULL
SQL
end
end
end
end

25
db/structure.sql
View File

@@ -1997,11 +1997,6 @@ CREATE TABLE public.domain_e621_posts (
num_favorites integer,
num_comments integer,
change_seq integer,
flags_array jsonb,
pools_array jsonb,
sources_array jsonb,
artists_array jsonb,
tags_array jsonb,
file_id bigint,
parent_e621_id bigint,
deleted_at timestamp(6) without time zone,
@@ -6056,13 +6051,6 @@ CREATE INDEX index_domain_e621_posts_on_file_id ON public.domain_e621_posts USIN
SET default_tablespace = '';
--
-- Name: index_domain_e621_posts_on_json_attributes; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_e621_posts_on_json_attributes ON public.domain_e621_posts USING gin (json_attributes);
--
-- Name: index_domain_e621_posts_on_md5; Type: INDEX; Schema: public; Owner: -
--
@@ -6445,6 +6433,17 @@ CREATE UNIQUE INDEX index_domain_twitter_users_on_name ON public.domain_twitter_
CREATE UNIQUE INDEX index_domain_twitter_users_on_tw_id ON public.domain_twitter_users USING btree (tw_id);
SET default_tablespace = '';
--
-- Name: index_e621_posts_on_scanned_post_favs_at; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_e621_posts_on_scanned_post_favs_at ON public.domain_e621_posts USING btree (((json_attributes -> 'scanned_post_favs_at'::text)));
SET default_tablespace = mirai;
--
-- Name: index_flat_sst_entries_on_key; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
@@ -7845,6 +7844,8 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250129174128'),
('20250129173329'),
('20250129061805'),
('20250129061217'),
('20250128142856'),

481
sorbet/rbi/dsl/domain/e621/post.rbi generated
View File

@@ -12,8 +12,8 @@ class Domain::E621::Post
extend CommonRelationMethods
extend GeneratedRelationMethods
sig { returns(T::Array[Symbol]) }
def attr_json_registry; end
sig { returns(T.nilable(Domain::E621::Post::FileError)) }
def file_error; end
private
@@ -21,7 +21,13 @@ class Domain::E621::Post
def to_ary; end
class << self
sig { params(name: Symbol, type: Symbol, options: T.nilable(T::Hash[Symbol, T.untyped])).void }
sig do
params(
name: Symbol,
type: T.any(Symbol, ActiveModel::Type::Value),
options: T.nilable(T::Hash[Symbol, T.untyped])
).void
end
def attr_json(name, type, options = nil); end
sig do
@@ -33,6 +39,9 @@ class Domain::E621::Post
end
def attr_json_config(default_container_attribute: nil, bad_cast: nil, unknown_key: nil); end
sig { returns(T::Array[Symbol]) }
def attr_json_registry; end
sig do
params(
attributes: T.untyped,
@@ -830,6 +839,51 @@ class Domain::E621::Post
sig { void }
def artists_array_will_change!; end
sig { returns(T.nilable(::Integer)) }
def caused_by_entry_id; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def caused_by_entry_id=(value); end
sig { returns(T::Boolean) }
def caused_by_entry_id?; end
sig { returns(T.nilable(::Integer)) }
def caused_by_entry_id_before_last_save; end
sig { returns(T.untyped) }
def caused_by_entry_id_before_type_cast; end
sig { returns(T::Boolean) }
def caused_by_entry_id_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def caused_by_entry_id_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def caused_by_entry_id_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def caused_by_entry_id_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def caused_by_entry_id_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def caused_by_entry_id_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def caused_by_entry_id_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def caused_by_entry_id_previously_was; end
sig { returns(T.nilable(::Integer)) }
def caused_by_entry_id_was; end
sig { void }
def caused_by_entry_id_will_change!; end
sig { returns(T.nilable(::Integer)) }
def change_seq; end
@@ -1075,6 +1129,106 @@ class Domain::E621::Post
sig { void }
def e621_id_will_change!; end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def e621_updated_at; end
sig { params(value: T.nilable(::ActiveSupport::TimeWithZone)).returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def e621_updated_at=(value); end
sig { returns(T::Boolean) }
def e621_updated_at?; end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def e621_updated_at_before_last_save; end
sig { returns(T.untyped) }
def e621_updated_at_before_type_cast; end
sig { returns(T::Boolean) }
def e621_updated_at_came_from_user?; end
sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
def e621_updated_at_change; end
sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
def e621_updated_at_change_to_be_saved; end
sig do
params(
from: T.nilable(::ActiveSupport::TimeWithZone),
to: T.nilable(::ActiveSupport::TimeWithZone)
).returns(T::Boolean)
end
def e621_updated_at_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def e621_updated_at_in_database; end
sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
def e621_updated_at_previous_change; end
sig do
params(
from: T.nilable(::ActiveSupport::TimeWithZone),
to: T.nilable(::ActiveSupport::TimeWithZone)
).returns(T::Boolean)
end
def e621_updated_at_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def e621_updated_at_previously_was; end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def e621_updated_at_was; end
sig { void }
def e621_updated_at_will_change!; end
sig { returns(T.untyped) }
def file_error; end
sig { params(value: T.untyped).returns(T.untyped) }
def file_error=(value); end
sig { returns(T::Boolean) }
def file_error?; end
sig { returns(T.untyped) }
def file_error_before_last_save; end
sig { returns(T.untyped) }
def file_error_before_type_cast; end
sig { returns(T::Boolean) }
def file_error_came_from_user?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def file_error_change; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def file_error_change_to_be_saved; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def file_error_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def file_error_in_database; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def file_error_previous_change; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def file_error_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def file_error_previously_was; end
sig { returns(T.untyped) }
def file_error_was; end
sig { void }
def file_error_will_change!; end
sig { returns(T.nilable(::Integer)) }
def file_id; end
@@ -1300,6 +1454,51 @@ class Domain::E621::Post
sig { void }
def id_will_change!; end
sig { returns(T.untyped) }
def index_page_ids; end
sig { params(value: T.untyped).returns(T.untyped) }
def index_page_ids=(value); end
sig { returns(T::Boolean) }
def index_page_ids?; end
sig { returns(T.untyped) }
def index_page_ids_before_last_save; end
sig { returns(T.untyped) }
def index_page_ids_before_type_cast; end
sig { returns(T::Boolean) }
def index_page_ids_came_from_user?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def index_page_ids_change; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def index_page_ids_change_to_be_saved; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def index_page_ids_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def index_page_ids_in_database; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def index_page_ids_previous_change; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def index_page_ids_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def index_page_ids_previously_was; end
sig { returns(T.untyped) }
def index_page_ids_was; end
sig { void }
def index_page_ids_will_change!; end
sig { returns(T.untyped) }
def json_attributes; end
@@ -1345,6 +1544,51 @@ class Domain::E621::Post
sig { void }
def json_attributes_will_change!; end
sig { returns(T.nilable(::Integer)) }
def last_index_page_id; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def last_index_page_id=(value); end
sig { returns(T::Boolean) }
def last_index_page_id?; end
sig { returns(T.nilable(::Integer)) }
def last_index_page_id_before_last_save; end
sig { returns(T.untyped) }
def last_index_page_id_before_type_cast; end
sig { returns(T::Boolean) }
def last_index_page_id_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def last_index_page_id_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def last_index_page_id_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def last_index_page_id_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def last_index_page_id_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def last_index_page_id_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def last_index_page_id_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def last_index_page_id_previously_was; end
sig { returns(T.nilable(::Integer)) }
def last_index_page_id_was; end
sig { void }
def last_index_page_id_will_change!; end
sig { returns(T.nilable(::String)) }
def md5; end
@@ -1625,6 +1869,51 @@ class Domain::E621::Post
sig { void }
def posted_at_will_change!; end
sig { returns(T.untyped) }
def prev_md5s; end
sig { params(value: T.untyped).returns(T.untyped) }
def prev_md5s=(value); end
sig { returns(T::Boolean) }
def prev_md5s?; end
sig { returns(T.untyped) }
def prev_md5s_before_last_save; end
sig { returns(T.untyped) }
def prev_md5s_before_type_cast; end
sig { returns(T::Boolean) }
def prev_md5s_came_from_user?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def prev_md5s_change; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def prev_md5s_change_to_be_saved; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def prev_md5s_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def prev_md5s_in_database; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def prev_md5s_previous_change; end
sig { params(from: T.untyped, to: T.untyped).returns(T::Boolean) }
def prev_md5s_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.untyped) }
def prev_md5s_previously_was; end
sig { returns(T.untyped) }
def prev_md5s_was; end
sig { void }
def prev_md5s_will_change!; end
sig { returns(T.nilable(::String)) }
def rating; end
@@ -1687,6 +1976,9 @@ class Domain::E621::Post
sig { void }
def restore_artists_array!; end
sig { void }
def restore_caused_by_entry_id!; end
sig { void }
def restore_change_seq!; end
@@ -1702,6 +1994,12 @@ class Domain::E621::Post
sig { void }
def restore_e621_id!; end
sig { void }
def restore_e621_updated_at!; end
sig { void }
def restore_file_error!; end
sig { void }
def restore_file_id!; end
@@ -1717,9 +2015,15 @@ class Domain::E621::Post
sig { void }
def restore_id_value!; end
sig { void }
def restore_index_page_ids!; end
sig { void }
def restore_json_attributes!; end
sig { void }
def restore_last_index_page_id!; end
sig { void }
def restore_md5!; end
@@ -1738,9 +2042,18 @@ class Domain::E621::Post
sig { void }
def restore_posted_at!; end
sig { void }
def restore_prev_md5s!; end
sig { void }
def restore_rating!; end
sig { void }
def restore_scan_error!; end
sig { void }
def restore_scan_log_entry_id!; end
sig { void }
def restore_scanned_post_favs_at!; end
@@ -1774,6 +2087,12 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def saved_change_to_artists_array?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_caused_by_entry_id; end
sig { returns(T::Boolean) }
def saved_change_to_caused_by_entry_id?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_change_seq; end
@@ -1804,6 +2123,18 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def saved_change_to_e621_id?; end
sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
def saved_change_to_e621_updated_at; end
sig { returns(T::Boolean) }
def saved_change_to_e621_updated_at?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_file_error; end
sig { returns(T::Boolean) }
def saved_change_to_file_error?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_file_id; end
@@ -1834,12 +2165,24 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def saved_change_to_id_value?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_index_page_ids; end
sig { returns(T::Boolean) }
def saved_change_to_index_page_ids?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_json_attributes; end
sig { returns(T::Boolean) }
def saved_change_to_json_attributes?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_last_index_page_id; end
sig { returns(T::Boolean) }
def saved_change_to_last_index_page_id?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def saved_change_to_md5; end
@@ -1876,12 +2219,30 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def saved_change_to_posted_at?; end
sig { returns(T.nilable([T.untyped, T.untyped])) }
def saved_change_to_prev_md5s; end
sig { returns(T::Boolean) }
def saved_change_to_prev_md5s?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def saved_change_to_rating; end
sig { returns(T::Boolean) }
def saved_change_to_rating?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def saved_change_to_scan_error; end
sig { returns(T::Boolean) }
def saved_change_to_scan_error?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def saved_change_to_scan_log_entry_id; end
sig { returns(T::Boolean) }
def saved_change_to_scan_log_entry_id?; end
sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
def saved_change_to_scanned_post_favs_at; end
@@ -1936,6 +2297,96 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def saved_change_to_updated_at?; end
sig { returns(T.nilable(::String)) }
def scan_error; end
sig { params(value: T.nilable(::String)).returns(T.nilable(::String)) }
def scan_error=(value); end
sig { returns(T::Boolean) }
def scan_error?; end
sig { returns(T.nilable(::String)) }
def scan_error_before_last_save; end
sig { returns(T.untyped) }
def scan_error_before_type_cast; end
sig { returns(T::Boolean) }
def scan_error_came_from_user?; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def scan_error_change; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def scan_error_change_to_be_saved; end
sig { params(from: T.nilable(::String), to: T.nilable(::String)).returns(T::Boolean) }
def scan_error_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::String)) }
def scan_error_in_database; end
sig { returns(T.nilable([T.nilable(::String), T.nilable(::String)])) }
def scan_error_previous_change; end
sig { params(from: T.nilable(::String), to: T.nilable(::String)).returns(T::Boolean) }
def scan_error_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::String)) }
def scan_error_previously_was; end
sig { returns(T.nilable(::String)) }
def scan_error_was; end
sig { void }
def scan_error_will_change!; end
sig { returns(T.nilable(::Integer)) }
def scan_log_entry_id; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def scan_log_entry_id=(value); end
sig { returns(T::Boolean) }
def scan_log_entry_id?; end
sig { returns(T.nilable(::Integer)) }
def scan_log_entry_id_before_last_save; end
sig { returns(T.untyped) }
def scan_log_entry_id_before_type_cast; end
sig { returns(T::Boolean) }
def scan_log_entry_id_came_from_user?; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def scan_log_entry_id_change; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def scan_log_entry_id_change_to_be_saved; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def scan_log_entry_id_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def scan_log_entry_id_in_database; end
sig { returns(T.nilable([T.nilable(::Integer), T.nilable(::Integer)])) }
def scan_log_entry_id_previous_change; end
sig { params(from: T.nilable(::Integer), to: T.nilable(::Integer)).returns(T::Boolean) }
def scan_log_entry_id_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
sig { returns(T.nilable(::Integer)) }
def scan_log_entry_id_previously_was; end
sig { returns(T.nilable(::Integer)) }
def scan_log_entry_id_was; end
sig { void }
def scan_log_entry_id_will_change!; end
sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
def scanned_post_favs_at; end
@@ -2378,6 +2829,9 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def will_save_change_to_artists_array?; end
sig { returns(T::Boolean) }
def will_save_change_to_caused_by_entry_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_change_seq?; end
@@ -2393,6 +2847,12 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def will_save_change_to_e621_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_e621_updated_at?; end
sig { returns(T::Boolean) }
def will_save_change_to_file_error?; end
sig { returns(T::Boolean) }
def will_save_change_to_file_id?; end
@@ -2408,9 +2868,15 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def will_save_change_to_id_value?; end
sig { returns(T::Boolean) }
def will_save_change_to_index_page_ids?; end
sig { returns(T::Boolean) }
def will_save_change_to_json_attributes?; end
sig { returns(T::Boolean) }
def will_save_change_to_last_index_page_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_md5?; end
@@ -2429,9 +2895,18 @@ class Domain::E621::Post
sig { returns(T::Boolean) }
def will_save_change_to_posted_at?; end
sig { returns(T::Boolean) }
def will_save_change_to_prev_md5s?; end
sig { returns(T::Boolean) }
def will_save_change_to_rating?; end
sig { returns(T::Boolean) }
def will_save_change_to_scan_error?; end
sig { returns(T::Boolean) }
def will_save_change_to_scan_log_entry_id?; end
sig { returns(T::Boolean) }
def will_save_change_to_scanned_post_favs_at?; end

26
sorbet/rbi/dsl/domain/e621/post/file_error.rbi generated Normal file
View File

@@ -0,0 +1,26 @@
# typed: true
# DO NOT EDIT MANUALLY
# This is an autogenerated file for dynamic methods in `Domain::E621::Post::FileError`.
# Please instead update this file by running `bin/tapioca dsl Domain::E621::Post::FileError`.
class Domain::E621::Post::FileError
sig { returns(T.nilable(::Integer)) }
def log_entry_id; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def log_entry_id=(value); end
sig { returns(T.nilable(::Integer)) }
def retry_count; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def retry_count=(value); end
sig { returns(T.nilable(::Integer)) }
def status_code; end
sig { params(value: T.nilable(::Integer)).returns(T.nilable(::Integer)) }
def status_code=(value); end
end

14
sorbet/rbi/dsl/domain/e621/user.rbi generated
View File

@@ -11,16 +11,19 @@ class Domain::E621::User
extend CommonRelationMethods
extend GeneratedRelationMethods
sig { returns(T::Array[Symbol]) }
def attr_json_registry; end
private
sig { returns(NilClass) }
def to_ary; end
class << self
sig { params(name: Symbol, type: Symbol, options: T.nilable(T::Hash[Symbol, T.untyped])).void }
sig do
params(
name: Symbol,
type: T.any(Symbol, ActiveModel::Type::Value),
options: T.nilable(T::Hash[Symbol, T.untyped])
).void
end
def attr_json(name, type, options = nil); end
sig do
@@ -32,6 +35,9 @@ class Domain::E621::User
end
def attr_json_config(default_container_attribute: nil, bad_cast: nil, unknown_key: nil); end
sig { returns(T::Array[Symbol]) }
def attr_json_registry; end
sig do
params(
attributes: T.untyped,

1
sorbet/rbi/todo.rbi
View File

@@ -8,3 +8,4 @@ module SyntaxTree::Haml; end
module SyntaxTree::Haml::Format::Formatter; end
module SyntaxTree::RBS; end
module SyntaxTree::RBS::Formatter; end
module Tapioca::Dsl::Compilers::ActiveModelAttributes; end

32
sorbet/tapioca/compilers/attr_json_activemodel_dsl.rb Normal file
View File

@@ -0,0 +1,32 @@
# typed: strict
module Tapioca::Compilers
class AttrJsonActivemodelDsl < Tapioca::Dsl::Compiler
extend T::Sig
ConstantType = type_member { { fixed: T.class_of(::AttrJson::Model) } }
sig { override.returns(T::Enumerable[Module]) }
def self.gather_constants
all_classes.select { |c| c < ::AttrJson::Model }
end
sig { override.void }
def decorate
root.create_path(constant) do |klass|
compiler =
T.unsafe(Tapioca::Dsl::Compilers::ActiveModelAttributes).allocate
registry = T.unsafe(constant).attr_json_registry
T
.cast(registry.attribute_names, T::Array[Symbol])
.each do |name|
attribute = T.cast(registry[name], AttrJson::AttributeDefinition)
type = attribute.type
type = T.unsafe(compiler).send(:type_for, type)
T.unsafe(compiler).send(:generate_method, klass, name.to_s, type)
T.unsafe(compiler).send(:generate_method, klass, "#{name}=", type)
end
end
end
end
end

64
sorbet/tapioca/compilers/attr_json_dsl.rb
View File

@@ -39,7 +39,10 @@ module Tapioca::Compilers
"attr_json",
parameters: [
create_param("name", type: "Symbol"),
create_param("type", type: "Symbol"),
create_param(
"type",
type: "T.any(Symbol, ActiveModel::Type::Value)",
),
create_opt_param(
"options",
type: "T.nilable(T::Hash[Symbol, T.untyped])",
@@ -52,53 +55,22 @@ module Tapioca::Compilers
klass.create_method(
"attr_json_registry",
return_type: "T::Array[Symbol]",
class_method: true,
)
attribute_names =
T.cast(
T.unsafe(constant).attr_json_registry.attribute_names,
T::Array[Symbol],
)
# attribute_names.each do |attribute_name|
# attr_type =
# T.cast(
# T
# .unsafe(constant)
# .attr_json_registry
# .type_for_attribute(attribute_name),
# ActiveModel::Type::Value,
# )
# type_name =
# case attr_type.type
# when :boolean
# "T::Boolean"
# when :big_integer
# "Integer"
# when :binary
# "String"
# when :date
# "Date"
# when :datetime
# "DateTime"
# when :decimal
# "Decimal"
# when :float
# "Float"
# when :immutable_string
# "String"
# when :integer
# "Integer"
# when :string
# "String"
# else
# raise("Unknown type: #{attr_type.type}")
# end
# # klass.create_method(
# # attribute_name.to_s,
# # return_type: "T.nilable(#{type_name})",
# # )
# end
registry = T.unsafe(constant).attr_json_registry
T
.cast(registry.attribute_names, T::Array[Symbol])
.each do |name|
attribute = T.cast(registry[name], AttrJson::AttributeDefinition)
if (type = attribute.type) && type.is_a?(AttrJson::Type::Model) &&
(model = type.model) && model < AttrJson::Model
klass.create_method(
name.to_s,
return_type: "T.nilable(#{model.name})",
)
end
end
end
end
end

2
spec/jobs/domain/e621/job/posts_index_job_spec.rb
View File

@@ -5,7 +5,7 @@ describe Domain::E621::Job::PostsIndexJob do
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
before { Scraper::ClientFactory.http_client_mock = http_client_mock }
it "works", quiet: false do
it "works" do
file = create(:http_log_entry)
log_entries =

12
spec/jobs/domain/e621/job/static_file_job_spec.rb
View File

@@ -59,12 +59,12 @@ describe Domain::E621::Job::StaticFileJob do
described_class.perform_now({ post: post, caused_by_entry: hle })
post.reload
expect(post.state).to eq("file_error")
expect(post.state_detail["file_error"]).to eq(
{
"status_code" => 404,
"log_entry_id" => mock_log_entries[0].id,
"retry_count" => 1,
},
expect(post.file_error).to eq(
Domain::E621::Post::FileError.new(
status_code: 404,
log_entry_id: mock_log_entries[0].id,
retry_count: 1,
),
)
end
end