dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate e621 favs to own table

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-20 22:10:57 +00:00
parent 6381067235
commit 8bd6c4b2ae
22 changed files with 1710 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/jobs/domain/e621/job/scan_user_favs_job.rb
View File

@@ -121,9 +121,11 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
logger.info "upserting #{post_ids.size} favs"
post_ids.each_slice(1000) do |slice|
ReduxApplicationRecord.transaction do
Domain::UserPostFav.upsert_all(
slice.map { |post_id| { user_id: user.id, post_id: post_id } },
unique_by: :index_domain_user_post_favs_on_user_id_and_post_id,
Domain::UserPostFav::E621UserPostFav.upsert_all(
slice.map do |post_id|
{ user_id: user.id, post_id: post_id, removed: false }
end,
unique_by: %i[user_id post_id],
)
end
end

2
app/models/domain/post/e621_post.rb
View File

@@ -3,7 +3,7 @@ class Domain::Post::E621Post < Domain::Post
aux_table :e621
has_single_file!
has_faving_users! Domain::User::E621User
has_faving_users! Domain::User::E621User, Domain::UserPostFav::E621UserPostFav
belongs_to_groups! :pools,
Domain::PostGroup::E621Pool,
Domain::PostGroupJoin::E621PoolJoin

25
app/models/domain/user.rb
View File

@@ -147,9 +147,30 @@ class Domain::User < ReduxApplicationRecord
class_name: klass.name
end
sig { params(klass: T.class_of(Domain::Post)).void }
def self.has_faved_posts!(klass)
sig do
params(
klass: T.class_of(Domain::Post),
fav_model_type: T.class_of(Domain::UserPostFav),
fav_model_order: T.untyped,
).void
end
def self.has_faved_posts!(
klass,
fav_model_type = Domain::UserPostFav,
fav_model_order: nil
)
self.class_has_faved_posts = klass
has_many :user_post_favs,
-> do
rel = extending(CounterCacheWithFallback[:user_post_favs])
rel = rel.order(fav_model_order) if fav_model_order
rel
end,
class_name: fav_model_type.name,
inverse_of: :user,
dependent: :destroy
has_many :faved_posts,
-> { order(klass.param_order_attribute => :desc) },
through: :user_post_favs,

2
app/models/domain/user/e621_user.rb
View File

@@ -17,7 +17,7 @@ class Domain::User::E621User < Domain::User
validates :e621_id, presence: true
validates :name, length: { minimum: 1 }, allow_nil: false
has_faved_posts! Domain::Post::E621Post
has_faved_posts! Domain::Post::E621Post, Domain::UserPostFav::E621UserPostFav
sig { override.returns([String, Symbol]) }
def self.param_prefix_and_attribute

6
app/models/domain/user/fa_user.rb
View File

@@ -29,7 +29,11 @@ class Domain::User::FaUser < Domain::User
has_followed_users! Domain::User::FaUser
has_followed_by_users! Domain::User::FaUser
has_created_posts! Domain::Post::FaPost
has_faved_posts! Domain::Post::FaPost
has_faved_posts! Domain::Post::FaPost,
Domain::UserPostFav::FaUserPostFav,
fav_model_order: {
fa_fav_id: :desc,
}
enum :state,
{ ok: "ok", account_disabled: "account_disabled", error: "error" },

12
app/models/domain/user_post_fav.rb
View File

@@ -16,6 +16,18 @@ class Domain::UserPostFav < ReduxApplicationRecord
belongs_to :post, class_name: "Domain::Post", inverse_of: :user_post_favs
sig { params(user_klass: T.class_of(Domain::User), post_klass: T.class_of(Domain::Post)).void }
def self.user_post_fav_relationships(user_klass, post_klass)
belongs_to_with_counter_cache :user,
class_name: user_klass.name,
inverse_of: :user_post_favs,
counter_cache: :user_post_favs_count
belongs_to :post,
class_name: post_klass.name,
inverse_of: :user_post_favs
end
scope :for_post_type,
->(post_klass) do
post_klass = T.cast(post_klass, T.class_of(Domain::Post))

7
app/models/domain/user_post_fav/e621_user_post_fav.rb Normal file
View File

@@ -0,0 +1,7 @@
# typed: strict
# frozen_string_literal: true
class Domain::UserPostFav::E621UserPostFav < Domain::UserPostFav
self.table_name = "domain_user_post_favs_e621"
user_post_fav_relationships Domain::User::E621User, Domain::Post::E621Post
end

15
app/models/domain/user_post_fav/fa_user_post_fav.rb
View File

@@ -1,24 +1,11 @@
# typed: strict
class Domain::UserPostFav::FaUserPostFav < Domain::UserPostFav
self.table_name = "domain_user_post_favs_fa"
belongs_to_with_counter_cache :user,
class_name: "Domain::User::FaUser",
inverse_of: :user_post_favs,
counter_cache: :user_post_favs_count
belongs_to :post,
class_name: "Domain::Post::FaPost",
inverse_of: :user_post_favs
user_post_fav_relationships Domain::User::FaUser, Domain::Post::FaPost
scope :with_explicit_time_and_id,
-> { where.not(explicit_time: nil).where.not(fa_fav_id: nil) }
scope :with_inferred_time_and_id,
-> { where.not(inferred_time: nil).where.not(fa_fav_id: nil) }
scope :with_fa_fav_id, -> { where.not(fa_fav_id: nil) }
validates :fa_fav_id, uniqueness: true, if: :fa_fav_id?
before_save :set_inferred_time

12
db/migrate/20250820204436_remove_fa_user_post_fav_rows.rb Normal file
View File

@@ -0,0 +1,12 @@
class RemoveFaUserPostFavRows < ActiveRecord::Migration[7.2]
def up
execute <<-SQL
DELETE FROM domain_user_post_favs
WHERE type = 'Domain::UserPostFav::FaUserPostFav_INVALID'
SQL
end
def down
raise ActiveRecord::IrreversibleMigration
end
end

14
db/migrate/20250820205149_create_user_post_favs_e621.rb Normal file
View File

@@ -0,0 +1,14 @@
# typed: strict
# frozen_string_literal: true
class CreateUserPostFavsE621 < ActiveRecord::Migration[7.2]
sig { void }
def change
create_table :domain_user_post_favs_e621,
primary_key: %i[user_id post_id] do |t|
t.bigint :user_id, null: false
t.bigint :post_id, null: false
t.boolean :removed, null: false, default: false
end
end
end

70
db/migrate/20250820210922_migrate_domain_user_post_favs_e621.rb Normal file
View File

@@ -0,0 +1,70 @@
# typed: strict
# frozen_string_literal: true
class MigrateDomainUserPostFavsE621 < ActiveRecord::Migration[7.2]
disable_ddl_transaction!
sig { void }
def up
puts "Getting min/max user id..."
min_user_id =
Domain::User.where(type: "Domain::User::E621User").minimum(:id)
max_user_id =
Domain::User.where(type: "Domain::User::E621User").maximum(:id) + 1
max_batch_size = 1000
batch_count = ((max_user_id - min_user_id) / max_batch_size.to_f).ceil
puts "Migrating #{batch_count} batches..."
shards =
T.cast(
batch_count.times.map do |batch_index|
start_user_id = min_user_id + batch_index * max_batch_size
end_user_id = [start_user_id + max_batch_size, max_user_id].min
Domain::User.where(
type: "Domain::User::E621User",
id: start_user_id...end_user_id,
).pluck(:id)
end,
T::Array[T::Array[Integer]],
)
num_threads = 4
pool =
T.let(
Concurrent::FixedThreadPool.new(num_threads),
Concurrent::FixedThreadPool,
)
shards.each_with_index do |shard, index|
pool.post do
puts "migrate shard #{index + 1} of #{shards.size}: #{shard.minmax.join(" -> ")} (#{shard.size} users)"
migrate_shard(shard)
puts "done: shard #{index + 1} of #{shards.size}: #{shard.minmax.join(" -> ")}"
end
end
pool.shutdown
pool.wait_for_termination
end
sig { params(user_ids: T::Array[Integer]).void }
def migrate_shard(user_ids)
ActiveRecord::Base.with_connection do |connection|
connection.execute <<-SQL
INSERT INTO
domain_user_post_favs_e621 (
user_id,
post_id,
removed
)
SELECT
user_id,
post_id,
removed
FROM domain_user_post_favs
WHERE user_id IN (#{user_ids.join(", ")})
ON CONFLICT (user_id, post_id) DO NOTHING
SQL
end
end
end

21
db/migrate/20250820212318_remove_idx_domain_user_post_favs_on_fav_id.rb Normal file
View File

@@ -0,0 +1,21 @@
class RemoveIdxDomainUserPostFavsOnFavId < ActiveRecord::Migration[7.2]
def up
execute <<-SQL
DROP INDEX index_domain_user_post_favs_on_type_and_user_id;
DROP INDEX idx_domain_user_post_favs_on_fav_id;
SQL
end
def down
execute <<-SQL
CREATE UNIQUE INDEX idx_domain_user_post_favs_on_fav_id
ON domain_user_post_favs USING btree
(((json_attributes ->> 'fav_id'::text)::integer) ASC NULLS LAST)
WHERE type = 'Domain::UserPostFav::FaUserPostFav_INVALID'::domain_user_post_fav_type;
CREATE INDEX index_domain_user_post_favs_on_type_and_user_id
ON domain_user_post_favs USING btree
(type ASC NULLS LAST, user_id ASC NULLS LAST);
SQL
end
end

33
db/migrate/20250820215340_add_indexes_to_user_post_favs_e621.rb Normal file
View File

@@ -0,0 +1,33 @@
# typed: strict
#
class AddIndexesToUserPostFavsE621 < ActiveRecord::Migration[7.2]
sig { void }
def change
change_table :domain_user_post_favs_e621 do |t|
t.index %i[post_id user_id]
end
reversible do |dir|
dir.up do
add_foreign_key :domain_user_post_favs_e621,
:domain_users_e621_aux,
primary_key: :base_table_id,
column: :user_id,
index: false,
name: "fk_domain_user_post_favs_e621_user_id"
add_foreign_key :domain_user_post_favs_e621,
:domain_posts_e621_aux,
primary_key: :base_table_id,
column: :post_id,
index: false,
name: "fk_domain_user_post_favs_e621_post_id"
end
dir.down do
remove_foreign_key :domain_user_post_favs_e621,
name: "fk_domain_user_post_favs_e621_user_id"
remove_foreign_key :domain_user_post_favs_e621,
name: "fk_domain_user_post_favs_e621_post_id"
end
end
end
end

60
db/structure.sql
View File

@@ -1,4 +1,4 @@
\restrict huxfo3NM6M7Nj4JUdHDWXGq7DoLnw9sDIrV5g3aVIBWghR5vgItCQUJG6cJ3EJf
\restrict fsnJecu1BUZk39yUe5E3zqbZEe6yKihIQHKEWYdhLLHCBtdUoLtB3L0VBXECSF8
-- Dumped from database version 17.6 (Debian 17.6-1.pgdg13+1)
-- Dumped by pg_dump version 17.6 (Debian 17.6-1.pgdg12+1)
@@ -1972,6 +1972,19 @@ CREATE TABLE public.domain_user_post_favs (
WITH (autovacuum_enabled='true');
--
-- Name: domain_user_post_favs_e621; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.domain_user_post_favs_e621 (
user_id bigint NOT NULL,
post_id bigint NOT NULL,
removed boolean DEFAULT false NOT NULL,
explicit_time timestamp(6) without time zone,
inferred_time timestamp(6) without time zone
);
--
-- Name: domain_user_post_favs_fa; Type: TABLE; Schema: public; Owner: -
--
@@ -3624,6 +3637,14 @@ ALTER TABLE ONLY public.domain_user_job_event_profile_scans
ADD CONSTRAINT domain_user_job_event_profile_scans_pkey PRIMARY KEY (id);
--
-- Name: domain_user_post_favs_e621 domain_user_post_favs_e621_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_e621
ADD CONSTRAINT domain_user_post_favs_e621_pkey PRIMARY KEY (user_id, post_id);
--
-- Name: domain_user_post_favs_fa domain_user_post_favs_fa_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -3913,13 +3934,6 @@ CREATE UNIQUE INDEX idx_domain_post_groups_on_sofurry_id ON public.domain_post_g
CREATE UNIQUE INDEX idx_domain_posts_on_sofurry_id ON public.domain_posts USING btree ((((json_attributes ->> 'sofurry_id'::text))::integer)) WHERE (type = 'Domain::Post::SofurryPost'::public.domain_post_type);
--
-- Name: idx_domain_user_post_favs_on_fav_id; Type: INDEX; Schema: public; Owner: -
--
CREATE UNIQUE INDEX idx_domain_user_post_favs_on_fav_id ON public.domain_user_post_favs USING btree ((((json_attributes ->> 'fav_id'::text))::integer)) WHERE (type = 'Domain::UserPostFav::FaUserPostFav_INVALID'::public.domain_user_post_fav_type);
--
-- Name: idx_domain_users_e621_on_name_lower; Type: INDEX; Schema: public; Owner: -
--
@@ -4732,6 +4746,13 @@ CREATE INDEX index_domain_user_post_fav_user_factors_on_embedding ON public.doma
CREATE UNIQUE INDEX index_domain_user_post_fav_user_factors_on_user_id ON public.domain_user_post_fav_user_factors USING btree (user_id);
--
-- Name: index_domain_user_post_favs_e621_on_post_id_and_user_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_user_post_favs_e621_on_post_id_and_user_id ON public.domain_user_post_favs_e621 USING btree (post_id, user_id);
--
-- Name: index_domain_user_post_favs_fa_on_fa_fav_id; Type: INDEX; Schema: public; Owner: -
--
@@ -5656,6 +5677,22 @@ ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob
ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob_files_63_on_sha256;
--
-- Name: domain_user_post_favs_e621 fk_domain_user_post_favs_e621_post_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_e621
ADD CONSTRAINT fk_domain_user_post_favs_e621_post_id FOREIGN KEY (post_id) REFERENCES public.domain_posts_e621_aux(base_table_id);
--
-- Name: domain_user_post_favs_e621 fk_domain_user_post_favs_e621_user_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_e621
ADD CONSTRAINT fk_domain_user_post_favs_e621_user_id FOREIGN KEY (user_id) REFERENCES public.domain_users_e621_aux(base_table_id);
--
-- Name: domain_user_post_favs_fa fk_domain_user_post_favs_fa_post_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -6092,11 +6129,16 @@ ALTER TABLE ONLY public.domain_twitter_tweets
-- PostgreSQL database dump complete
--
\unrestrict huxfo3NM6M7Nj4JUdHDWXGq7DoLnw9sDIrV5g3aVIBWghR5vgItCQUJG6cJ3EJf
\unrestrict fsnJecu1BUZk39yUe5E3zqbZEe6yKihIQHKEWYdhLLHCBtdUoLtB3L0VBXECSF8
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250820215340'),
('20250820212318'),
('20250820210922'),
('20250820205149'),
('20250820204436'),
('20250820145726'),
('20250819012459'),
('20250819001506'),

4
sorbet/rbi/dsl/domain/post/e621_post.rbi generated
View File

@@ -719,10 +719,10 @@ class Domain::Post::E621Post
# This method is created by ActiveRecord on the `Domain::Post::E621Post` class because it declared `has_many :user_post_favs`.
# 🔗 [Rails guide for `has_many` association](https://guides.rubyonrails.org/association_basics.html#the-has-many-association)
sig { returns(::Domain::UserPostFav::PrivateCollectionProxy) }
sig { returns(::Domain::UserPostFav::E621UserPostFav::PrivateCollectionProxy) }
def user_post_favs; end
sig { params(value: T::Enumerable[::Domain::UserPostFav]).void }
sig { params(value: T::Enumerable[::Domain::UserPostFav::E621UserPostFav]).void }
def user_post_favs=(value); end
end

6
sorbet/rbi/dsl/domain/user/e621_user.rbi generated
View File

@@ -672,12 +672,12 @@ class Domain::User::E621User
sig { params(ids: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
def user_post_fav_ids=(ids); end
# This method is created by ActiveRecord on the `Domain::User` class because it declared `has_many :user_post_favs`.
# This method is created by ActiveRecord on the `Domain::User::E621User` class because it declared `has_many :user_post_favs`.
# 🔗 [Rails guide for `has_many` association](https://guides.rubyonrails.org/association_basics.html#the-has-many-association)
sig { returns(::Domain::UserPostFav::PrivateCollectionProxy) }
sig { returns(::Domain::UserPostFav::E621UserPostFav::PrivateCollectionProxy) }
def user_post_favs; end
sig { params(value: T::Enumerable[::Domain::UserPostFav]).void }
sig { params(value: T::Enumerable[::Domain::UserPostFav::E621UserPostFav]).void }
def user_post_favs=(value); end
sig { returns(T::Array[T.untyped]) }

2
sorbet/rbi/dsl/domain/user/inkbunny_user.rbi generated
View File

@@ -699,7 +699,7 @@ class Domain::User::InkbunnyUser
sig { params(ids: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
def user_post_fav_ids=(ids); end
# This method is created by ActiveRecord on the `Domain::User` class because it declared `has_many :user_post_favs`.
# This method is created by ActiveRecord on the `Domain::User::InkbunnyUser` class because it declared `has_many :user_post_favs`.
# 🔗 [Rails guide for `has_many` association](https://guides.rubyonrails.org/association_basics.html#the-has-many-association)
sig { returns(::Domain::UserPostFav::PrivateCollectionProxy) }
def user_post_favs; end

1436
sorbet/rbi/dsl/domain/user_post_fav/e621_user_post_fav.rbi generated Normal file
View File

File diff suppressed because it is too large Load Diff

12
sorbet/rbi/dsl/domain/user_post_fav/fa_user_post_fav.rbi generated
View File

@@ -665,12 +665,6 @@ class Domain::UserPostFav::FaUserPostFav
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def with_explicit_time_and_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def with_fa_fav_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def with_inferred_time_and_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def with_recursive(*args, &blk); end
@@ -1288,12 +1282,6 @@ class Domain::UserPostFav::FaUserPostFav
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def with_explicit_time_and_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def with_fa_fav_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def with_inferred_time_and_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def with_recursive(*args, &blk); end

10
spec/controllers/domain/users_controller_spec.rb
View File

@@ -221,8 +221,14 @@ RSpec.describe Domain::UsersController, type: :controller do
def setup_faving_users
# Create E621-specific user-post-fav relationships
Domain::UserPostFav.create!(user: faving_user1, post: domain_post)
Domain::UserPostFav.create!(user: faving_user2, post: domain_post)
Domain::UserPostFav::E621UserPostFav.create!(
user: faving_user1,
post: domain_post,
)
Domain::UserPostFav::E621UserPostFav.create!(
user: faving_user2,
post: domain_post,
)
end
include_examples "users_faving_post action for post type",

6
spec/jobs/domain/e621/job/scan_user_favs_job_spec.rb
View File

@@ -39,7 +39,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
expect(Domain::Post::E621Post.pluck(:e621_id)).to match_array(
[5_212_363, 5_214_461, 5_306_537, 2_518_409, 5_129_881],
)
expect(Domain::UserPostFav.count).to eq(5)
expect(Domain::UserPostFav::E621UserPostFav.count).to eq(5)
post5212363 = Domain::Post::E621Post.find_by(e621_id: 5_212_363)
expect(post5212363).to be_present
@@ -63,7 +63,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
)
# Verify fav relationship
fav = Domain::UserPostFav.find_by(user: user, post: post)
fav = Domain::UserPostFav::E621UserPostFav.find_by(user: user, post: post)
expect(fav).to be_present
end
@@ -137,7 +137,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
it "does not create any favs" do
expect { perform_now({ user: user }) }.not_to change(
Domain::UserPostFav,
Domain::UserPostFav::E621UserPostFav,
:count,
)
end

2
spec/models/domain/user_counter_cache_spec.rb
View File

@@ -159,7 +159,7 @@ RSpec.describe "Domain::User counter caches", type: :model do
user.reload
expect(user.user_post_favs_count).to be_nil
expect(user.user_post_favs.size).to eq(0)
expect(user.user_post_favs.size).to eq(1)
expect(user.user_post_favs.count).to eq(1)
# recompute the value of the counter cache