migrate e621 favs to own table

app/jobs/domain/e621/job/scan_user_favs_job.rb

@@ -121,9 +121,11 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
     logger.info "upserting #{post_ids.size} favs"
     post_ids.each_slice(1000) do |slice|
       ReduxApplicationRecord.transaction do
-        Domain::UserPostFav.upsert_all(
+        Domain::UserPostFav::E621UserPostFav.upsert_all(
-          slice.map { |post_id| { user_id: user.id, post_id: post_id } },
+          slice.map do |post_id|
-          unique_by: :index_domain_user_post_favs_on_user_id_and_post_id,
+            { user_id: user.id, post_id: post_id, removed: false }
+          end,
+          unique_by: %i[user_id post_id],
         )
       end
     end

app/models/domain/post/e621_post.rb

@@ -3,7 +3,7 @@ class Domain::Post::E621Post < Domain::Post
   aux_table :e621
 
   has_single_file!
-  has_faving_users! Domain::User::E621User
+  has_faving_users! Domain::User::E621User, Domain::UserPostFav::E621UserPostFav
   belongs_to_groups! :pools,
                      Domain::PostGroup::E621Pool,
                      Domain::PostGroupJoin::E621PoolJoin

app/models/domain/user.rb

@@ -147,9 +147,30 @@ class Domain::User < ReduxApplicationRecord
              class_name: klass.name
   end
 
-  sig { params(klass: T.class_of(Domain::Post)).void }
+  sig do
-  def self.has_faved_posts!(klass)
+    params(
+      klass: T.class_of(Domain::Post),
+      fav_model_type: T.class_of(Domain::UserPostFav),
+      fav_model_order: T.untyped,
+    ).void
+  end
+  def self.has_faved_posts!(
+    klass,
+    fav_model_type = Domain::UserPostFav,
+    fav_model_order: nil
+  )
     self.class_has_faved_posts = klass
+
+    has_many :user_post_favs,
+             -> do
+               rel = extending(CounterCacheWithFallback[:user_post_favs])
+               rel = rel.order(fav_model_order) if fav_model_order
+               rel
+             end,
+             class_name: fav_model_type.name,
+             inverse_of: :user,
+             dependent: :destroy
+
     has_many :faved_posts,
              -> { order(klass.param_order_attribute => :desc) },
              through: :user_post_favs,

app/models/domain/user/e621_user.rb

@@ -17,7 +17,7 @@ class Domain::User::E621User < Domain::User
   validates :e621_id, presence: true
   validates :name, length: { minimum: 1 }, allow_nil: false
 
-  has_faved_posts! Domain::Post::E621Post
+  has_faved_posts! Domain::Post::E621Post, Domain::UserPostFav::E621UserPostFav
 
   sig { override.returns([String, Symbol]) }
   def self.param_prefix_and_attribute

app/models/domain/user/fa_user.rb

@@ -29,7 +29,11 @@ class Domain::User::FaUser < Domain::User
   has_followed_users! Domain::User::FaUser
   has_followed_by_users! Domain::User::FaUser
   has_created_posts! Domain::Post::FaPost
-  has_faved_posts! Domain::Post::FaPost
+  has_faved_posts! Domain::Post::FaPost,
+                   Domain::UserPostFav::FaUserPostFav,
+                   fav_model_order: {
+                     fa_fav_id: :desc,
+                   }
 
   enum :state,
        { ok: "ok", account_disabled: "account_disabled", error: "error" },

app/models/domain/user_post_fav.rb

@@ -16,6 +16,18 @@ class Domain::UserPostFav < ReduxApplicationRecord
 
   belongs_to :post, class_name: "Domain::Post", inverse_of: :user_post_favs
 
+  sig { params(user_klass: T.class_of(Domain::User), post_klass: T.class_of(Domain::Post)).void }
+  def self.user_post_fav_relationships(user_klass, post_klass)
+    belongs_to_with_counter_cache :user,
+                                  class_name: user_klass.name,
+                                  inverse_of: :user_post_favs,
+                                  counter_cache: :user_post_favs_count
+
+    belongs_to :post,
+               class_name: post_klass.name,
+               inverse_of: :user_post_favs
+  end
+
   scope :for_post_type,
         ->(post_klass) do
           post_klass = T.cast(post_klass, T.class_of(Domain::Post))

app/models/domain/user_post_fav/e621_user_post_fav.rb

@@ -0,0 +1,7 @@
+# typed: strict
+# frozen_string_literal: true
+
+class Domain::UserPostFav::E621UserPostFav < Domain::UserPostFav
+  self.table_name = "domain_user_post_favs_e621"
+  user_post_fav_relationships Domain::User::E621User, Domain::Post::E621Post
+end

app/models/domain/user_post_fav/fa_user_post_fav.rb

@@ -1,24 +1,11 @@
 # typed: strict
 class Domain::UserPostFav::FaUserPostFav < Domain::UserPostFav
   self.table_name = "domain_user_post_favs_fa"
-
+  user_post_fav_relationships Domain::User::FaUser, Domain::Post::FaPost
-  belongs_to_with_counter_cache :user,
-                                class_name: "Domain::User::FaUser",
-                                inverse_of: :user_post_favs,
-                                counter_cache: :user_post_favs_count
-
-  belongs_to :post,
-             class_name: "Domain::Post::FaPost",
-             inverse_of: :user_post_favs
 
   scope :with_explicit_time_and_id,
         -> { where.not(explicit_time: nil).where.not(fa_fav_id: nil) }
 
-  scope :with_inferred_time_and_id,
-        -> { where.not(inferred_time: nil).where.not(fa_fav_id: nil) }
-
-  scope :with_fa_fav_id, -> { where.not(fa_fav_id: nil) }
-
   validates :fa_fav_id, uniqueness: true, if: :fa_fav_id?
 
   before_save :set_inferred_time

db/migrate/20250820204436_remove_fa_user_post_fav_rows.rb

@@ -0,0 +1,12 @@
+class RemoveFaUserPostFavRows < ActiveRecord::Migration[7.2]
+  def up
+    execute <<-SQL
+      DELETE FROM domain_user_post_favs
+      WHERE type = 'Domain::UserPostFav::FaUserPostFav_INVALID'
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20250820205149_create_user_post_favs_e621.rb

@@ -0,0 +1,14 @@
+# typed: strict
+# frozen_string_literal: true
+
+class CreateUserPostFavsE621 < ActiveRecord::Migration[7.2]
+  sig { void }
+  def change
+    create_table :domain_user_post_favs_e621,
+                 primary_key: %i[user_id post_id] do |t|
+      t.bigint :user_id, null: false
+      t.bigint :post_id, null: false
+      t.boolean :removed, null: false, default: false
+    end
+  end
+end

db/migrate/20250820210922_migrate_domain_user_post_favs_e621.rb

@@ -0,0 +1,70 @@
+# typed: strict
+# frozen_string_literal: true
+
+class MigrateDomainUserPostFavsE621 < ActiveRecord::Migration[7.2]
+  disable_ddl_transaction!
+
+  sig { void }
+  def up
+    puts "Getting min/max user id..."
+    min_user_id =
+      Domain::User.where(type: "Domain::User::E621User").minimum(:id)
+    max_user_id =
+      Domain::User.where(type: "Domain::User::E621User").maximum(:id) + 1
+    max_batch_size = 1000
+    batch_count = ((max_user_id - min_user_id) / max_batch_size.to_f).ceil
+
+    puts "Migrating #{batch_count} batches..."
+    shards =
+      T.cast(
+        batch_count.times.map do |batch_index|
+          start_user_id = min_user_id + batch_index * max_batch_size
+          end_user_id = [start_user_id + max_batch_size, max_user_id].min
+
+          Domain::User.where(
+            type: "Domain::User::E621User",
+            id: start_user_id...end_user_id,
+          ).pluck(:id)
+        end,
+        T::Array[T::Array[Integer]],
+      )
+
+    num_threads = 4
+    pool =
+      T.let(
+        Concurrent::FixedThreadPool.new(num_threads),
+        Concurrent::FixedThreadPool,
+      )
+    shards.each_with_index do |shard, index|
+      pool.post do
+        puts "migrate shard #{index + 1} of #{shards.size}: #{shard.minmax.join(" -> ")} (#{shard.size} users)"
+        migrate_shard(shard)
+        puts "done: shard #{index + 1} of #{shards.size}: #{shard.minmax.join(" -> ")}"
+      end
+    end
+
+    pool.shutdown
+    pool.wait_for_termination
+  end
+
+  sig { params(user_ids: T::Array[Integer]).void }
+  def migrate_shard(user_ids)
+    ActiveRecord::Base.with_connection do |connection|
+      connection.execute <<-SQL
+        INSERT INTO
+          domain_user_post_favs_e621 (
+            user_id,
+            post_id,
+            removed
+          )
+        SELECT
+          user_id,
+          post_id,
+          removed
+        FROM domain_user_post_favs
+        WHERE user_id IN (#{user_ids.join(", ")})
+        ON CONFLICT (user_id, post_id) DO NOTHING
+      SQL
+    end
+  end
+end

db/migrate/20250820212318_remove_idx_domain_user_post_favs_on_fav_id.rb

@@ -0,0 +1,21 @@
+class RemoveIdxDomainUserPostFavsOnFavId < ActiveRecord::Migration[7.2]
+  def up
+    execute <<-SQL
+      DROP INDEX index_domain_user_post_favs_on_type_and_user_id;
+      DROP INDEX idx_domain_user_post_favs_on_fav_id;
+    SQL
+  end
+
+  def down
+    execute <<-SQL
+      CREATE UNIQUE INDEX idx_domain_user_post_favs_on_fav_id
+      ON domain_user_post_favs USING btree
+      (((json_attributes ->> 'fav_id'::text)::integer) ASC NULLS LAST)
+      WHERE type = 'Domain::UserPostFav::FaUserPostFav_INVALID'::domain_user_post_fav_type;
+
+      CREATE INDEX index_domain_user_post_favs_on_type_and_user_id
+      ON domain_user_post_favs USING btree
+      (type ASC NULLS LAST, user_id ASC NULLS LAST);
+    SQL
+  end
+end

db/migrate/20250820215340_add_indexes_to_user_post_favs_e621.rb

@@ -0,0 +1,33 @@
+# typed: strict
+#
+class AddIndexesToUserPostFavsE621 < ActiveRecord::Migration[7.2]
+  sig { void }
+  def change
+    change_table :domain_user_post_favs_e621 do |t|
+      t.index %i[post_id user_id]
+    end
+
+    reversible do |dir|
+      dir.up do
+        add_foreign_key :domain_user_post_favs_e621,
+                        :domain_users_e621_aux,
+                        primary_key: :base_table_id,
+                        column: :user_id,
+                        index: false,
+                        name: "fk_domain_user_post_favs_e621_user_id"
+        add_foreign_key :domain_user_post_favs_e621,
+                        :domain_posts_e621_aux,
+                        primary_key: :base_table_id,
+                        column: :post_id,
+                        index: false,
+                        name: "fk_domain_user_post_favs_e621_post_id"
+      end
+      dir.down do
+        remove_foreign_key :domain_user_post_favs_e621,
+                           name: "fk_domain_user_post_favs_e621_user_id"
+        remove_foreign_key :domain_user_post_favs_e621,
+                           name: "fk_domain_user_post_favs_e621_post_id"
+      end
+    end
+  end
+end

db/structure.sql

@@ -1,4 +1,4 @@
-\restrict huxfo3NM6M7Nj4JUdHDWXGq7DoLnw9sDIrV5g3aVIBWghR5vgItCQUJG6cJ3EJf
+\restrict fsnJecu1BUZk39yUe5E3zqbZEe6yKihIQHKEWYdhLLHCBtdUoLtB3L0VBXECSF8
 
 -- Dumped from database version 17.6 (Debian 17.6-1.pgdg13+1)
 -- Dumped by pg_dump version 17.6 (Debian 17.6-1.pgdg12+1)
@@ -1972,6 +1972,19 @@ CREATE TABLE public.domain_user_post_favs (
 WITH (autovacuum_enabled='true');
 
 
+--
+-- Name: domain_user_post_favs_e621; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.domain_user_post_favs_e621 (
+    user_id bigint NOT NULL,
+    post_id bigint NOT NULL,
+    removed boolean DEFAULT false NOT NULL,
+    explicit_time timestamp(6) without time zone,
+    inferred_time timestamp(6) without time zone
+);
+
+
 --
 -- Name: domain_user_post_favs_fa; Type: TABLE; Schema: public; Owner: -
 --
@@ -3624,6 +3637,14 @@ ALTER TABLE ONLY public.domain_user_job_event_profile_scans
     ADD CONSTRAINT domain_user_job_event_profile_scans_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: domain_user_post_favs_e621 domain_user_post_favs_e621_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_user_post_favs_e621
+    ADD CONSTRAINT domain_user_post_favs_e621_pkey PRIMARY KEY (user_id, post_id);
+
+
 --
 -- Name: domain_user_post_favs_fa domain_user_post_favs_fa_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -3913,13 +3934,6 @@ CREATE UNIQUE INDEX idx_domain_post_groups_on_sofurry_id ON public.domain_post_g
 CREATE UNIQUE INDEX idx_domain_posts_on_sofurry_id ON public.domain_posts USING btree ((((json_attributes ->> 'sofurry_id'::text))::integer)) WHERE (type = 'Domain::Post::SofurryPost'::public.domain_post_type);
 
 
---
--- Name: idx_domain_user_post_favs_on_fav_id; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE UNIQUE INDEX idx_domain_user_post_favs_on_fav_id ON public.domain_user_post_favs USING btree ((((json_attributes ->> 'fav_id'::text))::integer)) WHERE (type = 'Domain::UserPostFav::FaUserPostFav_INVALID'::public.domain_user_post_fav_type);
-
-
 --
 -- Name: idx_domain_users_e621_on_name_lower; Type: INDEX; Schema: public; Owner: -
 --
@@ -4732,6 +4746,13 @@ CREATE INDEX index_domain_user_post_fav_user_factors_on_embedding ON public.doma
 CREATE UNIQUE INDEX index_domain_user_post_fav_user_factors_on_user_id ON public.domain_user_post_fav_user_factors USING btree (user_id);
 
 
+--
+-- Name: index_domain_user_post_favs_e621_on_post_id_and_user_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_domain_user_post_favs_e621_on_post_id_and_user_id ON public.domain_user_post_favs_e621 USING btree (post_id, user_id);
+
+
 --
 -- Name: index_domain_user_post_favs_fa_on_fa_fav_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -5656,6 +5677,22 @@ ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob
 ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob_files_63_on_sha256;
 
 
+--
+-- Name: domain_user_post_favs_e621 fk_domain_user_post_favs_e621_post_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_user_post_favs_e621
+    ADD CONSTRAINT fk_domain_user_post_favs_e621_post_id FOREIGN KEY (post_id) REFERENCES public.domain_posts_e621_aux(base_table_id);
+
+
+--
+-- Name: domain_user_post_favs_e621 fk_domain_user_post_favs_e621_user_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_user_post_favs_e621
+    ADD CONSTRAINT fk_domain_user_post_favs_e621_user_id FOREIGN KEY (user_id) REFERENCES public.domain_users_e621_aux(base_table_id);
+
+
 --
 -- Name: domain_user_post_favs_fa fk_domain_user_post_favs_fa_post_id; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -6092,11 +6129,16 @@ ALTER TABLE ONLY public.domain_twitter_tweets
 -- PostgreSQL database dump complete
 --
 
-\unrestrict huxfo3NM6M7Nj4JUdHDWXGq7DoLnw9sDIrV5g3aVIBWghR5vgItCQUJG6cJ3EJf
+\unrestrict fsnJecu1BUZk39yUe5E3zqbZEe6yKihIQHKEWYdhLLHCBtdUoLtB3L0VBXECSF8
 
 SET search_path TO "$user", public;
 
 INSERT INTO "schema_migrations" (version) VALUES
+('20250820215340'),
+('20250820212318'),
+('20250820210922'),
+('20250820205149'),
+('20250820204436'),
 ('20250820145726'),
 ('20250819012459'),
 ('20250819001506'),

sorbet/rbi/dsl/domain/post/e621_post.rbi

@@ -719,10 +719,10 @@ class Domain::Post::E621Post
 
     # This method is created by ActiveRecord on the `Domain::Post::E621Post` class because it declared `has_many :user_post_favs`.
     # 🔗 [Rails guide for `has_many` association](https://guides.rubyonrails.org/association_basics.html#the-has-many-association)
-    sig { returns(::Domain::UserPostFav::PrivateCollectionProxy) }
+    sig { returns(::Domain::UserPostFav::E621UserPostFav::PrivateCollectionProxy) }
     def user_post_favs; end
 
-    sig { params(value: T::Enumerable[::Domain::UserPostFav]).void }
+    sig { params(value: T::Enumerable[::Domain::UserPostFav::E621UserPostFav]).void }
     def user_post_favs=(value); end
   end
 

sorbet/rbi/dsl/domain/user/e621_user.rbi

@@ -672,12 +672,12 @@ class Domain::User::E621User
     sig { params(ids: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
     def user_post_fav_ids=(ids); end
 
-    # This method is created by ActiveRecord on the `Domain::User` class because it declared `has_many :user_post_favs`.
+    # This method is created by ActiveRecord on the `Domain::User::E621User` class because it declared `has_many :user_post_favs`.
     # 🔗 [Rails guide for `has_many` association](https://guides.rubyonrails.org/association_basics.html#the-has-many-association)
-    sig { returns(::Domain::UserPostFav::PrivateCollectionProxy) }
+    sig { returns(::Domain::UserPostFav::E621UserPostFav::PrivateCollectionProxy) }
     def user_post_favs; end
 
-    sig { params(value: T::Enumerable[::Domain::UserPostFav]).void }
+    sig { params(value: T::Enumerable[::Domain::UserPostFav::E621UserPostFav]).void }
     def user_post_favs=(value); end
 
     sig { returns(T::Array[T.untyped]) }

sorbet/rbi/dsl/domain/user/inkbunny_user.rbi

@@ -699,7 +699,7 @@ class Domain::User::InkbunnyUser
     sig { params(ids: T::Array[T.untyped]).returns(T::Array[T.untyped]) }
     def user_post_fav_ids=(ids); end
 
-    # This method is created by ActiveRecord on the `Domain::User` class because it declared `has_many :user_post_favs`.
+    # This method is created by ActiveRecord on the `Domain::User::InkbunnyUser` class because it declared `has_many :user_post_favs`.
     # 🔗 [Rails guide for `has_many` association](https://guides.rubyonrails.org/association_basics.html#the-has-many-association)
     sig { returns(::Domain::UserPostFav::PrivateCollectionProxy) }
     def user_post_favs; end

sorbet/rbi/dsl/domain/user_post_fav/fa_user_post_fav.rbi

@@ -665,12 +665,6 @@ class Domain::UserPostFav::FaUserPostFav
     sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
     def with_explicit_time_and_id(*args, &blk); end
 
-    sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
-    def with_fa_fav_id(*args, &blk); end
-
-    sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
-    def with_inferred_time_and_id(*args, &blk); end
-
     sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
     def with_recursive(*args, &blk); end
 
@@ -1288,12 +1282,6 @@ class Domain::UserPostFav::FaUserPostFav
     sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
     def with_explicit_time_and_id(*args, &blk); end
 
-    sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
-    def with_fa_fav_id(*args, &blk); end
-
-    sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
-    def with_inferred_time_and_id(*args, &blk); end
-
     sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
     def with_recursive(*args, &blk); end
 

spec/controllers/domain/users_controller_spec.rb

@@ -221,8 +221,14 @@ RSpec.describe Domain::UsersController, type: :controller do
 
       def setup_faving_users
         # Create E621-specific user-post-fav relationships
-        Domain::UserPostFav.create!(user: faving_user1, post: domain_post)
+        Domain::UserPostFav::E621UserPostFav.create!(
-        Domain::UserPostFav.create!(user: faving_user2, post: domain_post)
+          user: faving_user1,
+          post: domain_post,
+        )
+        Domain::UserPostFav::E621UserPostFav.create!(
+          user: faving_user2,
+          post: domain_post,
+        )
       end
 
       include_examples "users_faving_post action for post type",

spec/jobs/domain/e621/job/scan_user_favs_job_spec.rb

@@ -39,7 +39,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
       expect(Domain::Post::E621Post.pluck(:e621_id)).to match_array(
         [5_212_363, 5_214_461, 5_306_537, 2_518_409, 5_129_881],
       )
-      expect(Domain::UserPostFav.count).to eq(5)
+      expect(Domain::UserPostFav::E621UserPostFav.count).to eq(5)
 
       post5212363 = Domain::Post::E621Post.find_by(e621_id: 5_212_363)
       expect(post5212363).to be_present
@@ -63,7 +63,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
       )
 
       # Verify fav relationship
-      fav = Domain::UserPostFav.find_by(user: user, post: post)
+      fav = Domain::UserPostFav::E621UserPostFav.find_by(user: user, post: post)
       expect(fav).to be_present
     end
 
@@ -137,7 +137,7 @@ RSpec.describe Domain::E621::Job::ScanUserFavsJob do
 
       it "does not create any favs" do
         expect { perform_now({ user: user }) }.not_to change(
-          Domain::UserPostFav,
+          Domain::UserPostFav::E621UserPostFav,
           :count,
         )
       end

spec/models/domain/user_counter_cache_spec.rb

@@ -159,7 +159,7 @@ RSpec.describe "Domain::User counter caches", type: :model do
 
     user.reload
     expect(user.user_post_favs_count).to be_nil
-    expect(user.user_post_favs.size).to eq(0)
+    expect(user.user_post_favs.size).to eq(1)
     expect(user.user_post_favs.count).to eq(1)
 
     # recompute the value of the counter cache