dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more migration of fa user favs to own table

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-20 04:58:06 +00:00
parent 7bca1452e4
commit a76b10634e
7 changed files with 120 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
db/migrate/20250818235935_create_domain_user_post_favs_fa.rb
View File

@@ -4,19 +4,10 @@
class CreateDomainUserPostFavsFa < ActiveRecord::Migration[7.2]
sig { void }
def change
create_table :domain_user_post_favs_fa, id: false do |t|
t.references :user,
null: false,
index: false,
foreign_key: {
to_table: :domain_users,
}
t.references :post,
null: false,
index: false,
foreign_key: {
to_table: :domain_posts,
}
create_table :domain_user_post_favs_fa,
primary_key: %i[user_id post_id] do |t|
t.bigint :user_id, null: false
t.bigint :post_id, null: false
t.integer :fa_fav_id
t.boolean :removed, null: false, default: false
t.datetime :explicit_time

44
db/migrate/20250819001506_migrate_domain_user_post_favs_fa.rb
View File

@@ -2,9 +2,45 @@
# frozen_string_literal: true
class MigrateDomainUserPostFavsFa < ActiveRecord::Migration[7.2]
disable_ddl_transaction!
sig { void }
def change
up_only { execute <<-SQL }
def up
min_user_id = Domain::User.minimum(:id)
max_user_id = Domain::User.maximum(:id) + 1
max_batch_size = 1000
batch_count = ((max_user_id - min_user_id) / max_batch_size.to_f).ceil
puts "Migrating #{batch_count} batches..."
shards =
batch_count.times.map do |batch_index|
start_user_id = min_user_id + batch_index * max_batch_size
end_user_id = [start_user_id + max_batch_size, max_user_id].min
[start_user_id, end_user_id]
end
num_threads = 4
pool =
T.let(
Concurrent::FixedThreadPool.new(num_threads),
Concurrent::FixedThreadPool,
)
shards.each_with_index do |shard, index|
pool.post do
puts "migrate shard #{index + 1} of #{shards.size}: #{shard.join(" -> ")}"
migrate_shard(*shard)
puts "done: shard #{index + 1} of #{shards.size}: #{shard.join(" -> ")}"
end
end
pool.shutdown
pool.wait_for_termination
end
sig { params(start_user_id: Integer, end_user_id: Integer).void }
def migrate_shard(start_user_id, end_user_id)
ActiveRecord::Base.with_connection do |connection|
connection.execute <<-SQL
INSERT INTO
domain_user_post_favs_fa (
user_id,
@@ -23,6 +59,10 @@ class MigrateDomainUserPostFavsFa < ActiveRecord::Migration[7.2]
to_timestamp((json_attributes->>'inferred_time')::integer) as inferred_time
FROM domain_user_post_favs
WHERE type = 'Domain::UserPostFav::FaUserPostFav'
AND user_id >= #{start_user_id}
AND user_id < #{end_user_id}
ON CONFLICT (user_id, post_id) DO NOTHING
SQL
end
end
end

25
db/migrate/20250819012459_add_indexes_to_user_post_favs_fa.rb
View File

@@ -5,10 +5,31 @@ class AddIndexesToUserPostFavsFa < ActiveRecord::Migration[7.2]
sig { void }
def change
change_table :domain_user_post_favs_fa do |t|
t.index %i[user_id post_id], unique: true
t.index %i[user_id fa_fav_id]
t.index %i[user_id inferred_time]
t.index %i[post_id user_id]
t.index %i[fa_fav_id], unique: true
t.index %i[fa_fav_id explicit_time], where: "explicit_time IS NOT NULL"
end
reversible do |dir|
dir.up do
add_foreign_key :domain_user_post_favs_fa,
:domain_users,
column: :user_id,
index: false,
name: "fk_domain_user_post_favs_fa_user_id"
add_foreign_key :domain_user_post_favs_fa,
:domain_posts,
column: :post_id,
index: false,
name: "fk_domain_user_post_favs_fa_post_id"
end
dir.down do
remove_foreign_key :domain_user_post_favs_fa,
name: "fk_domain_user_post_favs_fa_user_id"
remove_foreign_key :domain_user_post_favs_fa,
name: "fk_domain_user_post_favs_fa_post_id"
end
end
end
end

62
db/structure.sql
View File

@@ -1,4 +1,4 @@
\restrict M2vzhnOdFSJ3FdEc5CUTZFy6hlZAYKe2pY8nWNsQNVQUOuVrSc4yVheBevgryE1
\restrict aPpsXuEdX9tzHTE6ehm7KJzJ68I4JRZwxT9c4Yxb7h7LBSCM3b4TJPJHoQTvG10
-- Dumped from database version 17.6 (Debian 17.6-1.pgdg13+1)
-- Dumped by pg_dump version 17.6 (Debian 17.6-1.pgdg12+1)
@@ -3624,6 +3624,14 @@ ALTER TABLE ONLY public.domain_user_job_event_profile_scans
ADD CONSTRAINT domain_user_job_event_profile_scans_pkey PRIMARY KEY (id);
--
-- Name: domain_user_post_favs_fa domain_user_post_favs_fa_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT domain_user_post_favs_fa_pkey PRIMARY KEY (user_id, post_id);
--
-- Name: domain_user_search_names domain_user_search_names_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -4731,6 +4739,13 @@ CREATE UNIQUE INDEX index_domain_user_post_fav_user_factors_on_user_id ON public
CREATE UNIQUE INDEX index_domain_user_post_favs_fa_on_fa_fav_id ON public.domain_user_post_favs_fa USING btree (fa_fav_id);
--
-- Name: index_domain_user_post_favs_fa_on_fa_fav_id_and_explicit_time; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_user_post_favs_fa_on_fa_fav_id_and_explicit_time ON public.domain_user_post_favs_fa USING btree (fa_fav_id, explicit_time) WHERE (explicit_time IS NOT NULL);
--
-- Name: index_domain_user_post_favs_fa_on_post_id_and_user_id; Type: INDEX; Schema: public; Owner: -
--
@@ -4739,17 +4754,10 @@ CREATE INDEX index_domain_user_post_favs_fa_on_post_id_and_user_id ON public.dom
--
-- Name: index_domain_user_post_favs_fa_on_user_id_and_fa_fav_id; Type: INDEX; Schema: public; Owner: -
-- Name: index_domain_user_post_favs_fa_on_user_id_and_inferred_time; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_user_post_favs_fa_on_user_id_and_fa_fav_id ON public.domain_user_post_favs_fa USING btree (user_id, fa_fav_id);
--
-- Name: index_domain_user_post_favs_fa_on_user_id_and_post_id; Type: INDEX; Schema: public; Owner: -
--
CREATE UNIQUE INDEX index_domain_user_post_favs_fa_on_user_id_and_post_id ON public.domain_user_post_favs_fa USING btree (user_id, post_id);
CREATE INDEX index_domain_user_post_favs_fa_on_user_id_and_inferred_time ON public.domain_user_post_favs_fa USING btree (user_id, inferred_time);
--
@@ -5648,6 +5656,22 @@ ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob
ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob_files_63_on_sha256;
--
-- Name: domain_user_post_favs_fa fk_domain_user_post_favs_fa_post_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_domain_user_post_favs_fa_post_id FOREIGN KEY (post_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_user_post_favs_fa fk_domain_user_post_favs_fa_user_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_domain_user_post_favs_fa_user_id FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: telegram_bot_logs fk_rails_001ca2ed89; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5824,14 +5848,6 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_73ac068c64 FOREIGN KEY (uploader_user_id) REFERENCES public.domain_users_e621_aux(base_table_id);
--
-- Name: domain_user_post_favs_fa fk_rails_7d8aaceb23; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_rails_7d8aaceb23 FOREIGN KEY (post_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_posts_e621_aux fk_rails_7deb1f0178; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -6032,14 +6048,6 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_d691739802 FOREIGN KEY (caused_by_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_user_post_favs_fa fk_rails_d8fc01753d; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_rails_d8fc01753d FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_user_job_event_follow_scans fk_rails_ea2f8b74ab; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -6084,7 +6092,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
-- PostgreSQL database dump complete
--
\unrestrict M2vzhnOdFSJ3FdEc5CUTZFy6hlZAYKe2pY8nWNsQNVQUOuVrSc4yVheBevgryE1
\unrestrict aPpsXuEdX9tzHTE6ehm7KJzJ68I4JRZwxT9c4Yxb7h7LBSCM3b4TJPJHoQTvG10
SET search_path TO "$user", public;