dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more migration of fa user favs to own table

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-20 04:58:06 +00:00
parent 7bca1452e4
commit a76b10634e
7 changed files with 120 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/models/domain/user_post_fav/fa_user_post_fav.rb
View File

@@ -21,6 +21,14 @@ class Domain::UserPostFav::FaUserPostFav < Domain::UserPostFav
validates :fa_fav_id, uniqueness: true, if: :fa_fav_id?
before_save :set_inferred_time
sig { void }
def set_inferred_time
if (e = explicit_time)
self.inferred_time = e
end
end
sig(:final) { override.returns(T.nilable(FavedAt)) }
def faved_at
if (e = explicit_time)

2
app/views/domain/posts/index_type_headers/_user_created.html.erb
View File

@@ -1,5 +1,5 @@
<h1 class="text-2xl">
<%= link_to user.name_for_view, domain_user_path(user), class: "text-blue-600 hover:text-blue-800" %>'s posts
<%= page_str(params) %>
(<%= posts.total_count %> total)
(<%= relation.total_count %> total)
</h1>

9
backlog/tasks/task-92 - Remove-newlines-from-Bluesky-titles-in-galleries-and-visual-search-results.md
View File

@@ -14,7 +14,12 @@ dependencies: []
Bluesky titles containing newlines can break visual layout and formatting when displayed in gallery views and visual search results, leading to poor user experience and inconsistent presentation
## Acceptance Criteria
<!-- AC:BEGIN -->
- [ ] #1 Newlines are stripped from Bluesky titles in gallery views,Newlines are stripped from Bluesky titles in visual search results,Layout remains consistent across different title formats,No functional regressions in title display,Performance impact is negligible
- [ ] #2 Newlines are stripped from Bluesky titles in gallery views,Newlines are stripped from Bluesky titles in visual search results,Layout remains consistent across different title formats,No functional regressions in title display,Performance impact is negligible
- [ ] #1 Newlines are stripped from Bluesky titles in gallery views
- [ ] #2 Newlines are stripped from Bluesky titles in visual search results
- [ ] #3 Layout remains consistent across different title formats
- [ ] #4 No functional regressions in title display
- [ ] #5 Performance impact is negligible
<!-- AC:END -->

17
db/migrate/20250818235935_create_domain_user_post_favs_fa.rb
View File

@@ -4,19 +4,10 @@
class CreateDomainUserPostFavsFa < ActiveRecord::Migration[7.2]
sig { void }
def change
create_table :domain_user_post_favs_fa, id: false do |t|
t.references :user,
null: false,
index: false,
foreign_key: {
to_table: :domain_users,
}
t.references :post,
null: false,
index: false,
foreign_key: {
to_table: :domain_posts,
}
create_table :domain_user_post_favs_fa,
primary_key: %i[user_id post_id] do |t|
t.bigint :user_id, null: false
t.bigint :post_id, null: false
t.integer :fa_fav_id
t.boolean :removed, null: false, default: false
t.datetime :explicit_time

44
db/migrate/20250819001506_migrate_domain_user_post_favs_fa.rb
View File

@@ -2,9 +2,45 @@
# frozen_string_literal: true
class MigrateDomainUserPostFavsFa < ActiveRecord::Migration[7.2]
disable_ddl_transaction!
sig { void }
def change
up_only { execute <<-SQL }
def up
min_user_id = Domain::User.minimum(:id)
max_user_id = Domain::User.maximum(:id) + 1
max_batch_size = 1000
batch_count = ((max_user_id - min_user_id) / max_batch_size.to_f).ceil
puts "Migrating #{batch_count} batches..."
shards =
batch_count.times.map do |batch_index|
start_user_id = min_user_id + batch_index * max_batch_size
end_user_id = [start_user_id + max_batch_size, max_user_id].min
[start_user_id, end_user_id]
end
num_threads = 4
pool =
T.let(
Concurrent::FixedThreadPool.new(num_threads),
Concurrent::FixedThreadPool,
)
shards.each_with_index do |shard, index|
pool.post do
puts "migrate shard #{index + 1} of #{shards.size}: #{shard.join(" -> ")}"
migrate_shard(*shard)
puts "done: shard #{index + 1} of #{shards.size}: #{shard.join(" -> ")}"
end
end
pool.shutdown
pool.wait_for_termination
end
sig { params(start_user_id: Integer, end_user_id: Integer).void }
def migrate_shard(start_user_id, end_user_id)
ActiveRecord::Base.with_connection do |connection|
connection.execute <<-SQL
INSERT INTO
domain_user_post_favs_fa (
user_id,
@@ -23,6 +59,10 @@ class MigrateDomainUserPostFavsFa < ActiveRecord::Migration[7.2]
to_timestamp((json_attributes->>'inferred_time')::integer) as inferred_time
FROM domain_user_post_favs
WHERE type = 'Domain::UserPostFav::FaUserPostFav'
AND user_id >= #{start_user_id}
AND user_id < #{end_user_id}
ON CONFLICT (user_id, post_id) DO NOTHING
SQL
end
end
end

25
db/migrate/20250819012459_add_indexes_to_user_post_favs_fa.rb
View File

@@ -5,10 +5,31 @@ class AddIndexesToUserPostFavsFa < ActiveRecord::Migration[7.2]
sig { void }
def change
change_table :domain_user_post_favs_fa do |t|
t.index %i[user_id post_id], unique: true
t.index %i[user_id fa_fav_id]
t.index %i[user_id inferred_time]
t.index %i[post_id user_id]
t.index %i[fa_fav_id], unique: true
t.index %i[fa_fav_id explicit_time], where: "explicit_time IS NOT NULL"
end
reversible do |dir|
dir.up do
add_foreign_key :domain_user_post_favs_fa,
:domain_users,
column: :user_id,
index: false,
name: "fk_domain_user_post_favs_fa_user_id"
add_foreign_key :domain_user_post_favs_fa,
:domain_posts,
column: :post_id,
index: false,
name: "fk_domain_user_post_favs_fa_post_id"
end
dir.down do
remove_foreign_key :domain_user_post_favs_fa,
name: "fk_domain_user_post_favs_fa_user_id"
remove_foreign_key :domain_user_post_favs_fa,
name: "fk_domain_user_post_favs_fa_post_id"
end
end
end
end

62
db/structure.sql
View File

@@ -1,4 +1,4 @@
\restrict M2vzhnOdFSJ3FdEc5CUTZFy6hlZAYKe2pY8nWNsQNVQUOuVrSc4yVheBevgryE1
\restrict aPpsXuEdX9tzHTE6ehm7KJzJ68I4JRZwxT9c4Yxb7h7LBSCM3b4TJPJHoQTvG10
-- Dumped from database version 17.6 (Debian 17.6-1.pgdg13+1)
-- Dumped by pg_dump version 17.6 (Debian 17.6-1.pgdg12+1)
@@ -3624,6 +3624,14 @@ ALTER TABLE ONLY public.domain_user_job_event_profile_scans
ADD CONSTRAINT domain_user_job_event_profile_scans_pkey PRIMARY KEY (id);
--
-- Name: domain_user_post_favs_fa domain_user_post_favs_fa_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT domain_user_post_favs_fa_pkey PRIMARY KEY (user_id, post_id);
--
-- Name: domain_user_search_names domain_user_search_names_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -4731,6 +4739,13 @@ CREATE UNIQUE INDEX index_domain_user_post_fav_user_factors_on_user_id ON public
CREATE UNIQUE INDEX index_domain_user_post_favs_fa_on_fa_fav_id ON public.domain_user_post_favs_fa USING btree (fa_fav_id);
--
-- Name: index_domain_user_post_favs_fa_on_fa_fav_id_and_explicit_time; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_user_post_favs_fa_on_fa_fav_id_and_explicit_time ON public.domain_user_post_favs_fa USING btree (fa_fav_id, explicit_time) WHERE (explicit_time IS NOT NULL);
--
-- Name: index_domain_user_post_favs_fa_on_post_id_and_user_id; Type: INDEX; Schema: public; Owner: -
--
@@ -4739,17 +4754,10 @@ CREATE INDEX index_domain_user_post_favs_fa_on_post_id_and_user_id ON public.dom
--
-- Name: index_domain_user_post_favs_fa_on_user_id_and_fa_fav_id; Type: INDEX; Schema: public; Owner: -
-- Name: index_domain_user_post_favs_fa_on_user_id_and_inferred_time; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_user_post_favs_fa_on_user_id_and_fa_fav_id ON public.domain_user_post_favs_fa USING btree (user_id, fa_fav_id);
--
-- Name: index_domain_user_post_favs_fa_on_user_id_and_post_id; Type: INDEX; Schema: public; Owner: -
--
CREATE UNIQUE INDEX index_domain_user_post_favs_fa_on_user_id_and_post_id ON public.domain_user_post_favs_fa USING btree (user_id, post_id);
CREATE INDEX index_domain_user_post_favs_fa_on_user_id_and_inferred_time ON public.domain_user_post_favs_fa USING btree (user_id, inferred_time);
--
@@ -5648,6 +5656,22 @@ ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob
ALTER INDEX public.index_blob_files_on_sha256 ATTACH PARTITION public.index_blob_files_63_on_sha256;
--
-- Name: domain_user_post_favs_fa fk_domain_user_post_favs_fa_post_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_domain_user_post_favs_fa_post_id FOREIGN KEY (post_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_user_post_favs_fa fk_domain_user_post_favs_fa_user_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_domain_user_post_favs_fa_user_id FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: telegram_bot_logs fk_rails_001ca2ed89; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5824,14 +5848,6 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_73ac068c64 FOREIGN KEY (uploader_user_id) REFERENCES public.domain_users_e621_aux(base_table_id);
--
-- Name: domain_user_post_favs_fa fk_rails_7d8aaceb23; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_rails_7d8aaceb23 FOREIGN KEY (post_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_posts_e621_aux fk_rails_7deb1f0178; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -6032,14 +6048,6 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_d691739802 FOREIGN KEY (caused_by_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_user_post_favs_fa fk_rails_d8fc01753d; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_favs_fa
ADD CONSTRAINT fk_rails_d8fc01753d FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_user_job_event_follow_scans fk_rails_ea2f8b74ab; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -6084,7 +6092,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
-- PostgreSQL database dump complete
--
\unrestrict M2vzhnOdFSJ3FdEc5CUTZFy6hlZAYKe2pY8nWNsQNVQUOuVrSc4yVheBevgryE1
\unrestrict aPpsXuEdX9tzHTE6ehm7KJzJ68I4JRZwxT9c4Yxb7h7LBSCM3b4TJPJHoQTvG10
SET search_path TO "$user", public;