dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate Domain::Post::FaPost to aux table

Browse Source
This commit is contained in:
Dylan Knutson
2025-07-26 00:54:03 +00:00
parent 3d2599a4ab
commit ca4729f7d1
8 changed files with 228 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
db/migrate/20250723194407_migrate_e621_post_data_to_aux.rb
View File

@@ -5,7 +5,7 @@ class MigrateE621PostDataToAux < ActiveRecord::Migration[7.2]
sig { void }
def up
execute <<~SQL
INSERT INTO domain_posts_e621_aux (
INSERT INTO domain_posts_e621_aux (
base_table_id,
state,
e621_id,

97
db/migrate/20250725192431_migrate_fa_posts_to_aux.rb Normal file
View File

@@ -0,0 +1,97 @@
# typed: strict
# frozen_string_literal: true
class MigrateFaPostsToAux < ActiveRecord::Migration[7.2]
sig { void }
def up
cols = [
[:state, :string, {}],
[:title, :string, {}],
[:fa_id, :integer, {}],
[:category, :string, {}],
[:theme, :string, {}],
[:species, :string, {}],
[:gender, :string, {}],
[:description, :string, {}],
[:keywords, :jsonb, { default: [] }],
[:num_favorites, :integer, {}],
[:num_comments, :integer, {}],
[:num_views, :integer, {}],
[:scanned_at, :timestamp, {}],
[:scan_file_error, :string, {}],
[
:last_user_page,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[
:first_browse_page,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[
:first_gallery_page,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[
:first_seen_entry,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[:fuzzysearch_checked_at, :timestamp, {}],
[:fuzzysearch_json, :jsonb, {}],
[
:fuzzysearch_entry,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[:tried_from_fur_archiver, :boolean, { default: false }],
[:tried_from_tor, :boolean, { default: false }],
]
create_aux_table :domain_posts, :fa do |t|
cols.each { |name, type, opts| t.send(type, name, **opts) }
end
col_names =
cols.map do |name, type, opts|
type == :references ? "#{name}_id" : "#{name}"
end
col_selects =
cols.map do |name, type, opts|
if type == :references
"(json_attributes->>'#{name}_id')::integer as #{name}_id"
else
type =
case type
when :string
"text"
else
type.to_s
end
"(json_attributes->>'#{name}')::#{type} as #{name}"
end
end
sql = <<~SQL
INSERT INTO domain_posts_fa_aux (
base_table_id,
#{col_names.join(",\n ")}
)
SELECT
id as base_table_id,
#{col_selects.join(",\n ")}
FROM domain_posts
WHERE type = 'Domain::Post::FaPost'
SQL
execute sql
end
sig { void }
def down
drop_table :domain_posts_fa_aux
end
end

122
db/structure.sql
View File

@@ -1390,6 +1390,57 @@ CREATE SEQUENCE public.domain_posts_e621_aux_base_table_id_seq
ALTER SEQUENCE public.domain_posts_e621_aux_base_table_id_seq OWNED BY public.domain_posts_e621_aux.base_table_id;
--
-- Name: domain_posts_fa_aux; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.domain_posts_fa_aux (
base_table_id bigint NOT NULL,
state character varying,
title character varying,
fa_id integer,
category character varying,
theme character varying,
species character varying,
gender character varying,
description character varying,
keywords jsonb DEFAULT '[]'::jsonb,
num_favorites integer,
num_comments integer,
num_views integer,
scanned_at timestamp without time zone,
scan_file_error character varying,
last_user_page_id bigint,
first_browse_page_id bigint,
first_gallery_page_id bigint,
first_seen_entry_id bigint,
fuzzysearch_checked_at timestamp without time zone,
fuzzysearch_json jsonb,
fuzzysearch_entry_id bigint,
tried_from_fur_archiver boolean DEFAULT false,
tried_from_tor boolean DEFAULT false
);
--
-- Name: domain_posts_fa_aux_base_table_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.domain_posts_fa_aux_base_table_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: domain_posts_fa_aux_base_table_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.domain_posts_fa_aux_base_table_id_seq OWNED BY public.domain_posts_fa_aux.base_table_id;
--
-- Name: domain_posts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@@ -2820,6 +2871,13 @@ ALTER TABLE ONLY public.domain_posts ALTER COLUMN id SET DEFAULT nextval('public
ALTER TABLE ONLY public.domain_posts_e621_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_e621_aux_base_table_id_seq'::regclass);
--
-- Name: domain_posts_fa_aux base_table_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_fa_aux_base_table_id_seq'::regclass);
--
-- Name: domain_twitter_tweets id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -3017,6 +3075,14 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT domain_posts_e621_aux_pkey PRIMARY KEY (base_table_id);
--
-- Name: domain_posts_fa_aux domain_posts_fa_aux_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT domain_posts_fa_aux_pkey PRIMARY KEY (base_table_id);
--
-- Name: domain_posts domain_posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -3960,6 +4026,13 @@ CREATE INDEX index_domain_post_groups_on_type ON public.domain_post_groups USING
CREATE INDEX index_domain_posts_e621_aux_on_base_table_id ON public.domain_posts_e621_aux USING btree (base_table_id);
--
-- Name: index_domain_posts_fa_aux_on_base_table_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_posts_fa_aux_on_base_table_id ON public.domain_posts_fa_aux USING btree (base_table_id);
--
-- Name: index_domain_posts_on_posted_at; Type: INDEX; Schema: public; Owner: -
--
@@ -5055,6 +5128,14 @@ ALTER TABLE ONLY public.domain_user_user_follows
ADD CONSTRAINT fk_rails_4b2ab65400 FOREIGN KEY (from_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_fa_aux fk_rails_5ec6ca402e; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT fk_rails_5ec6ca402e FOREIGN KEY (first_seen_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_twitter_medias fk_rails_5fffa41fa6; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5103,6 +5184,14 @@ ALTER TABLE ONLY public.domain_user_search_names
ADD CONSTRAINT fk_rails_8475fe75b5 FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_fa_aux fk_rails_9231b05b1c; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT fk_rails_9231b05b1c FOREIGN KEY (first_browse_page_id) REFERENCES public.http_log_entries(id);
--
-- Name: good_job_execution_log_lines_collections fk_rails_98c288034f; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5127,6 +5216,14 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_a90522803d FOREIGN KEY (last_index_page_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_fa_aux fk_rails_addb1e1118; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT fk_rails_addb1e1118 FOREIGN KEY (first_gallery_page_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_e621_aux fk_rails_ae368c64c2; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5135,6 +5232,14 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_ae368c64c2 FOREIGN KEY (base_table_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_posts_fa_aux fk_rails_ae8beb22db; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT fk_rails_ae8beb22db FOREIGN KEY (fuzzysearch_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_user_user_follows fk_rails_b45e6e3979; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5159,6 +5264,14 @@ ALTER TABLE ONLY public.domain_users_e621_aux
ADD CONSTRAINT fk_rails_b5bacbced6 FOREIGN KEY (base_table_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_fa_aux fk_rails_be2be2e955; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT fk_rails_be2be2e955 FOREIGN KEY (base_table_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_users_inkbunny_aux fk_rails_c2d597dcc4; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5215,6 +5328,14 @@ ALTER TABLE ONLY public.domain_post_files
ADD CONSTRAINT fk_rails_d059c07f77 FOREIGN KEY (log_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_fa_aux fk_rails_d5abc02732; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT fk_rails_d5abc02732 FOREIGN KEY (last_user_page_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_e621_aux fk_rails_d691739802; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5262,6 +5383,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250725192431'),
('20250724213505'),
('20250723194407'),
('20250723193659'),