dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

e621 post aux table migration

Browse Source
This commit is contained in:
Dylan Knutson
2025-07-24 15:44:50 +00:00
parent ad229fbd4e
commit 19fc98e4ef
6 changed files with 252 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
db/migrate/20250723193659_migrate_e621_posts_to_aux.rb Normal file
View File

@@ -0,0 +1,58 @@
# typed: strict
# frozen_string_literal: true
#
class MigrateE621PostsToAux < ActiveRecord::Migration[7.2]
sig { void }
def change
create_aux_table :domain_posts, :e621 do |t|
t =
T.cast(t, ActiveRecord::ConnectionAdapters::PostgreSQL::TableDefinition)
t.string :state, null: false
t.integer :e621_id, null: false
t.timestamp :scanned_post_favs_at
t.string :rating
t.jsonb :tags_array
t.jsonb :flags_array
t.jsonb :pools_array
t.jsonb :sources_array
t.jsonb :artists_array
t.timestamp :e621_updated_at
t.integer :parent_post_e621_id
t.references :last_index_page,
index: false,
foreign_key: {
to_table: :http_log_entries,
}
t.references :caused_by_entry,
index: false,
foreign_key: {
to_table: :http_log_entries,
}
t.references :scan_log_entry,
index: false,
foreign_key: {
to_table: :http_log_entries,
}
t.jsonb :index_page_ids
t.string :description
t.integer :score
t.integer :score_up
t.integer :score_down
t.integer :num_favorites
t.integer :num_comments
t.integer :change_seq
t.string :md5
t.jsonb :prev_md5s
t.string :scan_error
t.references :uploader_user,
index: false,
foreign_key: {
to_table: :domain_users_e621_aux,
primary_key: :base_table_id,
}
end
end
end

73
db/migrate/20250723194407_migrate_e621_post_data_to_aux.rb Normal file
View File

@@ -0,0 +1,73 @@
# typed: strict
# frozen_string_literal: true
class MigrateE621PostDataToAux < ActiveRecord::Migration[7.2]
sig { void }
def up
execute <<~SQL
INSERT INTO domain_posts_e621_aux (
base_table_id,
state,
e621_id,
scanned_post_favs_at,
rating,
tags_array,
flags_array,
pools_array,
sources_array,
artists_array,
e621_updated_at,
parent_post_e621_id,
last_index_page_id,
caused_by_entry_id,
scan_log_entry_id,
index_page_ids,
description,
score,
score_up,
score_down,
num_favorites,
num_comments,
change_seq,
md5,
prev_md5s,
scan_error,
uploader_user_id
)
SELECT
id as base_table_id,
(json_attributes->>'state')::text as state,
(json_attributes->>'e621_id')::integer as e621_id,
(json_attributes->>'scanned_post_favs_at')::timestamp as scanned_post_favs_at,
(json_attributes->>'rating')::text as rating,
(json_attributes->>'tags_array')::jsonb as tags_array,
(json_attributes->>'flags_array')::jsonb as flags_array,
(json_attributes->>'pools_array')::jsonb as pools_array,
(json_attributes->>'sources_array')::jsonb as sources_array,
(json_attributes->>'artists_array')::jsonb as artists_array,
(json_attributes->>'e621_updated_at')::timestamp as e621_updated_at,
(json_attributes->>'parent_post_e621_id')::integer as parent_post_e621_id,
(json_attributes->>'last_index_page_id')::integer as last_index_page_id,
(json_attributes->>'caused_by_entry_id')::integer as caused_by_entry_id,
(json_attributes->>'scan_log_entry_id')::integer as scan_log_entry_id,
(json_attributes->>'index_page_ids')::jsonb as index_page_ids,
(json_attributes->>'description')::text as description,
(json_attributes->>'score')::integer as score,
(json_attributes->>'score_up')::integer as score_up,
(json_attributes->>'score_down')::integer as score_down,
(json_attributes->>'num_favorites')::integer as num_favorites,
(json_attributes->>'num_comments')::integer as num_comments,
(json_attributes->>'change_seq')::integer as change_seq,
(json_attributes->>'md5')::text as md5,
(json_attributes->>'prev_md5s')::jsonb as prev_md5s,
(json_attributes->>'scan_error')::text as scan_error,
(json_attributes->>'uploader_user_id')::integer as uploader_user_id
FROM domain_posts
WHERE type = 'Domain::Post::E621Post'
SQL
end
sig { void }
def down
end
end

118
db/structure.sql
View File

@@ -1336,6 +1336,60 @@ CREATE TABLE public.domain_posts (
);
--
-- Name: domain_posts_e621_aux; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.domain_posts_e621_aux (
base_table_id bigint NOT NULL,
state character varying NOT NULL,
e621_id integer NOT NULL,
scanned_post_favs_at timestamp without time zone,
rating character varying,
tags_array jsonb,
flags_array jsonb,
pools_array jsonb,
sources_array jsonb,
artists_array jsonb,
e621_updated_at timestamp without time zone,
parent_post_e621_id integer,
last_index_page_id bigint,
caused_by_entry_id bigint,
scan_log_entry_id bigint,
index_page_ids jsonb,
description character varying,
score integer,
score_up integer,
score_down integer,
num_favorites integer,
num_comments integer,
change_seq integer,
md5 character varying,
prev_md5s jsonb,
scan_error character varying,
uploader_user_id bigint
);
--
-- Name: domain_posts_e621_aux_base_table_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.domain_posts_e621_aux_base_table_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: domain_posts_e621_aux_base_table_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.domain_posts_e621_aux_base_table_id_seq OWNED BY public.domain_posts_e621_aux.base_table_id;
--
-- Name: domain_posts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@@ -2759,6 +2813,13 @@ ALTER TABLE ONLY public.domain_post_groups ALTER COLUMN id SET DEFAULT nextval('
ALTER TABLE ONLY public.domain_posts ALTER COLUMN id SET DEFAULT nextval('public.domain_posts_id_seq'::regclass);
--
-- Name: domain_posts_e621_aux base_table_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_e621_aux_base_table_id_seq'::regclass);
--
-- Name: domain_twitter_tweets id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -2948,6 +3009,14 @@ ALTER TABLE ONLY public.domain_post_groups
ADD CONSTRAINT domain_post_groups_pkey PRIMARY KEY (id);
--
-- Name: domain_posts_e621_aux domain_posts_e621_aux_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT domain_posts_e621_aux_pkey PRIMARY KEY (base_table_id);
--
-- Name: domain_posts domain_posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -3905,6 +3974,13 @@ CREATE INDEX index_domain_post_group_joins_on_type ON public.domain_post_group_j
CREATE INDEX index_domain_post_groups_on_type ON public.domain_post_groups USING btree (type);
--
-- Name: index_domain_posts_e621_aux_on_base_table_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_posts_e621_aux_on_base_table_id ON public.domain_posts_e621_aux USING btree (base_table_id);
--
-- Name: index_domain_posts_on_posted_at; Type: INDEX; Schema: public; Owner: -
--
@@ -5022,6 +5098,22 @@ ALTER TABLE ONLY public.domain_twitter_medias
ADD CONSTRAINT fk_rails_5fffa41fa6 FOREIGN KEY (file_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_e621_aux fk_rails_73ac068c64; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_73ac068c64 FOREIGN KEY (uploader_user_id) REFERENCES public.domain_users_e621_aux(base_table_id);
--
-- Name: domain_posts_e621_aux fk_rails_7deb1f0178; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_7deb1f0178 FOREIGN KEY (scan_log_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_users_fa_aux fk_rails_7e51f8bfbc; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5062,6 +5154,22 @@ ALTER TABLE ONLY public.domain_user_post_creations
ADD CONSTRAINT fk_rails_9f4b85bc57 FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_e621_aux fk_rails_a90522803d; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_a90522803d FOREIGN KEY (last_index_page_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_e621_aux fk_rails_ae368c64c2; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_ae368c64c2 FOREIGN KEY (base_table_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_user_user_follows fk_rails_b45e6e3979; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5142,6 +5250,14 @@ ALTER TABLE ONLY public.domain_post_files
ADD CONSTRAINT fk_rails_d059c07f77 FOREIGN KEY (log_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_e621_aux fk_rails_d691739802; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_d691739802 FOREIGN KEY (caused_by_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_post_group_joins fk_rails_eddd0a9390; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5181,6 +5297,8 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250723194407'),
('20250723193659'),
('20250722235434'),
('20250722153048'),
('20250722152949'),