dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

skyfall gem, rework migrations

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-05 19:21:38 +00:00
parent e9f3b0e822
commit 3512c3f32e
24 changed files with 10298 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
db/migrate/20250805070114_add_aux_tables_for_domain_users_bluesky_users.rb
View File

@@ -6,29 +6,33 @@ class AddAuxTablesForDomainUsersBlueskyUsers < ActiveRecord::Migration[7.2]
sig { void }
def change
add_enum_value :domain_user_type, "Domain::User::BlueskyUser"
create_aux_table :domain_users, :bluesky do |t|
t.string :state, null: false
# handle is the "username"
t.string :handle, null: false, index: true
t.string :display_name
# profile "bio"
t.text :description
t.string :state, null: false, default: "ok"
# Bluesky-specific fields
t.string :did, index: true # Decentralized identifier
t.string :avatar_url
t.integer :followers_count, default: 0
t.integer :following_count, default: 0
t.integer :posts_count, default: 0
t.integer :followers_count
t.integer :following_count
t.integer :posts_count
# avatar is tracked via Domain::UserAvatar, not here
# Scanning timestamps
t.datetime :scanned_profile_at
t.datetime :scanned_posts_at
# Log entries for tracking scans
t.references :last_profile_scan,
t.references :first_seen_entry,
foreign_key: {
to_table: :http_log_entries,
}
t.references :last_posts_scan,
t.references :last_seen_entry,
foreign_key: {
to_table: :http_log_entries,
}

17
db/migrate/20250805070115_add_aux_tables_for_domain_posts_bluesky_posts.rb
View File

@@ -6,21 +6,22 @@ class AddAuxTablesForDomainPostsBlueskyPosts < ActiveRecord::Migration[7.2]
sig { void }
def change
add_enum_value :domain_post_type, "Domain::Post::BlueskyPost"
create_aux_table :domain_posts, :bluesky do |t|
t.string :state, null: false
t.string :bluesky_id, null: false, index: true # Record key from AT URI
t.string :at_uri, index: true # Full AT Protocol URI
t.text :text # Post content
t.string :state, null: false, default: "ok"
# Post metadata (posted_at is in main table)
t.datetime :scanned_at
t.string :language, limit: 10
t.string :language
# Engagement metrics
t.integer :like_count, default: 0
t.integer :repost_count, default: 0
t.integer :reply_count, default: 0
t.integer :quote_count, default: 0
t.integer :like_count
t.integer :repost_count
t.integer :reply_count
t.integer :quote_count
# Content arrays
t.column :hashtags, :jsonb, default: []
@@ -32,10 +33,6 @@ class AddAuxTablesForDomainPostsBlueskyPosts < ActiveRecord::Migration[7.2]
t.string :quote_uri # AT URI of post being quoted
# Log entries for tracking scans
t.references :last_profile_scan,
foreign_key: {
to_table: :http_log_entries,
}
t.references :first_seen_entry,
foreign_key: {
to_table: :http_log_entries,

15
db/migrate/20250805071440_remove_last_profile_scan_from_bluesky_posts.rb
View File

@@ -1,15 +0,0 @@
# typed: strict
# frozen_string_literal: true
class RemoveLastProfileScanFromBlueskyPosts < ActiveRecord::Migration[7.2]
extend T::Sig
sig { void }
def change
remove_reference :domain_posts_bluesky_aux,
:last_profile_scan,
foreign_key: {
to_table: :http_log_entries,
}
end
end

14
db/migrate/20250805191557_add_aux_tables_for_domain_post_files_bluesky.rb Normal file
View File

@@ -0,0 +1,14 @@
# typed: strict
# frozen_string_literal: true
class AddAuxTablesForDomainPostFilesBluesky < ActiveRecord::Migration[7.2]
extend T::Sig
sig { void }
def change
add_enum_value :domain_post_file_type, "Domain::PostFile::BlueskyPostFile"
create_aux_table :domain_post_files, :bluesky do |t|
t.integer :file_order, null: false
end
end
end

145
db/structure.sql
View File

@@ -100,7 +100,8 @@ COMMENT ON EXTENSION vector IS 'vector data type and ivfflat access method';
CREATE TYPE public.domain_post_file_type AS ENUM (
'Domain::PostFile',
'Domain::PostFile::InkbunnyPostFile'
'Domain::PostFile::InkbunnyPostFile',
'Domain::PostFile::BlueskyPostFile'
);
@@ -135,7 +136,8 @@ CREATE TYPE public.domain_post_type AS ENUM (
'Domain::Post::E621Post',
'Domain::Post::InkbunnyPost',
'Domain::Post::SofurryPost',
'Domain::Post::WeasylPost'
'Domain::Post::WeasylPost',
'Domain::Post::BlueskyPost'
);
@@ -158,7 +160,8 @@ CREATE TYPE public.domain_user_type AS ENUM (
'Domain::User::E621User',
'Domain::User::InkbunnyUser',
'Domain::User::SofurryUser',
'Domain::User::WeasylUser'
'Domain::User::WeasylUser',
'Domain::User::BlueskyUser'
);
@@ -1221,6 +1224,35 @@ CREATE TABLE public.domain_post_files (
);
--
-- Name: domain_post_files_bluesky_aux; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.domain_post_files_bluesky_aux (
base_table_id bigint NOT NULL,
file_order integer NOT NULL
);
--
-- Name: domain_post_files_bluesky_aux_base_table_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.domain_post_files_bluesky_aux_base_table_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: domain_post_files_bluesky_aux_base_table_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.domain_post_files_bluesky_aux_base_table_id_seq OWNED BY public.domain_post_files_bluesky_aux.base_table_id;
--
-- Name: domain_post_files_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@@ -1342,16 +1374,16 @@ CREATE TABLE public.domain_posts (
CREATE TABLE public.domain_posts_bluesky_aux (
base_table_id bigint NOT NULL,
state character varying NOT NULL,
bluesky_id character varying NOT NULL,
at_uri character varying,
text text,
state character varying DEFAULT 'ok'::character varying NOT NULL,
scanned_at timestamp(6) without time zone,
language character varying(10),
like_count integer DEFAULT 0,
repost_count integer DEFAULT 0,
reply_count integer DEFAULT 0,
quote_count integer DEFAULT 0,
language character varying,
like_count integer,
repost_count integer,
reply_count integer,
quote_count integer,
hashtags jsonb DEFAULT '[]'::jsonb,
mentions jsonb DEFAULT '[]'::jsonb,
links jsonb DEFAULT '[]'::jsonb,
@@ -1879,19 +1911,18 @@ CREATE TABLE public.domain_users (
CREATE TABLE public.domain_users_bluesky_aux (
base_table_id bigint NOT NULL,
state character varying NOT NULL,
handle character varying NOT NULL,
display_name character varying,
description text,
state character varying DEFAULT 'ok'::character varying NOT NULL,
did character varying,
avatar_url character varying,
followers_count integer DEFAULT 0,
following_count integer DEFAULT 0,
posts_count integer DEFAULT 0,
followers_count integer,
following_count integer,
posts_count integer,
scanned_profile_at timestamp(6) without time zone,
scanned_posts_at timestamp(6) without time zone,
last_profile_scan_id bigint,
last_posts_scan_id bigint,
first_seen_entry_id bigint,
last_seen_entry_id bigint,
profile_raw jsonb DEFAULT '{}'::jsonb
);
@@ -3023,6 +3054,13 @@ ALTER TABLE ONLY public.domain_post_file_thumbnails ALTER COLUMN id SET DEFAULT
ALTER TABLE ONLY public.domain_post_files ALTER COLUMN id SET DEFAULT nextval('public.domain_post_files_id_seq'::regclass);
--
-- Name: domain_post_files_bluesky_aux base_table_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_post_files_bluesky_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_post_files_bluesky_aux_base_table_id_seq'::regclass);
--
-- Name: domain_post_files_inkbunny_aux base_table_id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -3251,6 +3289,14 @@ ALTER TABLE ONLY public.domain_post_file_thumbnails
ADD CONSTRAINT domain_post_file_thumbnails_pkey PRIMARY KEY (id);
--
-- Name: domain_post_files_bluesky_aux domain_post_files_bluesky_aux_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_post_files_bluesky_aux
ADD CONSTRAINT domain_post_files_bluesky_aux_pkey PRIMARY KEY (base_table_id);
--
-- Name: domain_post_files_inkbunny_aux domain_post_files_inkbunny_aux_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -4203,6 +4249,13 @@ CREATE INDEX index_domain_fa_fav_id_and_dates_on_user_id ON public.domain_fa_fav
CREATE UNIQUE INDEX index_domain_fa_fav_id_and_dates_on_user_id_and_post_fa_id ON public.domain_fa_fav_id_and_dates USING btree (user_id, post_fa_id);
--
-- Name: index_domain_post_files_bluesky_aux_on_base_table_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_post_files_bluesky_aux_on_base_table_id ON public.domain_post_files_bluesky_aux USING btree (base_table_id);
--
-- Name: index_domain_post_files_inkbunny_aux_on_base_table_id; Type: INDEX; Schema: public; Owner: -
--
@@ -4560,6 +4613,13 @@ CREATE INDEX index_domain_users_bluesky_aux_on_base_table_id ON public.domain_us
CREATE INDEX index_domain_users_bluesky_aux_on_did ON public.domain_users_bluesky_aux USING btree (did);
--
-- Name: index_domain_users_bluesky_aux_on_first_seen_entry_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_users_bluesky_aux_on_first_seen_entry_id ON public.domain_users_bluesky_aux USING btree (first_seen_entry_id);
--
-- Name: index_domain_users_bluesky_aux_on_handle; Type: INDEX; Schema: public; Owner: -
--
@@ -4568,17 +4628,10 @@ CREATE INDEX index_domain_users_bluesky_aux_on_handle ON public.domain_users_blu
--
-- Name: index_domain_users_bluesky_aux_on_last_posts_scan_id; Type: INDEX; Schema: public; Owner: -
-- Name: index_domain_users_bluesky_aux_on_last_seen_entry_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_users_bluesky_aux_on_last_posts_scan_id ON public.domain_users_bluesky_aux USING btree (last_posts_scan_id);
--
-- Name: index_domain_users_bluesky_aux_on_last_profile_scan_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_users_bluesky_aux_on_last_profile_scan_id ON public.domain_users_bluesky_aux USING btree (last_profile_scan_id);
CREATE INDEX index_domain_users_bluesky_aux_on_last_seen_entry_id ON public.domain_users_bluesky_aux USING btree (last_seen_entry_id);
--
@@ -5495,6 +5548,14 @@ ALTER TABLE ONLY public.http_log_entries
ADD CONSTRAINT fk_rails_42f35e9da0 FOREIGN KEY (response_headers_id) REFERENCES public.http_log_entry_headers(id);
--
-- Name: domain_post_files_bluesky_aux fk_rails_47e4648919; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_post_files_bluesky_aux
ADD CONSTRAINT fk_rails_47e4648919 FOREIGN KEY (base_table_id) REFERENCES public.domain_post_files(id);
--
-- Name: domain_fa_fav_id_and_dates fk_rails_4ad7be007e; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5511,14 +5572,6 @@ ALTER TABLE ONLY public.domain_user_user_follows
ADD CONSTRAINT fk_rails_4b2ab65400 FOREIGN KEY (from_id) REFERENCES public.domain_users(id);
--
-- Name: domain_users_bluesky_aux fk_rails_56b1754cfb; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_users_bluesky_aux
ADD CONSTRAINT fk_rails_56b1754cfb FOREIGN KEY (last_profile_scan_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_posts_ib_aux fk_rails_5ee2c344bd; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5591,6 +5644,14 @@ ALTER TABLE ONLY public.domain_user_search_names
ADD CONSTRAINT fk_rails_8475fe75b5 FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_users_bluesky_aux fk_rails_986ba96a1b; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_users_bluesky_aux
ADD CONSTRAINT fk_rails_986ba96a1b FOREIGN KEY (first_seen_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: good_job_execution_log_lines_collections fk_rails_98c288034f; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5727,14 +5788,6 @@ ALTER TABLE ONLY public.domain_posts_e621_aux
ADD CONSTRAINT fk_rails_d691739802 FOREIGN KEY (caused_by_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_users_bluesky_aux fk_rails_d8ceec56ec; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_users_bluesky_aux
ADD CONSTRAINT fk_rails_d8ceec56ec FOREIGN KEY (last_posts_scan_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_post_group_joins fk_rails_eddd0a9390; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5759,6 +5812,14 @@ ALTER TABLE ONLY public.domain_user_avatars
ADD CONSTRAINT fk_rails_f89912a20f FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_users_bluesky_aux fk_rails_fd9e7916ad; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_users_bluesky_aux
ADD CONSTRAINT fk_rails_fd9e7916ad FOREIGN KEY (last_seen_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_twitter_tweets on_author_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5774,7 +5835,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250805071440'),
('20250805191557'),
('20250805070115'),
('20250805070114'),
('20250805045947'),