migrate ib posts to aux table

2025-07-27 17:54:29 +00:00
parent 1f44ec2fa2
commit c43d1ca197
9 changed files with 255 additions and 94 deletions
--- a/app/helpers/domain/posts_helper.rb
+++ b/app/helpers/domain/posts_helper.rb
@@ -470,9 +470,13 @@ module Domain::PostsHelper
    post.keywords.map(&:strip).reject(&:blank?).compact
  end

-  sig { params(post: Domain::Post::InkbunnyPost).returns(T::Array[String]) }
+  sig do
+    params(post: Domain::Post::InkbunnyPost).returns(
+      T.nilable(T::Array[String]),
+    )
+  end
  def keywords_for_ib_post(post)
-    post.keywords.map { |keyword| keyword["keyword_name"] }.compact
+    post.keywords&.map { |keyword| keyword["keyword_name"] }&.compact
  end

  sig do
--- a/app/models/domain/post/inkbunny_post.rb
+++ b/app/models/domain/post/inkbunny_post.rb
@@ -1,6 +1,8 @@
 # typed: strict
 # frozen_string_literal: true
 class Domain::Post::InkbunnyPost < Domain::Post
+  aux_table :ib, allow_redefining: :title
+
  SUBMISSION_TYPE_MAP =
    T.let(
      {
@@ -29,25 +31,6 @@ class Domain::Post::InkbunnyPost < Domain::Post
      T::Hash[Integer, String],
    )

-  attr_json :ib_id, :integer
-  attr_json :state, :string
-  attr_json :rating, :string
-  attr_json :submission_type, :string
-  attr_json :title, :string
-  attr_json :writing, :string
-  attr_json :description, :string
-  attr_json :num_views, :integer
-  attr_json :num_files, :integer
-  attr_json :num_favs, :integer
-  attr_json :num_comments, :integer
-  attr_json :keywords, ActiveModel::Type::Value.new
-  attr_json :last_file_updated_at, ActiveModelUtcTimeValue.new
-  attr_json :deep_update_log_entry_id, :integer
-  attr_json :shallow_update_log_entry_id, :integer
-  attr_json :shallow_updated_at, ActiveModelUtcTimeValue.new
-  attr_json :deep_updated_at, ActiveModelUtcTimeValue.new
-  attr_json :ib_detail_raw, ActiveModel::Type::Value.new
-
  has_multiple_files! Domain::PostFile::InkbunnyPostFile
  has_single_creator! Domain::User::InkbunnyUser
  has_faving_users! Domain::User::InkbunnyUser
@@ -105,11 +88,6 @@ class Domain::Post::InkbunnyPost < Domain::Post
    self.description
  end

-  sig { override.returns(T.nilable(String)) }
-  def title
-    super
-  end
-
  sig { override.returns(T.nilable(Domain::User)) }
  def primary_creator_for_view
    self.creator
--- a/app/views/domain/posts/e621/_section_title_details.html.erb
+++ b/app/views/domain/posts/e621/_section_title_details.html.erb
@@ -1,4 +1 @@
-<span>
-  <i class="fa-solid fa-tag mr-1"></i>
-  Rating: <%= post.rating_for_view %>
-</span>
+<%= render partial: "domain/posts/title_stat", locals: { label: "Rating", value: post.rating_for_view, icon_class: "fa-tag" } %>
--- a/app/views/domain/posts/ib/_section_tags.html.erb
+++ b/app/views/domain/posts/ib/_section_tags.html.erb
@@ -1,6 +1,6 @@
 <%= sky_section_tag("Keywords") do %>
  <% keywords = keywords_for_ib_post(post) %>
-  <% if keywords.any? %>
+  <% if keywords&.any? %>
    <div class="flex flex-wrap gap-2">
      <% keywords.each do |keyword| %>
        <span
--- a/app/views/domain/posts/ib/_section_title_details.html.erb
+++ b/app/views/domain/posts/ib/_section_title_details.html.erb
@@ -1,12 +1,3 @@
-<span>
-  <i class="fa-solid fa-eye mr-1"></i>
-  Views: <%= post.num_views %>
-</span>
-<span>
-  <i class="fa-solid fa-file-image mr-1"></i>
-  Files: <%= post.num_files %>
-</span>
-<span>
-  <i class="fa-solid fa-comment mr-1"></i>
-  Comments: <%= post.num_comments %>
-</span>
+<%= render partial: "domain/posts/title_stat", locals: { label: "Views", value: post.num_views, icon_class: "fa-eye" } %>
+<%= render partial: "domain/posts/title_stat", locals: { label: "Files", value: post.num_files, icon_class: "fa-file-image" } %>
+<%= render partial: "domain/posts/title_stat", locals: { label: "Comments", value: post.num_comments, icon_class: "fa-comment" } %>
--- a/db/migrate/20250727173100_migrate_ib_post_data_to_aux.rb
+++ b/db/migrate/20250727173100_migrate_ib_post_data_to_aux.rb
@@ -0,0 +1,72 @@
+# typed: strict
+# frozen_string_literal: true
+
+class MigrateIbPostDataToAux < ActiveRecord::Migration[7.2]
+  sig { void }
+  def up
+    cols = [
+      [:ib_id, :integer, { index: true }],
+      [:state, :string, {}],
+      [:rating, :string, {}],
+      [:submission_type, :string, {}],
+      [:title, :string, {}],
+      [:writing, :string, {}],
+      [:description, :string, {}],
+      [:num_views, :integer, {}],
+      [:num_files, :integer, {}],
+      [:num_favs, :integer, {}],
+      [:num_comments, :integer, {}],
+      [:keywords, :jsonb, { default: [] }],
+      [:last_file_updated_at, :timestamp, {}],
+      [
+        :deep_update_log_entry,
+        :references,
+        { foreign_key: { to_table: :http_log_entries }, index: false },
+      ],
+      [
+        :shallow_update_log_entry,
+        :references,
+        { foreign_key: { to_table: :http_log_entries }, index: false },
+      ],
+      [:shallow_updated_at, :timestamp, {}],
+      [:deep_updated_at, :timestamp, {}],
+      [:ib_detail_raw, :jsonb, {}],
+    ]
+
+    create_aux_table :domain_posts, :ib do |t|
+      cols.each { |name, type, opts| t.send(type, name, **opts) }
+    end
+
+    col_names =
+      cols.map do |name, type, opts|
+        type == :references ? "#{name}_id" : "#{name}"
+      end
+
+    col_selects =
+      cols.map do |name, type, opts|
+        if type == :references
+          "(json_attributes->>'#{name}_id')::integer as #{name}_id"
+        elsif type == :string
+          "(json_attributes->>'#{name}')::text as #{name}"
+        else
+          "(json_attributes->>'#{name}')::#{type} as #{name}"
+        end
+      end
+
+    execute <<~SQL
+      INSERT INTO domain_posts_ib_aux (
+        base_table_id,
+        #{col_names.join(",\n    ")}
+      )
+      SELECT
+        id as base_table_id,
+        #{col_selects.join(",\n    ")}
+      FROM domain_posts WHERE type = 'Domain::Post::InkbunnyPost'
+    SQL
+  end
+
+  sig { void }
+  def down
+    drop_table :domain_posts_ib_aux
+  end
+end
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1441,6 +1441,52 @@ CREATE SEQUENCE public.domain_posts_fa_aux_base_table_id_seq
 ALTER SEQUENCE public.domain_posts_fa_aux_base_table_id_seq OWNED BY public.domain_posts_fa_aux.base_table_id;


+--
+-- Name: domain_posts_ib_aux; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.domain_posts_ib_aux (
+    base_table_id bigint NOT NULL,
+    ib_id integer,
+    state character varying,
+    rating character varying,
+    submission_type character varying,
+    title character varying,
+    writing character varying,
+    description character varying,
+    num_views integer,
+    num_files integer,
+    num_favs integer,
+    num_comments integer,
+    keywords jsonb DEFAULT '[]'::jsonb,
+    last_file_updated_at timestamp without time zone,
+    deep_update_log_entry_id bigint,
+    shallow_update_log_entry_id bigint,
+    shallow_updated_at timestamp without time zone,
+    deep_updated_at timestamp without time zone,
+    ib_detail_raw jsonb
+);
+
+
+--
+-- Name: domain_posts_ib_aux_base_table_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.domain_posts_ib_aux_base_table_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: domain_posts_ib_aux_base_table_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.domain_posts_ib_aux_base_table_id_seq OWNED BY public.domain_posts_ib_aux.base_table_id;
+
+
 --
 -- Name: domain_posts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -2878,6 +2924,13 @@ ALTER TABLE ONLY public.domain_posts_e621_aux ALTER COLUMN base_table_id SET DEF
 ALTER TABLE ONLY public.domain_posts_fa_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_fa_aux_base_table_id_seq'::regclass);


+--
+-- Name: domain_posts_ib_aux base_table_id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_posts_ib_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_ib_aux_base_table_id_seq'::regclass);
+
+
 --
 -- Name: domain_twitter_tweets id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -3083,6 +3136,14 @@ ALTER TABLE ONLY public.domain_posts_fa_aux
    ADD CONSTRAINT domain_posts_fa_aux_pkey PRIMARY KEY (base_table_id);


+--
+-- Name: domain_posts_ib_aux domain_posts_ib_aux_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_posts_ib_aux
+    ADD CONSTRAINT domain_posts_ib_aux_pkey PRIMARY KEY (base_table_id);
+
+
 --
 -- Name: domain_posts domain_posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -4040,6 +4101,20 @@ CREATE INDEX index_domain_posts_fa_aux_on_base_table_id ON public.domain_posts_f
 CREATE UNIQUE INDEX index_domain_posts_fa_aux_on_fa_id ON public.domain_posts_fa_aux USING btree (fa_id);


+--
+-- Name: index_domain_posts_ib_aux_on_base_table_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_domain_posts_ib_aux_on_base_table_id ON public.domain_posts_ib_aux USING btree (base_table_id);
+
+
+--
+-- Name: index_domain_posts_ib_aux_on_ib_id; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_domain_posts_ib_aux_on_ib_id ON public.domain_posts_ib_aux USING btree (ib_id);
+
+
 --
 -- Name: index_domain_posts_on_posted_at; Type: INDEX; Schema: public; Owner: -
 --
@@ -5111,6 +5186,14 @@ ALTER TABLE ONLY public.domain_users_inkbunny_aux
    ADD CONSTRAINT fk_rails_304ea0307f FOREIGN KEY (base_table_id) REFERENCES public.domain_users(id);


+--
+-- Name: domain_posts_ib_aux fk_rails_3762390d41; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_posts_ib_aux
+    ADD CONSTRAINT fk_rails_3762390d41 FOREIGN KEY (shallow_update_log_entry_id) REFERENCES public.http_log_entries(id);
+
+
 --
 -- Name: http_log_entries fk_rails_42f35e9da0; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -5135,6 +5218,14 @@ ALTER TABLE ONLY public.domain_user_user_follows
    ADD CONSTRAINT fk_rails_4b2ab65400 FOREIGN KEY (from_id) REFERENCES public.domain_users(id);


+--
+-- Name: domain_posts_ib_aux fk_rails_5ee2c344bd; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_posts_ib_aux
+    ADD CONSTRAINT fk_rails_5ee2c344bd FOREIGN KEY (deep_update_log_entry_id) REFERENCES public.http_log_entries(id);
+
+
 --
 -- Name: domain_twitter_medias fk_rails_5fffa41fa6; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -5239,6 +5330,14 @@ ALTER TABLE ONLY public.domain_users_e621_aux
    ADD CONSTRAINT fk_rails_b5bacbced6 FOREIGN KEY (base_table_id) REFERENCES public.domain_users(id);


+--
+-- Name: domain_posts_ib_aux fk_rails_b94b311254; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.domain_posts_ib_aux
+    ADD CONSTRAINT fk_rails_b94b311254 FOREIGN KEY (base_table_id) REFERENCES public.domain_posts(id);
+
+
 --
 -- Name: domain_posts_fa_aux fk_rails_be2be2e955; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -5350,6 +5449,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
 SET search_path TO "$user", public;

 INSERT INTO "schema_migrations" (version) VALUES
+('20250727173100'),
 ('20250726051748'),
 ('20250726051451'),
 ('20250725192431'),
--- a/spec/jobs/domain/inkbunny/job/static_file_job_spec.rb
+++ b/spec/jobs/domain/inkbunny/job/static_file_job_spec.rb
@@ -12,6 +12,15 @@ end
 describe Domain::Inkbunny::Job::StaticFileJob do
  let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
  before { Scraper::ClientFactory.http_client_mock = http_client_mock }
+  let(:creator) do
+    create(:domain_user_inkbunny_user, ib_id: 12_345, name: "TheUser")
+  end
+  let(:post) do
+    create(:domain_post_inkbunny_post, ib_id: 67_891).tap do |p|
+      p.creator = creator
+      p.save!
+    end
+  end
  let(:file) do
    Domain::PostFile::InkbunnyPostFile.create!(
      {
@@ -25,15 +34,7 @@ describe Domain::Inkbunny::Job::StaticFileJob do
        md5s: {
          initial_file_md5: FileJobSpec::AN_IMAGE_MD5,
        },
-        post:
-          Domain::Post::InkbunnyPost.create!(
-            ib_id: 67_891,
-            creator:
-              Domain::User::InkbunnyUser.create!(
-                ib_id: 12_345,
-                name: "TheUser",
-              ),
-          ),
+        post: post,
      },
    )
  end
--- a/spec/jobs/domain/inkbunny/job/update_posts_job_spec.rb
+++ b/spec/jobs/domain/inkbunny/job/update_posts_job_spec.rb
@@ -40,42 +40,54 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
    end

    let!(:post_3104202) do
-      Domain::Post::InkbunnyPost.create!(
-        ib_id: 3_104_202,
-        creator: user_thendyart,
-        title: "Phantom Touch - Page 25",
-        posted_at: Time.parse("2023-08-27 21:31:40.365597+02"),
-        last_file_updated_at: Time.parse("2023-08-27 21:30:06.222262+02"),
-        num_files: 1,
-        rating: "adult",
-        submission_type: "comic",
-      )
+      Domain::Post::InkbunnyPost
+        .create!(
+          ib_id: 3_104_202,
+          title: "Phantom Touch - Page 25",
+          posted_at: Time.parse("2023-08-27 21:31:40.365597+02"),
+          last_file_updated_at: Time.parse("2023-08-27 21:30:06.222262+02"),
+          num_files: 1,
+          rating: "adult",
+          submission_type: "comic",
+        )
+        .tap do |p|
+          p.creator = user_thendyart
+          p.save!
+        end
    end

    let!(:post_3104200) do
-      Domain::Post::InkbunnyPost.create!(
-        ib_id: 3_104_200,
-        creator: user_seff,
-        title: "Camp Pines Sketch Dump (Aug 2023)",
-        posted_at: Time.parse("2023-08-27 21:30:59.308046+02"),
-        last_file_updated_at: Time.parse("2023-08-27 21:26:14.049+02"),
-        num_files: 4,
-        rating: "adult",
-        submission_type: "picture_pinup",
-      )
+      Domain::Post::InkbunnyPost
+        .create!(
+          ib_id: 3_104_200,
+          title: "Camp Pines Sketch Dump (Aug 2023)",
+          posted_at: Time.parse("2023-08-27 21:30:59.308046+02"),
+          last_file_updated_at: Time.parse("2023-08-27 21:26:14.049+02"),
+          num_files: 4,
+          rating: "adult",
+          submission_type: "picture_pinup",
+        )
+        .tap do |p|
+          p.creator = user_seff
+          p.save!
+        end
    end

    let!(:post_3104197) do
-      Domain::Post::InkbunnyPost.create!(
-        ib_id: 3_104_197,
-        creator: user_soulcentinel,
-        title: "Comm - BJ bird",
-        posted_at: Time.parse("2023-08-27 21:29:37.995264+02"),
-        last_file_updated_at: Time.parse("2023-08-27 21:24:23.653306+02"),
-        num_files: 1,
-        rating: "adult",
-        submission_type: "picture_pinup",
-      )
+      Domain::Post::InkbunnyPost
+        .create!(
+          ib_id: 3_104_197,
+          title: "Comm - BJ bird",
+          posted_at: Time.parse("2023-08-27 21:29:37.995264+02"),
+          last_file_updated_at: Time.parse("2023-08-27 21:24:23.653306+02"),
+          num_files: 1,
+          rating: "adult",
+          submission_type: "picture_pinup",
+        )
+        .tap do |p|
+          p.creator = user_soulcentinel
+          p.save!
+        end
    end

    let(:ib_post_ids) { [3_104_202, 3_104_200, 3_104_197] }
@@ -349,16 +361,20 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
    end

    let!(:post_1047334) do
-      Domain::Post::InkbunnyPost.create!(
-        ib_id: 1_047_334,
-        creator: user_zzreg,
-        title: "New Submission",
-        posted_at: Time.parse("2016-03-13 22:18:52.32319+01"),
-        last_file_updated_at: Time.parse("2016-03-13 22:18:52.32319+01"),
-        num_files: 1,
-        rating: "general",
-        submission_type: "picture_pinup",
-      )
+      Domain::Post::InkbunnyPost
+        .create!(
+          ib_id: 1_047_334,
+          title: "New Submission",
+          posted_at: Time.parse("2016-03-13 22:18:52.32319+01"),
+          last_file_updated_at: Time.parse("2016-03-13 22:18:52.32319+01"),
+          num_files: 1,
+          rating: "general",
+          submission_type: "picture_pinup",
+        )
+        .tap do |p|
+          p.creator = user_zzreg
+          p.save!
+        end
    end

    it "updates post with new file information" do
@@ -440,9 +456,11 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
      create(
        :domain_post_inkbunny_post,
        ib_id: 2_637_105,
-        creator: user_friar,
        num_files: 5,
-      )
+      ).tap do |p|
+        p.creator = user_friar
+        p.save!
+      end
    end

    it "handles files with null MD5 sums correctly" do