add separate lite trail versions table support

app/lib/domain/fa/parser/submission_parser_helper.rb

@@ -243,7 +243,6 @@ class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
   end
 
   def info_text_value_redux(info_section)
-    # binding.pry
     info_text_elem_redux.
       css(".highlight").
       find { |e| e.text == info_section }.

app/lib/lite_trail.rb

@@ -0,0 +1,2 @@
+module LiteTrail
+end

app/lib/lite_trail/active_record_class_methods.rb

@@ -0,0 +1,120 @@
+module LiteTrail::ActiveRecordClassMethods
+  def has_lite_trail(
+    schema_version:,
+    map_attribute: nil,
+    separate_versions_table: false
+  )
+    self_class = self
+
+    versions_table_name = if separate_versions_table.is_a?(String)
+        separate_versions_table
+      elsif separate_versions_table == true
+        self.table_name.singularize + "_versions"
+      else
+        nil
+      end
+
+    lite_trail_class = if versions_table_name.nil?
+        ::LiteTrail::Version
+      else
+        # separate table defined, use that, no need for polymorphism
+        klass = Class.new(::LiteTrail::AbstractVersion) do
+          self.table_name = versions_table_name
+          belongs_to :item, class_name: self_class.to_s
+        end
+
+        # "foo_bar_versions" => define  "LiteTrail::PerTable::FooBarVersions"
+        LiteTrail::PerTable.const_set(versions_table_name.camelize, klass)
+      end
+
+    class_attribute :lite_trail_class
+    self.lite_trail_class = lite_trail_class
+
+    class_attribute :lite_trail_options
+    self.lite_trail_options = {
+      schema_version: schema_version,
+      map_attribute: map_attribute,
+    }
+
+    if separate_versions_table.nil?
+      # using the polymorphic versions table
+      has_many :versions,
+               -> { order(created_at: :asc) },
+               class_name: lite_trail_class.name,
+               as: :item,
+               autosave: false
+    else
+      has_many :versions,
+               -> { order(created_at: :asc) },
+               class_name: lite_trail_class.name,
+               inverse_of: :item,
+               foreign_key: :item_id,
+               autosave: false
+    end
+
+    after_create do
+      if self.respond_to?(:created_at)
+        model_created_at = self.created_at
+      else
+        model_created_at = Time.now
+      end
+
+      self.versions << lite_trail_class.new({
+        event: "create",
+        item: self,
+        schema_version: schema_version,
+        created_at: model_created_at,
+      })
+    end
+
+    after_update do
+      changes = self.saved_changes
+      if changes.any?
+        changes = changes.dup if map_attribute&.any?
+
+        map_attribute.each do |attr_name, mapper|
+          if changes[attr_name]
+            # value before the update
+            changes[attr_name][0] = mapper.map_to(changes[attr_name][0]) if changes[attr_name][0]
+            # value after the update
+            changes[attr_name][1] = mapper.map_to(changes[attr_name][1]) if changes[attr_name][1]
+          end
+        end if map_attribute
+
+        if self.respond_to?(:updated_at)
+          model_updated_at = self.updated_at
+        else
+          model_updated_at = Time.now
+        end
+
+        self.versions << lite_trail_class.new({
+          event: "update",
+          item: self,
+          schema_version: schema_version,
+          diff: changes,
+          created_at: model_updated_at,
+        })
+      end
+    end
+
+    after_save do
+      self.versions.filter(&:new_record?).each(&:save!)
+    end
+
+    after_destroy do
+      attributes = self.attributes
+      map_attribute.each do |attr_name, mapper|
+        if attributes[attr_name]
+          attributes[attr_name] = mapper.map_to(attributes[attr_name])
+        end
+      end
+
+      self.versions << lite_trail_class.create!({
+        event: "destroy",
+        item: self,
+        schema_version: schema_version,
+        diff: attributes,
+      })
+    end
+  end
+end

app/lib/lite_trail/migration_extensions.rb

@@ -0,0 +1,16 @@
+require "active_support"
+
+module LiteTrail::MigrationExtensions
+  def create_versions_table(table_name)
+    versions_table_name = "#{table_name.to_s.singularize}_versions"
+    create_table versions_table_name do |t|
+      t.references :item
+      t.integer :schema_version
+      t.string :event, null: false
+      t.jsonb :diff
+      t.datetime :created_at, null: false
+    end
+
+    add_foreign_key versions_table_name, table_name, column: :item_id, validate: true
+  end
+end

app/lib/lite_trail/per_table.rb

@@ -0,0 +1,2 @@
+module LiteTrail::PerTable
+end

app/lib/sha256_attribute_mapper.rb

@@ -0,0 +1,8 @@
+class Sha256AttributeMapper
+  def self.map_to(attr_value)
+    HexUtil.bin2hex(attr_value)
+  end
+  def self.map_from(stored_value)
+    HexUtil.hex2bin(attr_value)
+  end
+end

app/models/concerns/immutable_model.rb

@@ -1,23 +1,8 @@
 module ImmutableModel
-  class UpdateImmutableException < Exception
-    def initialize
-      super("Immutable model can't be updated")
-    end
-  end
-
   extend ActiveSupport::Concern
-
   included do
-    # before_update :prevent_update
-
     def readonly?
       !new_record?
     end
   end
-
-  private
-
-  def prevent_update
-    raise UpdateImmutableException.new
-  end
 end

app/models/domain/e621/post.rb

@@ -1,6 +1,17 @@
 class Domain::E621::Post < ReduxApplicationRecord
   self.table_name = "domain_e621_posts"
-  has_lite_trail(schema_version: 1)
+  has_lite_trail(schema_version: 1, separate_versions_table: true)
+
+  # see state_detail for scan_error/file_error
+  enum state: [:ok, :scan_error, :file_error]
+
+  validates_presence_of(
+    :e621_id,
+    :state,
+  )
+  after_initialize do
+    self.state ||= :ok
+  end
 
   has_many :taggings,
     class_name: "Domain::E621::Tagging"

app/models/domain/fa/post.rb

@@ -4,7 +4,7 @@ class Domain::Fa::Post < ReduxApplicationRecord
   has_lite_trail(
     schema_version: 1,
     map_attribute: {
-      file_sha256: ::SHA256AttributeMapper,
+      file_sha256: ::Sha256AttributeMapper,
     },
   )
 

app/models/http_log_entry.rb

@@ -73,8 +73,6 @@ class HttpLogEntry < ReduxApplicationRecord
       })
       return record
     end
-
-    binding.pry
   end
 
   def uri=(uri)

app/models/lite_trail/abstract_version.rb

@@ -0,0 +1,58 @@
+class LiteTrail::AbstractVersion < ReduxApplicationRecord
+  self.abstract_class = true
+  include ImmutableModel
+
+  def reify
+    versions_arr = item.versions
+    self_idx = versions_arr.find_index(self)
+    if self_idx == nil
+      raise("item.versions (#{item.item_type}/#{item.item_id}) does not contain self: #{self.id}")
+    end
+
+    model = if self.event == "destroy"
+        self.item.class.new
+      else
+        self.item.dup
+      end
+
+    # unapply versions in reverse order
+    (versions_arr.length - 1).downto(self_idx).each do |idx|
+      version = versions_arr[idx]
+      version._unapply(model)
+    end
+
+    model
+  end
+
+  def _unapply(model)
+    mapper_config = item.class.lite_trail_options[:map_attribute] || {}
+
+    if self.event == "create"
+      raise("'create' cannot be undone")
+    elsif self.event == "update"
+      self.diff.each do |attr_name, change|
+        attr_before, attr_after = change
+
+        attr_name_sym = attr_name.to_sym
+        if mapper_config[attr_name_sym]
+          attr_before = mapper_config[attr_name_sym].map_from(attr_before) if attr_before
+          attr_after = mapper_config[attr_name_sym].map_from(attr_after) if attr_after
+        end
+
+        # sanity check - but ignore updated_at due to rounding issues
+        if model.send(attr_name.to_sym) != attr_after
+          raise("expected #{attr_name} to be #{attr_after}, was #{item_attributes[attr_name]}")
+        end if attr_name_sym != :updated_at
+
+        model.send(:"#{attr_name}=", attr_before)
+      end
+    elsif self.event == "destroy"
+      self.diff.each do |attr_name, attr_value|
+        if mapper_config[attr_name]
+          attr_value = mapper_config[attr_name].map_from(attr_value)
+        end
+        item.send(:"#{attr_name}=", attr_value)
+      end
+    end
+  end
+end

app/models/lite_trail/version.rb

@@ -1,57 +1,4 @@
-class LiteTrail::Version < ReduxApplicationRecord
+class LiteTrail::Version < LiteTrail::AbstractVersion
   self.table_name = "versions"
   belongs_to :item, polymorphic: true
-
-  def reify
-    versions_arr = item.versions.to_a
-    this_idx = versions_arr.find_index(self)
-    if this_idx == nil
-      raise("item.versions (#{item.item_type}/#{item.item_id}) does not contain self: #{self.id}")
-    end
-    unapply_versions = versions_arr[this_idx..].reverse
-
-    model = if self.event == "destroy"
-        self.item.class.new
-      else
-        self.item.dup
-      end
-
-    unapply_versions.each do |version|
-      version.unapply(model)
-    end
-
-    model
-  end
-
-  def unapply(model)
-    mapper_config = item.class.lite_trail_options[:map_attribute] || {}
-
-    if self.event == "create"
-      raise("'create' cannot be undone")
-    elsif self.event == "update"
-      self.diff.each do |attr_name, change|
-        attr_before, attr_after = change
-
-        attr_name_sym = attr_name.to_sym
-        if mapper_config[attr_name_sym]
-          attr_before = mapper_config[attr_name_sym].map_from(attr_before) if attr_before
-          attr_after = mapper_config[attr_name_sym].map_from(attr_after) if attr_after
-        end
-
-        # sanity check - but ignore updated_at due to rounding issues
-        if model.send(attr_name.to_sym) != attr_after
-          raise("expected #{attr_name} to be #{attr_after}, was #{item_attributes[attr_name]}")
-        end if attr_name_sym != :updated_at
-
-        model.send(:"#{attr_name}=", attr_before)
-      end
-    elsif self.event == "destroy"
-      self.diff.each do |attr_name, attr_value|
-        if mapper_config[attr_name]
-          attr_value = mapper_config[attr_name].map_from(attr_value)
-        end
-        item.send(:"#{attr_name}=", attr_value)
-      end
-    end
-  end
 end

app/models/redux_application_record.rb

@@ -1,69 +1,4 @@
 class ReduxApplicationRecord < ActiveRecord::Base
-  # self.primary_abstract_class = true
   self.abstract_class = true
   connects_to database: { writing: :redux, reading: :redux }
-
-  def self.has_lite_trail(schema_version:, map_attribute: nil)
-    class_attribute :lite_trail_options
-    self.lite_trail_options = {
-      schema_version: schema_version,
-      map_attribute: map_attribute,
-    }
-
-    has_many :versions,
-      -> { order(created_at: :asc) },
-      class_name: "::LiteTrail::Version",
-      as: :item
-
-    after_create do
-      ::LiteTrail::Version.create({
-        event: "create",
-        item: self,
-        schema_version: schema_version,
-      })
-    end
-
-    after_update do
-      changes = self.saved_changes
-      if changes.any?
-        map_attribute.each do |attr_name, mapper|
-          if changes[attr_name]
-            changes[attr_name][0] = mapper.map_to(changes[attr_name][0]) if changes[attr_name][0]
-            changes[attr_name][1] = mapper.map_to(changes[attr_name][1]) if changes[attr_name][1]
-          end
-        end if map_attribute
-        ::LiteTrail::Version.create({
-          event: "update",
-          item: self,
-          schema_version: schema_version,
-          diff: changes,
-        })
-      end
-    end
-
-    after_destroy do
-      attributes = self.attributes
-      map_attribute.each do |attr_name, mapper|
-        if attributes[attr_name]
-          attributes[attr_name] = mapper.map_to(attributes[attr_name])
-        end
-      end
-
-      ::LiteTrail::Version.create({
-        event: "destroy",
-        item: self,
-        schema_version: schema_version,
-        diff: attributes,
-      })
-    end
-  end
-end
-
-class SHA256AttributeMapper
-  def self.map_to(attr_value)
-    HexUtil.bin2hex(attr_value)
-  end
-  def self.map_from(stored_value)
-    HexUtil.hex2bin(attr_value)
-  end
 end

config/initializers/lite_trail_extensions_initializer.rb

@@ -0,0 +1,7 @@
+require_relative Rails.root.join("app/lib/lite_trail")
+require_relative Rails.root.join("app/lib/lite_trail/migration_extensions")
+ActiveRecord::Migration.send(:include, ::LiteTrail::MigrationExtensions)
+
+require_relative Rails.root.join("app/models/redux_application_record")
+require_relative Rails.root.join("app/lib/lite_trail/active_record_class_methods")
+ReduxApplicationRecord.send(:extend, ::LiteTrail::ActiveRecordClassMethods)

db/redux_migrate/20230226223215_create_domain_e621_posts.rb

@@ -26,6 +26,8 @@ class CreateDomainE621Posts < ActiveRecord::Migration[7.0]
 
       t.index :e621_id, unique: :true
     end
+
+    create_versions_table :domain_e621_posts
   end
 end
 

db/schema.rb

@@ -13,6 +13,7 @@
 ActiveRecord::Schema[7.0].define(version: 2023_03_01_013456) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
+  enable_extension "pg_trgm"
   enable_extension "plpgsql"
 
   create_table "blob_entries", id: false, force: :cascade do |t|
@@ -45,6 +46,15 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_01_013456) do
     t.index ["signature"], name: "delayed_jobs_signature_idx", unique: true
   end
 
+  create_table "domain_e621_post_versions", force: :cascade do |t|
+    t.bigint "item_id"
+    t.integer "schema_version"
+    t.string "event", null: false
+    t.jsonb "diff"
+    t.datetime "created_at"
+    t.index ["item_id"], name: "index_domain_e621_post_versions_on_item_id"
+  end
+
   create_table "domain_e621_posts", force: :cascade do |t|
     t.integer "e621_id", null: false
     t.integer "state", null: false
@@ -53,6 +63,8 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_01_013456) do
     t.string "description"
     t.integer "rating"
     t.integer "score"
+    t.integer "up_score"
+    t.integer "down_score"
     t.integer "status"
     t.integer "favorites"
     t.integer "file_width"
@@ -60,10 +72,12 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_01_013456) do
     t.jsonb "sources_array"
     t.jsonb "tags_array"
     t.bigint "file_id"
+    t.bigint "parent_e621_id"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["e621_id"], name: "index_domain_e621_posts_on_e621_id", unique: true
     t.index ["file_id"], name: "index_domain_e621_posts_on_file_id"
+    t.index ["parent_e621_id"], name: "index_domain_e621_posts_on_parent_e621_id"
   end
 
   create_table "domain_e621_taggings", force: :cascade do |t|
@@ -164,6 +178,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_01_013456) do
     t.index ["request_headers_id"], name: "index_http_log_entries_on_request_headers_id"
     t.index ["response_headers_id"], name: "index_http_log_entries_on_response_headers_id"
     t.index ["response_sha256"], name: "index_http_log_entries_on_response_sha256", using: :hash
+    t.index ["uri_host", "uri_path", "uri_query"], name: "index_http_log_entries_on_uri_host_path_query"
   end
 
   create_table "http_log_entry_headers", force: :cascade do |t|
@@ -193,6 +208,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_01_013456) do
   end
 
   add_foreign_key "blob_entries", "blob_entries", column: "base_sha256", primary_key: "sha256"
+  add_foreign_key "domain_e621_post_versions", "domain_e621_posts", column: "item_id"
   add_foreign_key "domain_fa_posts", "domain_fa_users", column: "creator_id"
   add_foreign_key "domain_fa_posts", "http_log_entries", column: "file_id"
   add_foreign_key "http_log_entries", "blob_entries", column: "response_sha256", primary_key: "sha256"

test/lib/domain/fa/scraper/http_client_test.rb

@@ -6,6 +6,7 @@ class Domain::Fa::Scraper::HttpClientTest < ActiveSupport::TestCase
       response_code: 200,
       response_body: "a plain text body",
     ))
+    client.logger.level = :error
     response = client.get("https://www.furaffinity.net/")
 
     assert_equal 200, response.status_code

test/lib/scraper/base_http_client_test.rb

@@ -25,6 +25,7 @@ class Scraper::BaseHttpClientTest < ActiveSupport::TestCase
       response_headers: { "content-type" => "text/plain" },
       response_body: "the response " + TestUtil.random_string(16),
     ))
+    client.logger.level = :error
 
     # note the lack of trailing slash - http client should set path to '/'
     response = client.get("https://www.example.com")

test/models/domain/e621/post_test.rb

@@ -0,0 +1,46 @@
+class Domain::E621::PostTest < ActiveSupport::TestCase
+  Subject = Domain::E621::Post
+
+  test "lite trail versions works" do
+    # the generated model should exist
+    refute_nil LiteTrail::PerTable::DomainE621PostVersions
+
+    post = Subject.new({
+      e621_id: 12345,
+    })
+    assert post.valid?, post.errors.full_messages
+    assert post.save
+    assert_equal 1, post.versions.length
+
+    check_create = proc {
+      created_at_version = post.versions.last
+      assert_equal "create", created_at_version.event
+      assert_equal post.created_at, created_at_version.created_at
+    }
+
+    check_create.call
+    post.reload
+    check_create.call
+
+    check_update = proc {
+      updated_rating_version = post.versions.last
+      assert_equal "update", updated_rating_version.event
+      assert_equal post.updated_at, updated_rating_version.created_at
+    }
+
+    post.rating = 10
+
+    assert post.save
+    check_update.call
+    post.reload
+    check_update.call
+
+    # should not be able to violate FK constraint on versions table
+    # in the future this may be relaxed with a cascade deletion
+    # maybe implement soft-deletion for rows?
+    assert_raises ActiveRecord::InvalidForeignKey do
+      post.destroy
+    end
+    assert post.persisted?
+  end
+end