Blob entry migration util

2023-05-18 20:12:10 -07:00
parent 67c28cb8d2
commit 9423a50bc3
7 changed files with 946 additions and 7 deletions
--- a/11
+++ b/11
@@ -37,10 +37,17 @@ end
 namespace :blob_entries do
  task :export_samples => :environment do
    limit = ENV["limit"]&.to_i || raise("need 'limit' (num)")
-    out = ENV["out"] || raise("need 'out' (file path, .json encoded)")
-    BlobEntrySampleExporter.new.export_samples(limit, out)
+    outfile = ENV["outfile"] || raise("need 'outfile' (file path, .json encoded)")
+    BlobEntrySampleExporter.new.export_samples(limit, outfile)
  end
  task :import_samples => :environment do
+    infile = ENV["infile"] || raise("need 'infile' (file path, .json encoded)")
+    BlobEntrySampleExporter.new.import_samples(infile)
+  end
+  task :migrate_entries => :environment do
+    start_at = ENV["start_at"]
+    batch_size = ENV["batch_size"]&.to_i || 64
+    BlobEntrySampleExporter.new.migrate_blob_entries(start_at, batch_size)
  end
 end

--- a/app/lib/blob_entry_sample_exporter.rb
+++ b/app/lib/blob_entry_sample_exporter.rb
@@ -17,16 +17,85 @@ class BlobEntrySampleExporter
    end
  end

+  def import_samples(file)
+    @num_read = 0
+    measure(proc {
+      "read #{@num_read} blob entries from #{file}"
+    }) do
+      File.open(file, "r") do |file|
+        while (line = file.readline) != nil
+          be = read_blob_entry(
+            line.chomp.strip,
+            file.readline.chomp.strip
+          )
+
+          if be.base_sha256.present?
+            base = read_blob_entry(
+              file.readline.chomp.strip,
+              file.readline.chomp.strip
+            )
+            base.save! unless base.persisted?
+          end
+          be.save! unless be.persisted?
+        end
+      end
+    end
+  end
+
+  def migrate_blob_entries(start_at, batch_size)
+    offset = start_at && HexUtil.hex2bin(start_at)
+    keep_going = true
+    total_imported = 0
+    measure(proc {
+      "imported #{total_imported} blob entry models"
+    }) do
+      while keep_going
+        missing = []
+        measure(proc {
+          "migrated #{missing.size} blob entries, #{total_imported} total, " +
+          "last offset #{HexUtil.bin2hex(offset)}"
+        }) do
+          be_sha256s = if offset
+            BlobEntry.where("sha256 > E'\\\\x#{HexUtil.bin2hex(offset)}'")
+          else
+            BlobEntry
+          end.order(sha256: :asc).limit(batch_size).pluck(:sha256)
+          if be_sha256s.empty?
+            keep_going = false
+            break
+          end
+          offset = be_sha256s.last
+
+          bep_sha256s = BlobEntryP.where(sha256: be_sha256s).pluck(:sha256)
+          missing = be_sha256s - bep_sha256s
+          next if missing.empty?
+          total_imported += missing.size
+
+          missing_formatted = missing.map do |sha256|
+            "E'\\\\x#{HexUtil.bin2hex(sha256)}'"
+          end.join(", ")
+          ReduxApplicationRecord.connection.execute <<-SQL
+        INSERT INTO blob_entries_p
+        (
+            SELECT sha256, base_sha256, content_type, size, contents, created_at
+            FROM blob_entries
+            WHERE sha256 IN (#{missing_formatted})
+        ) RETURNING sha256
+        SQL
+        end
+      end
+    end
+  end
+
  def write_blob_entry(file, blob_entry)
    hash = blob_entry.to_bulk_insert_hash
-    json_hash = {
+    file.puts({
      sha256: HexUtil.bin2hex(hash[:sha256]),
+      base_sha256: hash[:base_sha256] ? HexUtil.bin2hex(hash[:base_sha256]) : nil,
      content_type: hash[:content_type],
      size: hash[:size],
      created_at: blob_entry.created_at,
-    }
-    json_hash[:base_sha256] = HexUtil.bin2hex(hash[:base_sha256]) if hash[:base_sha256]
-    file.puts(json_hash.to_json)
+    }.to_json)
    file.puts(Base64.strict_encode64(blob_entry.read_attribute(:contents)))
    logger.info(
      "#{HexUtil.bin2hex(blob_entry.sha256)} - " +
@@ -37,4 +106,24 @@ class BlobEntrySampleExporter
    @bytes_written += blob_entry.bytes_stored
    @num_written += 1
  end
+
+  def read_blob_entry(line1, line2)
+    hash = JSON.parse(line1)
+    sha256 = HexUtil.hex2bin(hash["sha256"])
+    be = BlobEntry.find_by(sha256: sha256)
+    return be if be
+
+    contents = Base64.strict_decode64(line2)
+    be = BlobEntry.find_by(sha256: sha256) || BlobEntry.new({
+      sha256: sha256,
+      base_sha256: hash["base_sha256"] ? HexUtil.hex2bin(hash["base_sha256"]) : nil,
+      created_at: Time.parse(hash["created_at"]),
+      contents: contents,
+      size: hash["size"],
+      content_type: hash["content_type"],
+    })
+    logger.info("#{hash["sha256"]} - #{hash["content_type"]} - #{HexUtil.humansize(hash["size"])}")
+    @num_read += 1
+    be
+  end
 end
--- a/app/models/blob_entry.rb
+++ b/app/models/blob_entry.rb
@@ -19,6 +19,10 @@ class BlobEntry < ReduxApplicationRecord
  validates :sha256, length: { is: 32 }
  validates :base_sha256, length: { is: 32 }, if: :base_sha256

+  after_create do
+    BlobEntryP.create!(to_bulk_insert_hash.merge(created_at: created_at))
+  end
+
  def to_bulk_insert_hash
    {
      sha256: self.read_attribute(:sha256),
--- a/app/models/blob_entry_p.rb
+++ b/app/models/blob_entry_p.rb
@@ -0,0 +1,143 @@
+class BlobEntryP < ReduxApplicationRecord
+  self.table_name = "blob_entries_p"
+
+  include ImmutableModel
+  before_destroy { raise ActiveRecord::ReadOnlyRecord }
+
+  self.primary_key = :sha256
+  EMPTY_FILE_SHA256 = HexUtil.hex2bin("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+
+  belongs_to :base,
+    optional: true,
+    foreign_key: :base_sha256,
+    class_name: "::BlobEntryP"
+
+  validates_presence_of(
+    :sha256,
+    :content_type,
+    :size
+  )
+  validates :contents, length: { minimum: 0, allow_nil: false, message: "can't be nil" }
+  validates :sha256, length: { is: 32 }
+  validates :base_sha256, length: { is: 32 }, if: :base_sha256
+
+  def contents
+    @contents ||= begin
+        contents_raw = self.read_attribute(:contents)
+        if self.base
+          XDiff.patch(self.base.contents, contents_raw)
+        else
+          contents_raw
+        end
+      end
+  end
+
+  def bytes_stored
+    self.read_attribute(:contents).size
+  end
+
+  def self.find_or_build_from_legacy(legacy_be)
+    file_path = legacy_be.file_path
+    file_name = File.basename file_path
+
+    if file_name.length == 64
+      file_sha256_assumed = HexUtil.hex2bin(file_name)
+      # try to find existing file before going through all this hassle
+      entry = BlobEntry.find_by(sha256: file_sha256_assumed)
+      return entry if entry
+    end
+
+    return nil unless File.exist?(file_path)
+
+    # macos / linux slightly differ in their file type handling
+    if RUBY_PLATFORM =~ /darwin/
+      file_mime_flags = "-Ib"
+    elsif RUBY_PLATFORM =~ /linux/
+      file_mime_flags = "-ib"
+    else
+      raise("unknown platform #{RUBY_PLATFORM}")
+    end
+
+    file_contents = IO.binread(file_path)
+    file_mime = `file #{file_mime_flags} #{file_path}`
+    raise("error running `file` on #{file_path}: #{file_mime}") if $?.exitstatus != 0
+    file_mime.chomp!
+    record = find_or_build(content_type: file_mime, contents: file_contents)
+
+    # guess the name is a sha256 hash
+    if file_name.length == 64
+      if record.sha256 != file_sha256_assumed
+        # checksum for an empty file
+        if record.sha256 == EMPTY_FILE_SHA256
+          return nil
+        else
+          raise("checksum mismatch for #{file_path}: #{HexUtil.bin2hex(record.sha256)} != #{file_name}")
+        end
+      end
+    end
+
+    # puts ("[blob entry] built #{file_mime} (#{HexUtil.humansize(record.size)})")
+    record.created_at = legacy_be.created_at
+    record.updated_at = legacy_be.updated_at
+    record
+  end
+
+  def self.find_or_build(content_type:, contents:, candidates: [])
+    sha256 = Digest::SHA256.digest(contents)
+    BlobEntry.find_by(sha256: sha256) || begin
+      build_record(
+        content_type: content_type,
+        sha256: sha256,
+        contents: contents,
+        candidates: candidates,
+      )
+    end
+  end
+
+  DIFFABLE_CONTENT_TYPES = [
+    /text\/html/,
+    /text\/plain/,
+    /application\/json/,
+  ]
+
+  def self.build_record(content_type:, sha256:, contents:, candidates: [])
+    record = BlobEntry.new(sha256: sha256, content_type: content_type, size: contents.size)
+
+    smallest_patch_size = nil
+    smallest_patch = nil
+    smallest_candidate = nil
+
+    candidates.map do |candidate|
+      # only consider candidates with the same content type (may relax this later)
+      next nil if candidate.content_type != content_type
+      # only consider candidates who themselves aren't patch-based
+      next nil unless candidate.base.nil?
+      # only consider diffable content types
+      next nil unless DIFFABLE_CONTENT_TYPES.any? { |ct| content_type =~ ct }
+
+      [candidate, XDiff.diff(candidate.contents, contents)]
+    end.reject(&:nil?).each do |pair|
+      candidate, patch = pair
+      if smallest_patch_size.nil? || patch.size < smallest_patch_size
+        smallest_patch_size = patch.size
+        smallest_patch = patch
+        smallest_candidate = candidate
+      end
+    end
+
+    # only use a patch if it's <= 60% the original content size
+    if smallest_patch_size && smallest_patch_size <= (contents.size * 0.6)
+      record.base = smallest_candidate
+      record.contents = smallest_patch
+    else
+      # no candidate present, store the whole contents directly in the record
+      record.contents = contents
+    end
+
+    if record.contents != contents
+      raise RuntimeError.new("invariant!")
+    end
+
+    record
+  end
+end
--- a/db/redux_migrate/20230519002300_create_partitioned_blob_entries.rb
+++ b/db/redux_migrate/20230519002300_create_partitioned_blob_entries.rb
@@ -0,0 +1,37 @@
+class CreatePartitionedBlobEntries < ActiveRecord::Migration[7.0]
+  NUM_PARTITIONS = 64
+
+  def up
+    main_table_sql = <<~SQL.split("\n").map(&:strip).join(" ")
+      CREATE TABLE blob_entries_p (
+        sha256 bytea NOT NULL,
+        base_sha256 bytea,
+        content_type character varying NOT NULL,
+        size integer NOT NULL,
+        contents bytea NOT NULL,
+        created_at timestamp(6) without time zone NOT NULL
+      ) PARTITION BY HASH (sha256)
+    SQL
+    execute main_table_sql
+
+    NUM_PARTITIONS.times do |partnum|
+      partition_table_name = :"blob_entries_p_#{partnum.to_s.rjust(2, "0")}"
+      partition_table_sql = <<~SQL.split("\n").map(&:strip).join(" ")
+        CREATE TABLE #{partition_table_name}
+        PARTITION OF blob_entries_p FOR 
+        VALUES WITH (
+            MODULUS #{NUM_PARTITIONS}, 
+            REMAINDER #{partnum}
+          )
+      SQL
+      execute partition_table_sql
+      add_index partition_table_name, :sha256, unique: true
+    end
+
+    add_index :blob_entries_p, :sha256, unique: true
+  end
+
+  def down
+    drop_table :blob_entries_p
+  end
+end
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.

-ActiveRecord::Schema[7.0].define(version: 2023_05_03_042308) do
+ActiveRecord::Schema[7.0].define(version: 2023_05_19_002300) do
  # These are extensions that must be enabled in order to support this database
  enable_extension "pg_stat_statements"
  enable_extension "pg_trgm"
@@ -29,6 +29,656 @@ ActiveRecord::Schema[7.0].define(version: 2023_05_03_042308) do
    t.index ["sha256"], name: "index_blob_entries_on_sha256", unique: true
  end

+  create_table "blob_entries_p", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_00", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_00_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_01", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_01_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_02", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_02_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_03", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_03_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_04", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_04_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_05", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_05_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_06", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_06_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_07", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_07_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_08", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_08_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_09", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_09_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_10", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_10_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_11", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_11_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_12", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_12_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_13", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_13_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_14", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_14_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_15", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_15_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_16", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_16_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_17", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_17_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_18", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_18_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_19", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_19_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_20", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_20_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_21", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_21_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_22", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_22_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_23", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_23_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_24", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_24_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_25", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_25_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_26", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_26_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_27", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_27_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_28", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_28_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_29", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_29_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_30", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_30_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_31", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_31_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_32", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_32_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_33", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_33_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_34", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_34_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_35", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_35_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_36", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_36_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_37", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_37_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_38", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_38_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_39", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_39_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_40", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_40_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_41", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_41_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_42", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_42_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_43", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_43_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_44", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_44_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_45", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_45_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_46", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_46_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_47", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_47_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_48", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_48_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_49", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_49_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_50", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_50_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_51", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_51_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_52", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_52_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_53", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_53_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_54", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_54_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_55", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_55_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_56", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_56_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_57", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_57_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_58", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_58_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_59", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_59_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_60", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_60_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_61", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_61_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_62", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_62_on_sha256", unique: true
+  end
+
+  create_table "blob_entries_p_63", id: false, force: :cascade do |t|
+    t.binary "sha256", null: false
+    t.binary "base_sha256"
+    t.string "content_type", null: false
+    t.integer "size", null: false
+    t.binary "contents", null: false
+    t.datetime "created_at", null: false
+    t.index ["sha256"], name: "index_blob_entries_p_63_on_sha256", unique: true
+  end
+
  create_table "delayed_jobs", force: :cascade do |t|
    t.integer "priority", default: 0, null: false
    t.integer "attempts", default: 0, null: false
--- a/test/models/blob_entry_test.rb
+++ b/test/models/blob_entry_test.rb
@@ -39,4 +39,13 @@ class BlobEntryTest < ActiveSupport::TestCase
      model.destroy
    end
  end
+
+  test "model dual-writes a BlobEntryP model" do
+    model = TestUtil.build_blob_entry
+    model.save!
+    model_p = BlobEntryP.find_by(sha256: model.sha256)
+    [:sha256, :base_sha256, :contents, :size, :created_at, :content_type].each do |attr|
+      assert_equal model.send(attr), model_p.send(attr), "#{attr} mismatch"
+    end
+  end
 end