dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove legacy blob entry stuff, only write to BlobEntryP

Browse Source
This commit is contained in:
Dylan Knutson
2023-05-19 16:51:27 -07:00
parent 18a29fb57c
commit be5165f66d
17 changed files with 119 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/lib/scraper/gallery_dl_client.rb
View File

@@ -127,7 +127,7 @@ class Scraper::GalleryDlClient
retries = 0
begin
response_blob_entry = BlobEntry.find_or_build(
response_blob_entry = BlobEntryP.find_or_build(
content_type: content_type,
contents: http_event.body,
candidates: candidates,

2
app/lib/scraper/http_client.rb
View File

@@ -93,7 +93,7 @@ class Scraper::HttpClient
retries = 0
begin
response_blob_entry = BlobEntry.find_or_build(
response_blob_entry = BlobEntryP.find_or_build(
content_type: content_type,
contents: response_body,
candidates: candidates,

122
app/models/blob_entry.rb
View File

@@ -1,6 +1,7 @@
class BlobEntry < ReduxApplicationRecord
include ImmutableModel
before_destroy { raise ActiveRecord::ReadOnlyRecord }
before_create { raise ActiveRecord::ReadOnlyRecord }
self.primary_key = :sha256
EMPTY_FILE_SHA256 = HexUtil.hex2bin("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
@@ -24,13 +25,15 @@ class BlobEntry < ReduxApplicationRecord
end
def to_bulk_insert_hash
{
sha256: self.read_attribute(:sha256),
content_type: self.read_attribute(:content_type),
contents: self.read_attribute(:contents),
size: self.read_attribute(:size),
base_sha256: self.read_attribute(:base_sha256),
}
[
:sha256,
:content_type,
:contents,
:size,
:base_sha256,
].each do |attr|
[attr, self.read_attribute(attr)]
end.to_h
end
def contents
@@ -47,109 +50,4 @@ class BlobEntry < ReduxApplicationRecord
def bytes_stored
self.read_attribute(:contents).size
end
def self.find_or_build_from_legacy(legacy_be)
file_path = legacy_be.file_path
file_name = File.basename file_path
if file_name.length == 64
file_sha256_assumed = HexUtil.hex2bin(file_name)
# try to find existing file before going through all this hassle
entry = BlobEntry.find_by(sha256: file_sha256_assumed)
return entry if entry
end
return nil unless File.exist?(file_path)
# macos / linux slightly differ in their file type handling
if RUBY_PLATFORM =~ /darwin/
file_mime_flags = "-Ib"
elsif RUBY_PLATFORM =~ /linux/
file_mime_flags = "-ib"
else
raise("unknown platform #{RUBY_PLATFORM}")
end
file_contents = IO.binread(file_path)
file_mime = `file #{file_mime_flags} #{file_path}`
raise("error running `file` on #{file_path}: #{file_mime}") if $?.exitstatus != 0
file_mime.chomp!
record = find_or_build(content_type: file_mime, contents: file_contents)
# guess the name is a sha256 hash
if file_name.length == 64
if record.sha256 != file_sha256_assumed
# checksum for an empty file
if record.sha256 == EMPTY_FILE_SHA256
return nil
else
raise("checksum mismatch for #{file_path}: #{HexUtil.bin2hex(record.sha256)} != #{file_name}")
end
end
end
# puts ("[blob entry] built #{file_mime} (#{HexUtil.humansize(record.size)})")
record.created_at = legacy_be.created_at
record.updated_at = legacy_be.updated_at
record
end
def self.find_or_build(content_type:, contents:, candidates: [])
sha256 = Digest::SHA256.digest(contents)
BlobEntry.find_by(sha256: sha256) || begin
build_record(
content_type: content_type,
sha256: sha256,
contents: contents,
candidates: candidates,
)
end
end
DIFFABLE_CONTENT_TYPES = [
/text\/html/,
/text\/plain/,
/application\/json/,
]
def self.build_record(content_type:, sha256:, contents:, candidates: [])
record = BlobEntry.new(sha256: sha256, content_type: content_type, size: contents.size)
smallest_patch_size = nil
smallest_patch = nil
smallest_candidate = nil
candidates.map do |candidate|
# only consider candidates with the same content type (may relax this later)
next nil if candidate.content_type != content_type
# only consider candidates who themselves aren't patch-based
next nil unless candidate.base.nil?
# only consider diffable content types
next nil unless DIFFABLE_CONTENT_TYPES.any? { |ct| content_type =~ ct }
[candidate, XDiff.diff(candidate.contents, contents)]
end.reject(&:nil?).each do |pair|
candidate, patch = pair
if smallest_patch_size.nil? || patch.size < smallest_patch_size
smallest_patch_size = patch.size
smallest_patch = patch
smallest_candidate = candidate
end
end
# only use a patch if it's <= 60% the original content size
if smallest_patch_size && smallest_patch_size <= (contents.size * 0.6)
record.base = smallest_candidate
record.contents = smallest_patch
else
# no candidate present, store the whole contents directly in the record
record.contents = contents
end
if record.contents != contents
raise RuntimeError.new("invariant!")
end
record
end
end

70
app/models/blob_entry_p.rb
View File

@@ -12,6 +12,10 @@ class BlobEntryP < ReduxApplicationRecord
foreign_key: :base_sha256,
class_name: "::BlobEntryP"
def base
@base_model ||= super || self.class.ensure(base_sha256) if base_sha256
end
validates_presence_of(
:sha256,
:content_type,
@@ -21,6 +25,22 @@ class BlobEntryP < ReduxApplicationRecord
validates :sha256, length: { is: 32 }
validates :base_sha256, length: { is: 32 }, if: :base_sha256
def self.ensure(sha256)
find_by(sha256: sha256) || begin
be = BlobEntry.find_by(sha256) || raise("not found: #{HexUtil.bin2hex(sha256)}")
create!([
:sha256,
:base_sha256,
:content_type,
:size,
:contents,
:created_at,
].map do |attr|
[attr, be.read_attribute(attr)]
end.to_h)
end
end
def contents
@contents ||= begin
contents_raw = self.read_attribute(:contents)
@@ -36,55 +56,9 @@ class BlobEntryP < ReduxApplicationRecord
self.read_attribute(:contents).size
end
def self.find_or_build_from_legacy(legacy_be)
file_path = legacy_be.file_path
file_name = File.basename file_path
if file_name.length == 64
file_sha256_assumed = HexUtil.hex2bin(file_name)
# try to find existing file before going through all this hassle
entry = BlobEntry.find_by(sha256: file_sha256_assumed)
return entry if entry
end
return nil unless File.exist?(file_path)
# macos / linux slightly differ in their file type handling
if RUBY_PLATFORM =~ /darwin/
file_mime_flags = "-Ib"
elsif RUBY_PLATFORM =~ /linux/
file_mime_flags = "-ib"
else
raise("unknown platform #{RUBY_PLATFORM}")
end
file_contents = IO.binread(file_path)
file_mime = `file #{file_mime_flags} #{file_path}`
raise("error running `file` on #{file_path}: #{file_mime}") if $?.exitstatus != 0
file_mime.chomp!
record = find_or_build(content_type: file_mime, contents: file_contents)
# guess the name is a sha256 hash
if file_name.length == 64
if record.sha256 != file_sha256_assumed
# checksum for an empty file
if record.sha256 == EMPTY_FILE_SHA256
return nil
else
raise("checksum mismatch for #{file_path}: #{HexUtil.bin2hex(record.sha256)} != #{file_name}")
end
end
end
# puts ("[blob entry] built #{file_mime} (#{HexUtil.humansize(record.size)})")
record.created_at = legacy_be.created_at
record.updated_at = legacy_be.updated_at
record
end
def self.find_or_build(content_type:, contents:, candidates: [])
sha256 = Digest::SHA256.digest(contents)
BlobEntry.find_by(sha256: sha256) || begin
BlobEntryP.find_by(sha256: sha256) || begin
build_record(
content_type: content_type,
sha256: sha256,
@@ -101,7 +75,7 @@ class BlobEntryP < ReduxApplicationRecord
]
def self.build_record(content_type:, sha256:, contents:, candidates: [])
record = BlobEntry.new(sha256: sha256, content_type: content_type, size: contents.size)
record = BlobEntryP.new(sha256: sha256, content_type: content_type, size: contents.size)
smallest_patch_size = nil
smallest_patch = nil

118
app/models/domain/fa/post.rb
View File

@@ -113,124 +113,6 @@ class Domain::Fa::Post < ReduxApplicationRecord
self.file_id.present?
end
def self.find_or_build_from_legacy(legacy_post)
existing = find_by(fa_id: legacy_post.fa_id)
if existing && !existing.creator_id
existing.creator = self._find_or_build_legacy_creator(legacy_post)
end
return existing if existing
post = Domain::Fa::Post.new
post.creator = self._find_or_build_legacy_creator(legacy_post)
uri = Addressable::URI.parse(legacy_post.file_url) if legacy_post.file_url
uri.scheme ||= "https" if uri
if uri && !uri.host && (uri.path =~ /^\/full\//)
# puts "[domain fa post] (#{legacy_post.id}) invalid path for file, skipping: #{legacy_post.file_url}"
uri = nil
elsif uri && legacy_post.blob_entry
legacy_be = legacy_post.blob_entry
# yay, check if we need to make a new blob entry
blob_entry = ::BlobEntry.find_or_build_from_legacy(legacy_be)
# although the blob entry backing file may be missing
if blob_entry
# may be able to be found by original blob entry id
original_le = ::Legacy::HttpLogEntry.find_by(blob_entry_id: legacy_be.id)
log_entry = ::HttpLogEntry.find_by(id: original_le.id) || begin
::HttpLogEntry.build_from_legacy(original_le)
end if original_le
log_entry ||= begin
legacy_hle = ::Legacy::HttpLogEntry.find_by(
host: uri.host,
path: uri.path,
)
legacy_hle && ::HttpLogEntry.find_by(
id: legacy_hle.id,
response_sha256: blob_entry.sha256,
)
end
# couldn't reconstruct from a legacy http log entry, try to guess and make a new one
log_entry ||=
::HttpLogEntry.new({
uri: uri,
status_code: 200,
verb: :get,
response_time_ms: -1,
content_type: blob_entry.content_type,
requested_at: Time.now,
request_headers: ::HttpLogEntryHeader.find_or_create(headers: {}),
response_headers: ::HttpLogEntryHeader.find_or_create(headers: {}),
response: blob_entry,
performed_by: "legacy",
})
raise("mismatch") unless log_entry.response == blob_entry
raise("mismatch") unless HexUtil.bin2hex(log_entry.response.sha256) == legacy_be.sha256
post.file = log_entry
else
puts "[domain fa post] (#{legacy_post.id}) unable to reconstruct blob entry from #{legacy_be.id}"
end
end
post.state = case legacy_post.state.to_sym
when :seen_listing then :ok
when :scanned_submission then :ok
when :scan_error then :scan_error
when :static_error then :file_error
when :have_static then :ok
else raise("unhandled state: #{legacy_post.state}")
end
post.state_detail["legacy_state"] = {
state: legacy_post.state,
state_error: legacy_post.state_error,
}
fields_to_copy = [
:fa_id,
:title,
:category,
:theme,
:species,
:gender,
:keywords,
:description,
:num_favorites,
:num_comments,
:num_views,
:updated_at,
:created_at,
].each do |field|
post.send(:"#{field}=", legacy_post.send(field))
end
post.file_url_str = uri.to_s if uri
post
end
def self._find_or_build_legacy_creator(legacy_post)
if legacy_post.creator
::Domain::Fa::User.find_or_build_from_legacy(legacy_post.creator)
elsif legacy_post.creator_name
::Domain::Fa::User.find_by(url_name: ::Domain::Fa::User.name_to_url_name(legacy_post.creator_name)) ||
::Domain::Fa::User.new({
name: legacy_post.creator_name,
url_name: ::Domain::Fa::User.name_to_url_name(legacy_post.creator_name),
})
else
# post may have not been scanned or is marked as deleted
nil
end
end
def self.hash_from_submission_parser_helper(submission, first_seen_log_entry: nil)
creator = Domain::Fa::User.find_or_create_by({
url_name: submission.artist_url_name,

31
app/models/domain/fa/user.rb
View File

@@ -150,37 +150,6 @@ class Domain::Fa::User < ReduxApplicationRecord
end
end
def self.find_or_build_from_legacy(legacy_user)
existing = find_by(url_name: legacy_user.url_name)
return existing if existing
user = self.new({
name: legacy_user.name || raise("no name: #{legacy_user}"),
url_name: legacy_user.url_name || raise("no url_name: #{legacy_user}"),
full_name: legacy_user.full_name,
})
# bulk-copy existing fields
[
:artist_type,
:mood,
:profile_html,
:num_pageviews,
:num_submissions,
:num_comments_recieved,
:num_comments_given,
:num_journals,
:num_favorites,
:registered_at,
:created_at,
:updated_at,
].each do |field|
user.send(:"#{field}=", legacy_user.send(field))
end
user.scanned_gallery_at = legacy_user.scanned_gallery
user.scanned_page_at = legacy_user.scanned_page
user
end
def self.find_or_build_from_submission_parser(submission_parser)
unless submission_parser.is_a?(Domain::Fa::Parser::ListedSubmissionParserHelper) ||
submission_parser.is_a?(Domain::Fa::Parser::SubmissionParserHelper)

6
app/models/domain/fa/user_avatar.rb
View File

@@ -15,9 +15,13 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
end
belongs_to :user, class_name: "::Domain::Fa::User"
belongs_to :file, foreign_key: :file_sha256, class_name: "::BlobEntry", optional: true
belongs_to :file, foreign_key: :file_sha256, class_name: "::BlobEntryP", optional: true
belongs_to :log_entry, class_name: "::HttpLogEntry", optional: true
def file
@file_model ||= BlobEntryP.ensure(file_sha256) if file_sha256
end
before_validation do
file_uri = Addressable::URI.parse(file_url_str)
end

13
app/models/http_log_entry.rb
View File

@@ -12,11 +12,20 @@ class HttpLogEntry < ReduxApplicationRecord
serverhost-1
], _prefix: true
belongs_to :response,
belongs_to :response_legacy,
foreign_key: :response_sha256,
class_name: "::BlobEntry",
optional: true
belongs_to :response,
foreign_key: :response_sha256,
class_name: "::BlobEntryP",
autosave: true
def response
@response_model ||= super || BlobEntryP.ensure(response_sha256) if response_sha256
end
belongs_to :request_headers,
class_name: "::HttpLogEntryHeader"
@@ -63,7 +72,7 @@ class HttpLogEntry < ReduxApplicationRecord
legacy_model.blob_entry.present?
if can_reconstruct_be
blob_entry = ::BlobEntry.find_or_build_from_legacy(legacy_model.blob_entry)
blob_entry = ::BlobEntryP.find_or_build_from_legacy(legacy_model.blob_entry)
blob_sha256 = HexUtil.hex2bin(legacy_model.resp_body)
unless blob_entry.sha256 == blob_sha256
raise("mismatch for legacy http entry #{legacy_model.id} / legacy blob entry #{legacy_model.blob_entry.id}")

6
db/redux_migrate/20230519233707_remove_be_foreign_key_constraints.rb Normal file
View File

@@ -0,0 +1,6 @@
class RemoveBeForeignKeyConstraints < ActiveRecord::Migration[7.0]
def change
remove_foreign_key "domain_fa_user_avatars", "blob_entries", column: "file_sha256", primary_key: "sha256"
remove_foreign_key "http_log_entries", "blob_entries", column: "response_sha256", primary_key: "sha256"
end
end

4
db/schema.rb generated
View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 2023_05_19_002300) do
ActiveRecord::Schema[7.0].define(version: 2023_05_19_233707) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_stat_statements"
enable_extension "pg_trgm"
@@ -1036,7 +1036,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_05_19_002300) do
add_foreign_key "domain_fa_posts", "domain_fa_users", column: "creator_id"
add_foreign_key "domain_fa_posts", "http_log_entries", column: "file_id"
add_foreign_key "domain_fa_user_avatar_versions", "domain_fa_user_avatars", column: "item_id"
add_foreign_key "domain_fa_user_avatars", "blob_entries", column: "file_sha256", primary_key: "sha256"
add_foreign_key "domain_fa_user_avatars", "domain_fa_users", column: "user_id"
add_foreign_key "domain_fa_user_avatars", "http_log_entries", column: "log_entry_id"
add_foreign_key "domain_fa_user_factors", "domain_fa_users", column: "user_id"
@@ -1044,7 +1043,6 @@ ActiveRecord::Schema[7.0].define(version: 2023_05_19_002300) do
add_foreign_key "domain_twitter_medias", "http_log_entries", column: "file_id"
add_foreign_key "domain_twitter_tweets", "domain_twitter_users", column: "author_id"
add_foreign_key "domain_twitter_user_versions", "domain_twitter_users", column: "item_id"
add_foreign_key "http_log_entries", "blob_entries", column: "response_sha256", primary_key: "sha256"
add_foreign_key "http_log_entries", "http_log_entries", column: "caused_by_id"
add_foreign_key "http_log_entries", "http_log_entry_headers", column: "request_headers_id"
add_foreign_key "http_log_entries", "http_log_entry_headers", column: "response_headers_id"

2
spec/lib/domain/fa/user_enquerer_spec.rb
View File

@@ -21,7 +21,7 @@ describe Domain::Fa::UserEnqueuer do
enqueuer.run_once
expect(get_enqueued_users.call.length).to eq(5)
expect(get_enqueued_users.call).to eq(users[0...5])
expect(SpecUtil.enqueued_jobs.length).to eq(20)
expect(SpecUtil.enqueued_jobs.length).to eq(25)
SpecUtil.shift_jobs(Domain::Fa::Job::UserFollowsJob)
enqueuer.run_once

28
spec/models/domain/fa/user_spec.rb
View File

@@ -1,34 +1,6 @@
require "rails_helper"
describe Domain::Fa::User do
it "can be built from legacy" do
old_user = Legacy::Fa::User.find_by(name: "Meesh") || raise
new_user = Domain::Fa::User.find_or_build_from_legacy(old_user)
assert new_user.new_record?
assert_equal "Meesh", new_user.name
assert_equal "Meesh", new_user.full_name
assert_equal "meesh", new_user.url_name
assert_equal "PrOn Artist", new_user.artist_type
assert_match /Refunds will be granted/, new_user.profile_html
[
:registered_at,
:created_at,
:updated_at,
].each do |attr|
assert_equal old_user.send(attr), new_user.send(attr)
end
assert_equal old_user.scanned_gallery, new_user.scanned_gallery_at
assert_equal old_user.scanned_page, new_user.scanned_page_at
assert_equal 1715730, new_user.num_pageviews
assert_equal 1225, new_user.num_submissions
assert_equal 40123, new_user.num_comments_recieved
assert_equal 17386, new_user.num_comments_given
assert_equal 13, new_user.num_journals
assert_equal 893478, new_user.num_favorites
end
it "validates that url_name and name match using the conversion rules" do
user = Domain::Fa::User.create({
name: "Foo_User",

2
spec/spec_util.rb
View File

@@ -85,7 +85,7 @@ class SpecUtil
content_type: "text/plain",
contents: nil
)
BlobEntry.find_or_build(
BlobEntryP.find_or_build(
content_type: content_type,
contents: contents || random_string(1024),
)

59
test/models/blob_entry_p_test.rb Normal file
View File

@@ -0,0 +1,59 @@
require "test_helper"
class BlobEntryPTest < ActiveSupport::TestCase
test "building a blob works" do
blob = TestUtil.build_blob_entry
assert blob.valid?
assert blob.save
end
test "model cannot be updated" do
model = TestUtil.build_blob_entry
model.save!
model.contents = "new contenets"
assert_raises(ActiveRecord::ReadOnlyRecord) do
model.save!
end
end
test "model cannot be deleted" do
model = TestUtil.build_blob_entry
model.save!
assert_raises(ActiveRecord::ReadOnlyRecord) do
model.destroy
end
end
test "model dual-writes a BlobEntryP model" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntryP.find_by(sha256: model.sha256)
assert_same_blob_entry model, model_p
end
test "ensure works for creating a blob entry" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntryP.ensure(model.sha256)
assert_same_blob_entry model, model_p
end
def assert_same_blob_entry(model, model_p)
[
:sha256,
:base_sha256,
:contents,
:size,
:created_at,
:content_type,
].each do |attr|
expected = model.send(attr)
actual = model_p.send(attr)
if expected.nil?
assert_nil actual, "#{attr} mismatch"
else
assert_equal expected, actual, "#{attr} mismatch"
end
end
end
end

51
test/models/blob_entry_test.rb
View File

@@ -1,51 +0,0 @@
require "test_helper"
class BlobEntryTest < ActiveSupport::TestCase
test "building a blob works" do
blob = TestUtil.build_blob_entry
assert blob.valid?
assert blob.save
end
test "can create from a legacy blob entry" do
legacy_be = ::Legacy::BlobEntry.find(1370198)
assert File.exist?(legacy_be.file_path), legacy_be.file_path
new_be = ::BlobEntry.find_or_build_from_legacy(legacy_be)
refute_nil new_be
assert new_be.valid?, new_be.errors.full_messages
assert new_be.new_record?
assert_equal "5ed3a0400ac50f721123c7a8c638da8b19bf563f8e880f9abb36dcb38395bc82", HexUtil.bin2hex(new_be.sha256)
assert_equal legacy_be.file_size, new_be.bytes_stored
assert_equal 313065, new_be.bytes_stored
assert_equal "image/png; charset=binary", new_be.content_type
new_be.save!
assert_equal legacy_be.updated_at, new_be.updated_at
assert_equal legacy_be.created_at, new_be.created_at
end
test "model cannot be updated" do
model = TestUtil.build_blob_entry
model.save!
model.contents = "new contenets"
assert_raises(ActiveRecord::ReadOnlyRecord) do
model.save!
end
end
test "model cannot be deleted" do
model = TestUtil.build_blob_entry
model.save!
assert_raises(ActiveRecord::ReadOnlyRecord) do
model.destroy
end
end
test "model dual-writes a BlobEntryP model" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntryP.find_by(sha256: model.sha256)
[:sha256, :base_sha256, :contents, :size, :created_at, :content_type].each do |attr|
assert_equal model.send(attr), model_p.send(attr), "#{attr} mismatch"
end
end
end

132
test/models/domain/fa/post_test.rb
View File

@@ -1,86 +1,4 @@
class Domain::Fa::PostTest < ActiveSupport::TestCase
test "can build from legacy post without a legacy http entry" do
legacy_post = ::Legacy::Fa::Post.find_by!(fa_id: 19177819)
# made a valid, unpersisted record
new_post = ::Domain::Fa::Post.find_or_build_from_legacy(legacy_post)
assert new_post.valid?, new_post.errors.full_messages
refute new_post.persisted?
# the associated file was populated
assert new_post.file.present?
refute new_post.file.persisted?
assert_equal 19177819, new_post.fa_id
refute_nil new_post.creator
assert_equal "Meesh", new_post.creator.name
assert_equal "MASSIVE ART PACK 6 - Available now!", new_post.title
assert_equal "Artwork (Digital)", new_post.category
assert_equal "All", new_post.theme
assert_equal "Unspecified / Any", new_post.species
assert_equal "Any", new_post.gender
assert_equal "d.facdn.net", new_post.file_uri.host
assert_equal "/art/meesh/1456189705/1456189705.meesh_production5.png", new_post.file_uri.path
assert_equal 8, new_post.num_favorites
assert_equal 2, new_post.num_comments
assert_equal 904, new_post.num_views
assert_equal ["meesh", "nsfw", "art", "pack", "adult", "boner", "touching"], new_post.keywords
assert_match /with a handful that have special edits/, new_post.description
new_post.save!
assert_equal 0, new_post.versions.count
assert new_post.file.id
new_post.num_views = 1000
new_post.save!
assert_equal 1, new_post.versions.count
old_post = new_post.versions.last.reify
assert_equal 904, old_post.num_views
# check that it works after reload
new_post.reload
assert new_post.file.present?
assert new_post.creator.present?
end
test "can build from a legacy post with a legacy http entry" do
# a legacy post / log entry, but they're only associated by their
# blob entry (no direct association otherwise)
legacy_post = ::Legacy::Fa::Post.find(10117853)
legacy_le = ::Legacy::HttpLogEntry.find(1766)
expected_sha256 = HexUtil.hex2bin("41F8DAF7772D11F80AFE56B742087A2D1AB372E08B69E1284BE4FEFEC2AD0C7F")
assert_equal legacy_le.blob_entry_id, legacy_post.blob_entry_id
new_post = ::Domain::Fa::Post.find_or_build_from_legacy(legacy_post)
assert new_post.valid?, new_post.errors.full_messages
refute new_post.persisted?
assert new_post.file.present?
refute new_post.file.persisted?
assert_equal expected_sha256, new_post.file.response.sha256
assert_equal 21826851, new_post.fa_id
# TODO - populate creator
# assert_equal "Drake_Ergenthal", new_post.creator.name
assert_equal "Story", new_post.category
assert_equal "All", new_post.theme
[
:verb,
:content_type,
:requested_at,
:updated_at,
:created_at,
].each do |attr|
assert_equal legacy_le.send(attr), new_post.file.send(attr)
end
assert_equal legacy_le.full_path, new_post.file.uri_str
assert_equal legacy_le.response_time, new_post.file.response_time_ms
assert_equal legacy_le.status, new_post.file.status_code
end
test "remove buggy prefixes" do
# TODO - implement this
# Some posts have a title prefixed with "Font size adjustment: smallerlarger"
@@ -88,54 +6,4 @@ class Domain::Fa::PostTest < ActiveSupport::TestCase
# Legacy::Fa::Post.where("title like ?", "Font size adjustment: smallerlarger%").count
# => 7056
end
test "can build when the legacy post user does not exist" do
legacy_post = Legacy::Fa::Post.find(13950325)
new_post = Domain::Fa::Post.find_or_build_from_legacy(legacy_post)
assert new_post.valid?, new_post.errors.full_messages
new_post.save!
assert_equal 25793413, new_post.fa_id
assert_equal "Kenny-Mccormick", new_post.creator.name
assert_equal "kenny-mccormick", new_post.creator.url_name
assert_match /Like last time/, new_post.description
end
test "handles case where no creator could be identified" do
# this is a post which was removed
legacy_post = Legacy::Fa::Post.find(13950327)
new_post = Domain::Fa::Post.find_or_build_from_legacy(legacy_post)
assert new_post.valid?, new_post.errors.full_messages
new_post.save!
assert_equal 25793411, new_post.fa_id
end
test "skips posts with an invalid full file url" do
legacy_post = ::Legacy::Fa::Post.find(3234144)
new_post = ::Domain::Fa::Post.find_or_build_from_legacy(legacy_post)
assert new_post.valid?, new_post.errors.full_messages
assert new_post.file.nil?
new_post.save!
end
test "will update the creator if it didn't already have one" do
legacy_post = ::Legacy::Fa::Post.find(4936259)
# and that's all we create
post = ::Domain::Fa::Post.create!({
fa_id: legacy_post.fa_id,
state: :ok,
})
assert_nil post.creator
assert_nil ::Domain::Fa::User.find_by(name: "Rodrick-Dragon")
updated_post = ::Domain::Fa::Post.find_or_build_from_legacy(legacy_post)
assert_equal post.id, updated_post.id, "is not the same post"
assert updated_post.persisted?, "post should be persisted"
refute_nil updated_post.creator, "post should have a creator"
refute updated_post.creator.persisted?, "creator should be newly created"
assert_equal "Rodrick-Dragon", updated_post.creator.name, "not the right creator"
updated_post.save!
assert updated_post.creator.persisted?
end
end

2
test/test_util.rb
View File

@@ -33,7 +33,7 @@ module TestUtil
content_type: "text/plain",
contents: nil
)
BlobEntry.find_or_build(
BlobEntryP.find_or_build(
content_type: content_type,
contents: contents || random_string(1024),
)