get tests running against local legacy db fixtures

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "editor.formatOnSave": true,
+    "workbench.editor.titleScrollbarSizing": "large",
+    "window.title": "${activeEditorMedium}${separator}${rootName}${separator}${profileName}"
+}

Gemfile

@@ -66,6 +66,8 @@ group :development do
   gem "rack-mini-profiler", require: ["enable_rails_patches", "rack-mini-profiler"]
   # Speed up commands on slow machines / big apps [https://github.com/rails/spring]
   # gem "spring"
+
+  gem "rufo", require: false
 end
 
 group :log_watcher do

Gemfile.lock

@@ -265,6 +265,7 @@ GEM
       ruby-prof (~> 1.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
+    rufo (0.15.1)
     selenium-webdriver (4.8.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
@@ -352,6 +353,7 @@ DEPENDENCIES
   rspec-rails
   ruby-prof
   ruby-prof-speedscope
+  rufo
   selenium-webdriver
   sprockets-rails
   sqlite3 (~> 1.4)

app/models/blob_entry.rb

@@ -55,8 +55,17 @@ class BlobEntry < ReduxApplicationRecord
 
     return nil unless File.exist?(file_path)
 
+    # macos / linux slightly differ in their file type handling
+    if RUBY_PLATFORM =~ /darwin/
+      file_mime_flags = "-Ib"
+    elsif RUBY_PLATFORM =~ /linux/
+      file_mime_flags = "-ib"
+    else
+      raise("unknown platform #{RUBY_PLATFORM}")
+    end
+
     file_contents = IO.binread(file_path)
-    file_mime = `file -ib #{file_path}`
+    file_mime = `file #{file_mime_flags} #{file_path}`
     raise("error running `file` on #{file_path}: #{file_mime}") if $?.exitstatus != 0
     file_mime.chomp!
     record = find_or_build(content_type: file_mime, contents: file_contents)

app/models/legacy_application_record.rb

@@ -9,7 +9,11 @@ end
 
 class Legacy::SConfig
   def self.data_dir
-    "/home/scraper/scraper_data_original/scraper_data"
+    if Rails.env.test?
+      Rails.root.join "test/fixtures/legacy/s_config"
+    else
+      "/home/scraper/scraper_data_original/scraper_data"
+    end
   end
 
   def self.blob_data_dir

config/database.yml

@@ -19,39 +19,36 @@ redux_prod: &redux_prod
   migrations_paths: db/redux_migrate
   pool: 4
 
-redux_test: &redux_test
-  adapter: postgresql
-  host: postgres.local
-  port: 5432
-  database: redux_test
-  username: scraper_redux
-  password: pdkFLqRmQwPUPaDDC4pX
-  migrations_paths: db/redux_migrate
-  pool: 2
-
 legacy_prod: &legacy_prod
   adapter: postgresql
   host: postgres.local
   port: 5432
   database: legacy_prod
-  # username: scraper_legacy
-  # password: zL7zDRXycLhLFJLQj5Zh
   username: scraper_redux
   password: pdkFLqRmQwPUPaDDC4pX
   migrations_paths: db/legacy_migrate
   database_tasks: false
   pool: 2
 
-legacy_prod_readonly: &legacy_prod_readonly
+local_redux_test: &local_redux_test
   adapter: postgresql
-  host: postgres.local
+  host: localhost
   port: 5432
-  database: legacy_prod
-  username: scraper_legacy
-  password: zL7zDRXycLhLFJLQj5Zh
+  database: redux_test
+  username: scraper_redux
+  password: pdkFLqRmQwPUPaDDC4pX
+  migrations_paths: db/redux_migrate
+  pool: 4
+
+local_legacy_test: &local_legacy_test
+  adapter: postgresql
+  host: localhost
+  port: 5432
+  database: legacy_test
+  username: scraper_redux
+  password: pdkFLqRmQwPUPaDDC4pX
   migrations_paths: db/legacy_migrate
-  database_tasks: false
-  pool: 2
+  pool: 4
 
 development:
   redux:
@@ -64,9 +61,9 @@ development:
 # Do not set this db to the same as development or production.
 test:
   redux:
-    <<: *redux_test
+    <<: *local_redux_test
   legacy:
-    <<: *legacy_prod_readonly
+    <<: *local_legacy_test
 
 production:
   redux:

db/legacy_schema.rb

@@ -0,0 +1,397 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[7.0].define(version: 0) do
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "hstore"
+  enable_extension "intarray"
+  enable_extension "pg_stat_statements"
+  enable_extension "pg_trgm"
+  enable_extension "plpgsql"
+
+  create_table "blob_entries", id: :serial, force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "file_size"
+    t.integer "refcount"
+    t.string "sha256", limit: 64
+    t.integer "dir_depth", default: 2, null: false
+    t.index ["sha256"], name: "index_blob_entries_on_sha256", unique: true
+  end
+
+  create_table "cache_http_log_entries", id: :serial, force: :cascade do |t|
+    t.string "scheme"
+    t.string "host"
+    t.string "path"
+    t.string "query"
+    t.integer "verb"
+    t.integer "status"
+    t.integer "response_time"
+    t.string "content_type"
+    t.integer "response_size"
+    t.integer "parent_log_entry_id"
+    t.integer "blob_entry_id"
+    t.boolean "gzipped"
+    t.datetime "requested_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.binary "resp_body"
+    t.string "imported_from_file"
+    t.integer "req_headers_id"
+    t.integer "res_headers_id"
+    t.integer "diff_type", default: 0
+    t.binary "native_blob_entry_sha256"
+    t.index ["imported_from_file"], name: "index_cache_http_log_entries_on_imported_from_file"
+    t.index ["path", "host"], name: "index_cache_http_log_entries_on_path_and_host"
+    t.index ["path"], name: "cache_http_log_entries_path_idx", opclass: :gist_trgm_ops, using: :gist
+    t.index ["path"], name: "index_pattern_ops_on_hle_entry_path"
+    t.index ["requested_at"], name: "index_cache_http_log_entries_on_requested_at"
+  end
+
+  create_table "cache_http_log_entry_headers", id: :serial, force: :cascade do |t|
+    t.hstore "headers", null: false
+    t.binary "sha256", null: false
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.index ["sha256"], name: "index_cache_http_log_entry_headers_on_sha256", unique: true
+  end
+
+  create_table "e621_posts", id: :serial, force: :cascade do |t|
+    t.integer "e621_id", null: false
+    t.string "md5", null: false
+    t.string "sources"
+    t.string "file_url", null: false
+    t.string "file_ext", null: false
+    t.string "description"
+    t.integer "rating"
+    t.integer "width"
+    t.integer "height", null: false
+    t.string "tags_string", null: false
+    t.integer "status"
+    t.integer "score"
+    t.boolean "removed"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.string "artists"
+    t.integer "e621_count"
+    t.string "author"
+    t.string "e621_status"
+    t.integer "blob_entry_id"
+    t.integer "imgsearch_entry_id"
+    t.index ["blob_entry_id"], name: "index_e621_posts_on_blob_entry_id"
+    t.index ["e621_id"], name: "index_e621_posts_on_e621_id", unique: true
+    t.index ["imgsearch_entry_id"], name: "index_e621_posts_on_imgsearch_entry_id"
+    t.index ["md5"], name: "index_e621_posts_on_md5"
+  end
+
+  create_table "e621_taggings", id: :serial, force: :cascade do |t|
+    t.integer "tag_id"
+    t.integer "post_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["post_id"], name: "index_e621_taggings_on_post_id"
+    t.index ["tag_id"], name: "index_e621_taggings_on_tag_id"
+  end
+
+  create_table "e621_tags", id: :serial, force: :cascade do |t|
+    t.string "value", null: false
+    t.integer "e621_id"
+    t.integer "type"
+    t.integer "e621_count"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["value"], name: "index_e621_tags_on_value"
+  end
+
+  create_table "fa_failed_users", id: :serial, force: :cascade do |t|
+    t.string "url_name"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "fa_post_descriptions", id: :serial, force: :cascade do |t|
+    t.text "value", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "fa_posts", id: :serial, force: :cascade do |t|
+    t.integer "fa_id", null: false
+    t.string "creator_name"
+    t.integer "creator_id"
+    t.string "title"
+    t.string "category"
+    t.string "theme"
+    t.string "species"
+    t.string "gender"
+    t.string "keywords"
+    t.string "file_url"
+    t.integer "blob_entry_id"
+    t.string "old_files"
+    t.integer "num_favorites"
+    t.integer "num_comments"
+    t.integer "num_views"
+    t.datetime "posted_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.boolean "in_gallery"
+    t.string "in_folders"
+    t.integer "state", default: 0
+    t.string "state_error"
+    t.string "folder_name"
+    t.string "gallery_page_logfile"
+    t.string "submission_page_logfile"
+    t.integer "submission_page_log_entry_id"
+    t.integer "description_id"
+    t.integer "imgsearch_entry_id"
+    t.index ["blob_entry_id"], name: "index_fa_posts_on_blob_entry_id"
+    t.index ["creator_id"], name: "index_fa_posts_on_creator_id"
+    t.index ["creator_name"], name: "index_fa_posts_on_creator_name"
+    t.index ["fa_id"], name: "index_fa_posts_on_fa_id", unique: true
+    t.index ["imgsearch_entry_id"], name: "index_fa_posts_on_imgsearch_entry_id"
+  end
+
+  create_table "fa_users", id: :serial, force: :cascade do |t|
+    t.string "name", null: false
+    t.string "full_name"
+    t.string "artist_type"
+    t.string "mood"
+    t.text "profile_html"
+    t.integer "num_pageviews"
+    t.integer "num_submissions"
+    t.integer "num_comments_recieved"
+    t.integer "num_comments_given"
+    t.integer "num_journals"
+    t.integer "num_favorites"
+    t.datetime "registered_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.string "url_name", null: false
+    t.datetime "scanned_gallery"
+    t.datetime "scanned_page"
+    t.string "user_page_logfile"
+    t.integer "user_page_log_entry_id"
+    t.index ["name"], name: "index_fa_users_on_name", unique: true
+    t.index ["scanned_gallery"], name: "index_fa_users_on_scanned_gallery"
+    t.index ["url_name"], name: "index_fa_users_on_url_name", unique: true
+  end
+
+  create_table "ib_posts", id: :serial, force: :cascade do |t|
+    t.string "gallery_logfile"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "ib_user", id: :serial, force: :cascade do |t|
+  end
+
+  create_table "imgsearch_dbs", id: :serial, force: :cascade do |t|
+    t.string "path", null: false
+    t.string "name", null: false
+    t.integer "hash_bits", null: false
+    t.integer "max_error", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "lite_ad_hoc_posts", force: :cascade do |t|
+    t.string "file_ext"
+    t.string "file"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "lite_e621_posts", primary_key: "e621_id", force: :cascade do |t|
+    t.string "e621_file_url"
+    t.string "file_ext"
+    t.integer "main_asset_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "lite_fa_posts", primary_key: "fa_id", force: :cascade do |t|
+    t.string "file_ext"
+    t.integer "main_asset_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.string "file_url"
+    t.integer "creator_id"
+    t.string "title"
+    t.index ["creator_id"], name: "index_lite_fa_posts_on_creator_id"
+  end
+
+  create_table "lite_fa_users", force: :cascade do |t|
+    t.string "url_name", null: false
+    t.index ["url_name"], name: "index_lite_fa_users_on_url_name"
+  end
+
+  create_table "lite_media_file_fingerprints", force: :cascade do |t|
+    t.integer "object_type", limit: 2, null: false
+    t.integer "object_id", null: false
+    t.bigint "fingerprints", array: true
+    t.index ["object_type", "object_id"], name: "index_lite_media_file_fingerprints_on_object_type_and_object_id", unique: true
+  end
+
+  create_table "lite_s3_objects", force: :cascade do |t|
+    t.string "sha256"
+    t.string "key"
+    t.integer "size"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["key"], name: "index_lite_s3_objects_on_key", unique: true
+  end
+
+  create_table "pghero_query_stats", id: :serial, force: :cascade do |t|
+    t.text "database"
+    t.text "user"
+    t.text "query"
+    t.bigint "query_hash"
+    t.float "total_time"
+    t.bigint "calls"
+    t.datetime "captured_at"
+    t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at"
+  end
+
+  create_table "pghero_space_stats", id: :serial, force: :cascade do |t|
+    t.text "database"
+    t.text "schema"
+    t.text "relation"
+    t.bigint "size"
+    t.datetime "captured_at"
+    t.index ["database", "captured_at"], name: "index_pghero_space_stats_on_database_and_captured_at"
+  end
+
+  create_table "watch_jobs", id: :serial, force: :cascade do |t|
+    t.string "execute_class"
+    t.string "params"
+    t.string "return"
+    t.integer "state"
+    t.datetime "execute_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "enqueue_strategy", default: 0
+    t.integer "redo_delay_seconds"
+    t.integer "priority", default: 0
+    t.string "http_class"
+    t.index ["execute_at"], name: "index_watch_jobs_on_execute_at"
+    t.index ["execute_class"], name: "index_watch_jobs_on_execute_class"
+    t.index ["priority"], name: "index_watch_jobs_on_priority"
+    t.index ["state"], name: "index_watch_jobs_on_state"
+  end
+
+  create_table "watched_users", id: :serial, force: :cascade do |t|
+    t.integer "user_id", null: false
+    t.integer "watch_job_id", null: false
+    t.datetime "scanned_user_gallery"
+    t.datetime "scanned_user_page"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "scan_delay"
+  end
+
+  create_table "weasyl_attempted_posts", id: :serial, force: :cascade do |t|
+    t.integer "weasyl_id", null: false
+    t.integer "enum_type", null: false
+    t.integer "status"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["enum_type", "weasyl_id"], name: "index_weasyl_attempted_posts_on_enum_type_and_weasyl_id", unique: true
+  end
+
+  create_table "weasyl_descriptions", id: :serial, force: :cascade do |t|
+    t.text "value", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "weasyl_joins_user_follows", id: :serial, force: :cascade do |t|
+    t.integer "follower_id", null: false
+    t.integer "followed_id", null: false
+    t.index ["follower_id", "followed_id"], name: "index_weasyl_joins_user_follows_on_follower_id_and_followed_id", unique: true
+  end
+
+  create_table "weasyl_joins_user_friends", id: :serial, force: :cascade do |t|
+    t.integer "a_id", null: false
+    t.integer "b_id", null: false
+    t.index ["a_id", "b_id"], name: "index_weasyl_joins_user_friends_on_a_id_and_b_id", unique: true
+  end
+
+  create_table "weasyl_medias", id: :serial, force: :cascade do |t|
+    t.string "url", null: false
+    t.integer "mediaid"
+    t.integer "blob_entry_id"
+    t.integer "status"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["mediaid"], name: "index_weasyl_medias_on_mediaid", unique: true
+    t.index ["url"], name: "index_weasyl_medias_on_url", unique: true
+  end
+
+  create_table "weasyl_posts", id: :serial, force: :cascade do |t|
+    t.integer "weasyl_id", null: false
+    t.string "title", null: false
+    t.integer "description_id"
+    t.datetime "posted_at"
+    t.string "tags", array: true
+    t.integer "enum_type", null: false
+    t.integer "enum_subtype"
+    t.integer "enum_rating", null: false
+    t.integer "num_comments"
+    t.integer "num_views"
+    t.integer "num_favorites"
+    t.string "folder_name"
+    t.integer "weasyl_folder_id"
+    t.integer "owner_id", null: false
+    t.integer "submission_media_id"
+    t.datetime "full_scanned_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["enum_type", "weasyl_id"], name: "index_weasyl_posts_on_enum_type_and_weasyl_id", unique: true
+  end
+
+  create_table "weasyl_users", id: :serial, force: :cascade do |t|
+    t.string "full_name"
+    t.string "login_name", null: false
+    t.integer "description_id"
+    t.integer "avatar_media_id"
+    t.datetime "scanned_gallery_at"
+    t.datetime "scanned_userpage_at"
+    t.datetime "scanned_followers_following_at"
+    t.integer "userid"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["login_name"], name: "index_weasyl_users_on_login_name", unique: true
+  end
+
+  create_table "xtwitter_tweets", id: :serial, force: :cascade do |t|
+    t.string "creator_screen_name", null: false
+    t.integer "creator_user_id", null: false
+    t.string "twitter_id", null: false
+    t.json "attrs"
+    t.string "media_uri"
+    t.string "media_ext"
+    t.integer "http_log_entry_page_id"
+    t.integer "http_log_entry_media_id"
+    t.integer "blob_entry_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  create_table "xtwitter_users", id: :serial, force: :cascade do |t|
+    t.string "screen_name", null: false
+    t.integer "user_id", null: false
+    t.datetime "scanned_timeline_at"
+    t.datetime "scanned_profile_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+end

db/redux_schema.rb

@@ -1,16 +0,0 @@
-# This file is auto-generated from the current state of the database. Instead
-# of editing this file, please use the migrations feature of Active Record to
-# incrementally modify your database, and then regenerate this schema definition.
-#
-# This file is the source Rails uses to define your schema when running `bin/rails
-# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
-# be faster and is potentially less error prone than running all of your
-# migrations from scratch. Old migrations may fail to apply correctly if those
-# migrations use external dependencies or application code.
-#
-# It's strongly recommended that you check this file into your version control system.
-
-ActiveRecord::Schema[7.0].define(version: 0) do
-  # These are extensions that must be enabled in order to support this database
-  enable_extension "plpgsql"
-end

test/fixtures/legacy/blob_entries.yml

@@ -0,0 +1,32 @@
+---
+id_1370198:
+  id: 1370198
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-02-27 12:00:51.488220000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-02-27 12:00:51.488220000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-02-27 12:00:51.488220000 Z
+    zone: *1
+    time: 2016-02-27 12:00:51.488220000 Z
+  file_size: 313065
+  refcount: 1
+  sha256: 5ed3a0400ac50f721123c7a8c638da8b19bf563f8e880f9abb36dcb38395bc82
+  dir_depth: 2
+
+id_8663902:
+  id: 8663902
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-11-26 22:09:59.707602000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-11-26 22:09:59.707602000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-11-26 22:09:59.707602000 Z
+    zone: *1
+    time: 2016-11-26 22:09:59.707602000 Z
+  file_size: 6936
+  refcount: 2
+  sha256: 41f8daf7772d11f80afe56b742087a2d1ab372e08b69e1284be4fefec2ad0c7f
+  dir_depth: 4

test/fixtures/legacy/fa/post_descriptions.yml

@@ -0,0 +1,50 @@
+---
+id_11551023:
+  id: 11551023
+  value: "<a href=\"/user/kenny-mccormick/\"><img class=\"avatar\" alt=\"kenny-mccormick\"
+    src=\"//a.facdn.net/1485168311/kenny-mccormick.gif\"></a>\n                        <br><br>\n
+    \                       [copied from my DA page]<br>\r\n<br>\r\nLike last time,
+    mid week posts are just being dedicated to images that come from the south park
+    stories, starting with a particular fav of mine from chapter 2.<br>\r\n<br>\r\nIt
+    was very fun to design the outfit for Kenny, though it's very different from his
+    show attire. I think he looks really cute, tempted to make the design part of another
+    character at some point, it's just so cute &lt;3<br>\r\n<br>\r\n-<br>\r\n<br>\r\nImage
+    was made by Tato and colored by me.<br>\r\n<br>\r\nWant to support this content?<br>\r\n<a
+    href=\"https://www.patreon.com/KennyCrusader\" title=\"https://www.patreon.com/KennyCrusader\"
+    class=\"auto_link\">https://www.patreon.com/KennyCrusader</a>"
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-02-01 07:50:22.199757000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2018-02-01 07:50:22.199757000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-02-01 07:50:22.199757000 Z
+    zone: *1
+    time: 2018-02-01 07:50:22.199757000 Z
+
+id_144:
+  id: 144
+  value: "\n<a href=\"/user/meesh/\"><img class=\"avatar\" alt=\"meesh\" src=\"//a.facdn.net/1424255659/meesh.gif\"></a>\n<br><br>\n<a
+    href=\"http://meesh.pandachan.org/massive-art-collections/\" title=\"http://meesh.pandachan.org/massive-art-collections/\"
+    class=\"auto_link auto_link_shortened\">http://meesh.pandachan.org/massive-.....t-collections/</a><br>\n<br>\nMy
+    latest Art Pack is here! And it’s a doozy. It includes:<br>\n<br>\n-95% explicit
+    adult content, naturally. Gay, straight and some in-between<br>\n<br>\n-143 unique
+    sketch commissions, with a handful that have special edits. 56 of which have never
+    been posted to my account<br>\n<br>\n-69 sketches that never saw the light of day
+    outside of my weekly streams, or not at all<br>\n<br>\n-27 unique Tier 1 and Tier
+    2 commissions. Including Dog Pound, Pills Chronicles and others, with preliminary
+    stages such as sketches, concepts, and inks<br>\n<br>\n-High resolution and resized
+    versions of every single sketch commission, Tier 1 and Tier 2. All personal sketches
+    are in hi-res or have a hi-res version included<br>\n<br>\n-590 image files in total<br>\n<br>\nIt
+    all can be yours for just $10! You can purchase it right here:<a href=\"http://meesh.pandachan.org/massive-art-collections/\"
+    title=\"http://meesh.pandachan.org/massive-art-collections/\" class=\"auto_link
+    auto_link_shortened\">http://meesh.pandachan.org/massive-.....t-collections/</a> "
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-12-22 17:06:09.549412000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-12-22 17:06:09.549412000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-12-22 17:06:09.549412000 Z
+    zone: *1
+    time: 2016-12-22 17:06:09.549412000 Z

test/fixtures/legacy/fa/posts.yml

@@ -0,0 +1,273 @@
+---
+id_13950325:
+  id: 13950325
+  fa_id: 25793413
+  creator_name: Kenny-Mccormick
+  creator_id:
+  title: "[Commissioned] [Collab] Little Kenny - By Tato"
+  category: Artwork (Digital)
+  theme: Baby fur
+  species: Unspecified / Any
+  gender: Male
+  keywords:
+  - south
+  - park
+  - kenny
+  - mccormick
+  - tato
+  - age
+  - regression
+  - diaper
+  file_url: "//d.facdn.net/art/kenny-mccormick/1513691072/1513691072.kenny-mccormick_kenny_s_new_outfitf.png"
+  blob_entry_id:
+  old_files: []
+  num_favorites: 25
+  num_comments: 4
+  num_views: 263
+  posted_at:
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-02-01 07:50:22.177857000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2018-02-01 07:50:22.177857000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-02-01 07:50:22.201057000 Z
+    zone: *1
+    time: 2018-02-01 07:50:22.201057000 Z
+  in_gallery:
+  in_folders: []
+  state: scanned_submission
+  state_error:
+  folder_name:
+  gallery_page_logfile:
+  submission_page_logfile:
+  submission_page_log_entry_id: 19252126
+  description_id: 11551023
+  imgsearch_entry_id:
+
+id_13950327:
+  id: 13950327
+  fa_id: 25793411
+  creator_name:
+  creator_id:
+  title:
+  category:
+  theme:
+  species:
+  gender:
+  keywords: []
+  file_url:
+  blob_entry_id:
+  old_files: []
+  num_favorites:
+  num_comments:
+  num_views:
+  posted_at:
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-02-01 07:50:23.214173000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2018-02-01 07:50:23.214173000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-02-01 07:50:23.214173000 Z
+    zone: *1
+    time: 2018-02-01 07:50:23.214173000 Z
+  in_gallery:
+  in_folders: []
+  state: scan_error
+  state_error: post removed
+  folder_name:
+  gallery_page_logfile:
+  submission_page_logfile:
+  submission_page_log_entry_id:
+  description_id:
+  imgsearch_entry_id:
+
+id_4936259:
+  id: 4936259
+  fa_id: 7126769
+  creator_name: Rodrick-Dragon
+  creator_id: 27392
+  title: Profile ID
+  category: Artwork (Digital)
+  theme: Doodle
+  species: Dragon (Other)
+  gender: Male
+  keywords: []
+  file_url: "//d.facdn.net/art/rodrick-dragon/1325297838/1325297838.rodrick-dragon_bio.jpg"
+  blob_entry_id: 11923806
+  old_files: []
+  num_favorites: 5
+  num_comments: 3
+  num_views: 30
+  posted_at:
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-03-09 12:21:45.224977000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-03-09 12:21:45.224977000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-01-29 01:14:47.165636000 Z
+    zone: *1
+    time: 2018-01-29 01:14:47.165636000 Z
+  in_gallery:
+  in_folders: []
+  state: have_static
+  state_error:
+  folder_name: Scraps
+  gallery_page_logfile: listing_pages/rodrick-dragon/1457526104-Scraps_1.html
+  submission_page_logfile:
+  submission_page_log_entry_id: 6026946
+  description_id: 4858488
+  imgsearch_entry_id: 6310759
+
+id_3234144:
+  id: 3234144
+  fa_id: 8489215
+  creator_name: LokiLover
+  creator_id: 14101
+  title: Scribbles with Kier
+  category: Artwork (Digital)
+  theme: Doodle
+  species: Unspecified / Any
+  gender: Any
+  keywords:
+  - doodles
+  - OC
+  - opencanvas
+  - goofy
+  - silly
+  - chubby
+  - things
+  - dinosaurs
+  - cows
+  - kitties
+  - spider
+  - gremlins
+  - chibi
+  - super
+  - sexy
+  - cheerleader
+  file_url: "/full/8489215/"
+  blob_entry_id: 6877172
+  old_files: []
+  num_favorites: 5
+  num_comments: 0
+  num_views: 152
+  posted_at:
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-03-04 20:41:26.393592000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-03-04 20:41:26.393592000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-12-23 03:56:27.425752000 Z
+    zone: *1
+    time: 2016-12-23 03:56:27.425752000 Z
+  in_gallery:
+  in_folders: []
+  state: have_static
+  state_error:
+  folder_name: Gallery
+  gallery_page_logfile: listing_pages/lokilover/1476614903-Gallery_2.html
+  submission_page_logfile: submissions/lokilover/1463959274-8489215.html
+  submission_page_log_entry_id:
+  description_id: 3173583
+  imgsearch_entry_id:
+
+id_10117853:
+  id: 10117853
+  fa_id: 21826851
+  creator_name: Drake_Ergenthal
+  creator_id: 106983
+  title: |-
+    Font size adjustment: smallerlarger
+
+    Locust vs. Nature
+  category: Story
+  theme: All
+  species:
+  gender:
+  keywords:
+  - Locust
+  - Grenadier
+  - Hunter
+  - Elite
+  - Savage
+  - Kantus
+  - Miner
+  - quicksand
+  - Gears
+  - of
+  - War
+  file_url: "//d.facdn.net/art/drakeergenthal/stories/1480188610/1480188610.drakeergenthal_gow_fanfic.txt"
+  blob_entry_id: 8663902
+  old_files: []
+  num_favorites: 1
+  num_comments: 0
+  num_views: 11
+  posted_at:
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-11-26 19:44:43.419488000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-11-26 19:44:43.419488000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-12-23 20:32:59.798452000 Z
+    zone: *1
+    time: 2016-12-23 20:32:59.798452000 Z
+  in_gallery:
+  in_folders: []
+  state: have_static
+  state_error: guessing exists
+  folder_name:
+  gallery_page_logfile:
+  submission_page_logfile:
+  submission_page_log_entry_id: 1449
+  description_id: 8039358
+  imgsearch_entry_id:
+
+fa_id_19177819:
+  id: 1
+  fa_id: 19177819
+  creator_name: Meesh
+  creator_id: 1
+  title: MASSIVE ART PACK 6 - Available now!
+  category: Artwork (Digital)
+  theme: All
+  species: Unspecified / Any
+  gender: Any
+  keywords:
+  - meesh
+  - nsfw
+  - art
+  - pack
+  - adult
+  - boner
+  - touching
+  file_url: "//d.facdn.net/art/meesh/1456189705/1456189705.meesh_production5.png"
+  blob_entry_id: 1370198
+  old_files: []
+  num_favorites: 8
+  num_comments: 2
+  num_views: 904
+  posted_at:
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-02-28 00:26:18.966788000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-02-28 00:26:18.966788000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-01-27 23:26:36.605202000 Z
+    zone: *1
+    time: 2018-01-27 23:26:36.605202000 Z
+  in_gallery: true
+  in_folders: []
+  state: have_static
+  state_error:
+  folder_name: Main Gallery
+  gallery_page_logfile: listing_pages/meesh/1480115720-MainGallery_1.html
+  submission_page_logfile:
+  submission_page_log_entry_id:
+  description_id: 144
+  imgsearch_entry_id: 154

test/fixtures/legacy/fa/users.yml

@@ -0,0 +1,80 @@
+---
+id_1:
+  id: 1
+  name: Meesh
+  full_name: Meesh
+  artist_type: PrOn Artist
+  mood: optimistic
+  profile_html: "\n<b>Full
+    Name:</b> Meesh<br>\n<b>Artist
+    Type:</b> PrOn Artist<br>\n<b>Registered
+    since:</b> Dec 11th, 2005 11:28<br>\n<b>Current
+    mood:</b> optimistic<br>\n<b>Artist
+    Profile:</b><br>\n<div
+    class=\"bbcode bbcode_center\">Male | 28 | Housecat | Single | Straight</div>\n<br>\r\n<br>\r\n<a
+    href=\"/user/patreon\" class=\"iconusername\"><img src=\"//a.facdn.net/20170212/patreon.gif\"
+    align=\"middle\" title=\"patreon\" alt=\"patreon\"></a> I have a Patreon - Early
+    access for comics starts at $6/month! <a href=\"http://www.patreon.com/meesh\" title=\"http://www.patreon.com/meesh\"
+    class=\"auto_link\">www.patreon.com/meesh</a><br>\r\n<br>\r\n<span class=\"bbcode\"
+    style=\"color: #d92464;\"><strong class=\"bbcode bbcode_b\">BUSINESS INQUIRIES:
+    MEESHYMEESH@GMAIL.COM</strong></span><br>\r\n<a href=\"/user/oregonfurs\" class=\"iconusername\"><img
+    src=\"//a.facdn.net/20170212/oregonfurs.gif\" align=\"middle\" title=\"oregonfurs\"
+    alt=\"oregonfurs\">oregonfurs</a><br>\r\n<strong class=\"bbcode bbcode_b\">MY
+    STORE:</strong> <a href=\"http://www.meesh.pandachan.org\" title=\"http://www.meesh.pandachan.org\"
+    class=\"auto_link\">www.meesh.pandachan.org</a><br>\r\n<br>\r\n<span class=\"bbcode\"
+    style=\"color: #d92464;\"><strong class=\"bbcode bbcode_b\">TUMBLR:</strong></span><br>\r\n<a
+    href=\"http://www.meeshmeat.tumblr.com\" title=\"http://www.meeshmeat.tumblr.com\"
+    class=\"auto_link\">www.meeshmeat.tumblr.com</a><br>\r\n<br>\r\n<span class=\"bbcode\"
+    style=\"color: #d92464;\"><strong class=\"bbcode bbcode_b\">COMMISSIONS:</strong></span><strong
+    class=\"bbcode bbcode_b\">CLOSED.</strong> Email me at <a class=\"auto_link email\"
+    href=\"mailto:meeshymeesh@gmail.com\">meeshymeesh[at]gmail.com</a> to inquire about
+    commissions. Current turnaround time is approx one month.<br>\r\n<strong class=\"bbcode
+    bbcode_b\">Tier 1:</strong> <a href=\"https://www.furaffinity.net/view/17970865/\"
+    title=\"https://www.furaffinity.net/view/17970865/\" class=\"auto_link\">https://www.furaffinity.net/view/17970865/</a>
+    $600/character<br>\r\n<strong class=\"bbcode bbcode_b\">Tier 2:</strong> <a href=\"https://www.furaffinity.net/view/18356127/\"
+    title=\"https://www.furaffinity.net/view/18356127/\" class=\"auto_link\">https://www.furaffinity.net/view/18356127/</a>
+    $200/character<br>\r\n<strong class=\"bbcode bbcode_b\">Sketch:</strong> <a href=\"https://www.furaffinity.net/view/18329853/\"
+    title=\"https://www.furaffinity.net/view/18329853/\" class=\"auto_link\">https://www.furaffinity.net/view/18329853/</a>
+    $80/character<br>\r\n<br>\r\nMy gallery is a good example of what I will and will
+    not draw. <br>\r\n<br>\r\n<span class=\"bbcode\" style=\"color: #d92464;\"><strong
+    class=\"bbcode bbcode_b\">Terms of Service for commissions:</strong></span> I will
+    always have control over what I will and will not draw. I will not work towards
+    any illustration that is against the law, or is intended to be slanderous or hateful
+    towards any other individual or business. I have the right to change an estimated
+    date of completion depending on the situation. I have the right to refuse service
+    to anyone. Money must be paid in advance before work can begin on a project. You
+    have the right to ask for changes both in the finished product, and mid-process.
+    By hiring me, you understand and agree to these terms.<br>\r\n<br>\r\n<span class=\"bbcode\"
+    style=\"color: #d92464;\"><strong class=\"bbcode bbcode_b\">Refund policy:</strong></span>
+    Refunds will be granted in full only if work on a project has not yet begun. If
+    necessary, proof can be provided of the work done."
+  num_pageviews: 1715730
+  num_submissions: 1225
+  num_comments_recieved: 40123
+  num_comments_given: 17386
+  num_journals: 13
+  num_favorites: 893478
+  registered_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2005-12-11 11:28:00.000000000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2005-12-11 11:28:00.000000000 Z
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-02-28 00:26:18.692514000 Z
+    zone: *1
+    time: 2016-02-28 00:26:18.692514000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-10-01 11:16:55.673308000 Z
+    zone: *1
+    time: 2018-10-01 11:16:55.673308000 Z
+  url_name: meesh
+  scanned_gallery: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-10-01 11:16:55.671846000 Z
+    zone: *1
+    time: 2018-10-01 11:16:55.671846000 Z
+  scanned_page: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2017-02-12 23:13:56.517709000 Z
+    zone: *1
+    time: 2017-02-12 23:13:56.517709000 Z
+  user_page_logfile: user/meesh/1480115630-meesh.html
+  user_page_log_entry_id: 12175910

test/fixtures/legacy/http_log_entries.yml

@@ -0,0 +1,34 @@
+1766_id:
+  id: 1766
+  scheme: https
+  host: d.facdn.net
+  path: "/art/drakeergenthal/stories/1480188610/1480188610.drakeergenthal_gow_fanfic.txt"
+  query:
+  verb: get
+  status: 200
+  response_time: 192
+  content_type: text/plain
+  response_size: 6936
+  parent_log_entry_id:
+  blob_entry_id: 8663902
+  gzipped: false
+  requested_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-11-26 22:09:59.699304000 Z
+    zone: &1 !ruby/object:ActiveSupport::TimeZone
+      name: Etc/UTC
+    time: 2016-11-26 22:09:59.699304000 Z
+  created_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2016-11-26 22:09:59.753081000 Z
+    zone: *1
+    time: 2016-11-26 22:09:59.753081000 Z
+  updated_at: !ruby/object:ActiveSupport::TimeWithZone
+    utc: 2018-04-16 03:35:12.524953000 Z
+    zone: *1
+    time: 2018-04-16 03:35:12.524953000 Z
+  resp_body: 41F8DAF7772D11F80AFE56B742087A2D1AB372E08B69E1284BE4FEFEC2AD0C7F
+  imported_from_file:
+  req_headers_id: 3
+  res_headers_id: 575
+  diff_type: native
+  native_blob_entry_sha256: !binary |-
+    Qfja93ctEfgK/la3Qgh6LRqzcuCLaeEoS+T+/sKtDH8=

test/fixtures/legacy/s_config/blobs/static/4/1/f/8/41f8daf7772d11f80afe56b742087a2d1ab372e08b69e1284be4fefec2ad0c7f

@@ -0,0 +1,19 @@
+Locust vs. Nature
+
+Not much are known for what has happened to the rest of the Locust, during the war.  Some even faced a worse fate than dying from any weapon or from the Imulsion Countermeasure weapon.  The Lambent are the actual threat of all, if only the Locust would see it that way and join forces with the humans (aka ground walkers).  Let's just say what would happen to four unlucky Locust characters, while around the timeline of Gears of War 3.
+
+We have a Hunter Elite, on a mission to search for other members of the Locust that were apparently lost from the Jacinto incident.  Let's call him Chuck for this one.  The Locust don't have much for names around the low ranks.  The only names that were mentioned are RAAM, Skorge, and Karn.  As for Chuck, he has to venture above ground for his search.  But he wasn't alone, as he brings around a miner for connection to the lost ones.  We're gonna call the miner as Jules.  Chuck and Jules are wearing their proper Locust clothes to notify their ranks.  As much as they hate the humans living above ground, the two have to focus on searching for the lost Locust.  They stayed hidden from the inhabitants from certain cities.
+
+Chuck and Jules did come across some Lambent along the way, which are a nuisance to everyone.  Chuck has his Hammerburst with him to take some out, but Jules doesn't have any weapon on him.  But he did grab a Lancer from a dead COG soldier, and uses it against the Lambent.  Chuck was even surprised about Jules' capabilities.  As much as the Lambent are their enemy from the threat, they decide to hide from the humans and COG soldiers who arrived to provide reinforcements.  Jules keeps the Lancer with him, as it was his only thing that protected him.
+
+They soon arrive at a place called the Deadlands, and there are no signs of Lambent anywhere.  Let alone any Locust survivors to find.  Jules starts to have a feeling that they being watched.  Chuck notices a trap door around the middle of the arena.  For good measures, he shoots the door open and it only shows some ground underneath.  Like a pitfall.  While they avoided a sudden trap, they soon get jumped by their own Locust militia.  Only their outfits are much different compared to their own, when the regular Locust are well organized too.  These Locust are acting a bit feral of their way, and doesn't follow the ways of the Locust Queen.  They are savages compared to Chuck and Jules, and the two are also surrounded.  One of the savages is a grenadier, who soon falls into the trap while rushing towards Chuck and Jules.  We'll call him Zeke for that.  Despite putting up a good fight, the two were subdued by the Savage Locust.
+
+Chuck and Jules were disarmed and the savages prevents them from moving, as their leader shows up, who happens to be a Savage Kantus.  We'll call him Travis for no apparent reason.  Anyway, Travis was not amused about the outsider Locust putting up resistance against his own community.  As punishment, the two are thrown down where the trap door was to suffer endlessly.  The savage drone threw the two in, Travis decides to join them to make sure the outsiders will suffer to no end.  The savages waved goodbye to Travis, as he's gonna be joining them for eternity.  As for Chuck and Jules, they've reached the bottom of the fall, just to see Zeke with them.  They plan on getting out of this, only there's another reason why these characters are doomed here.  The pitfall is slowly sucking them under.  Zeke is already up to his knees in the sand, so he can't pull himself out.  Jules then notices that he can't lift up his feet, as his ankles are already under.  Chuck tries to climb out, as he quick to act from the situation.  While climbing, Travis falls on top of Chuck and they plummet down to Jules and Zeke.  Chuck opens his eyes, only to find himself up to his thighs within the sand.  Travis explains to them as this is quicksand, and it's impossible to escape.  He's willing to enjoy every minute of it, while the others struggle to get free.  Travis decides to take off some of his armor while he's sinking with the others, to reveal his red undergarment to them.
+
+Jules then yells for help, as the quicksand starts sucking up his thighs.  It doesn't help the dismay for Chuck or Zeke, as the living ground soon touches their groins.  Travis soon starts touching his crotch while laying on his knees.  The quicksand soon reaches Jules' crotch, and now three of them can barely move their legs.  It will soon include Travis as he's ready to unload his bladder.  The sinking Locust were taken by surprise, as the kantus would do something so very, very different!!!  Chuck, Jules and Zeke then realizes that they have to pee!  The two tried to resist the temptation, but Zeke points out that it's pointless to fight.  The savage grenadier soon let's it go, as his crotch is already suck under by the sands.  The piss was hard to ignore, and soon the buildup within Chuck's and Jules' became too great.  They unload their fill, as the quicksand begins to suck up their stomaches.  Travis was happy to watch his crotch sinking within the hungry sands.
+
+Chuck, and Jules were so deep in, the hunter even tried to pull himself out.  It didn't work, and now he can't even pull up his arms from the sands.  Jules uses up his remaining arm that was still up the surface, to activate his full helmet.  It didn't help, now that three of them are up to their shoulders and continues to sink.  Travis was thrilled to see their despair, even if the quicksand is reaching up to his chest.  Travis explains to them, that it's not the end for them.  Chuck believes that it is, as the sands begins to start sucking up his head down.  The kantus soon says they will not be forgotten, only going to sleep for a long time.  Zeke was soon sucked under, and soon claims Chuck too.  Jules did give out a question, saying if they'll be alright.  Travis says yes, before the miner was sucked under.
+
+As for Travis now up to his armpits within his quicksand, he begins playing around until it was time to enchant their long sleep within the sands.  Before he was submerged along with the others, Travis conjures his spell.  They disappeared from the earth, but somehow they will survive everything.
+
+For over twenty-five years, Travis will revive himself as well as the others since the quicksand has dried up.  Chuck, Jules and Zeke wake up to find themselves only up to their armpits in just dirt.  They dug themselves out, and notices Travis up to his neck within the solid sands.  Since he preserved them for all this time, the remaining Locust dug him up and climbed out.  Upon getting out, they soon noticed that everything has changed!  Travis soon wakes up, and was willing to help find a way to survive their new world.  They are also unaware that the essence of the Queen still exist somewhere.  Perhaps they are lucky than they thought.

test/models/blob_entry_test.rb

@@ -9,7 +9,9 @@ class BlobEntryTest < ActiveSupport::TestCase
 
   test "can create from a legacy blob entry" do
     legacy_be = ::Legacy::BlobEntry.find(1370198)
+    assert File.exist?(legacy_be.file_path), legacy_be.file_path
     new_be = ::BlobEntry.find_or_build_from_legacy(legacy_be)
+    refute_nil new_be
     assert new_be.valid?, new_be.errors.full_messages
     assert new_be.new_record?
     assert_equal "5ed3a0400ac50f721123c7a8c638da8b19bf563f8e880f9abb36dcb38395bc82", HexUtil.bin2hex(new_be.sha256)