dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

port over some files from test to spec

Browse Source
This commit is contained in:
Dylan Knutson
2023-04-01 14:38:09 +09:00
parent 8d85f7ebe1
commit 369c79f8df
6 changed files with 83 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
test/lib/domain/fa/parser/legacy_page_test.rb → spec/lib/domain/fa/parser/legacy_page_spec.rb
View File

@@ -1,9 +1,7 @@
# frozen_string_literal: true
require "rails_helper"
require "test_helper"
class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
def test_sees_not_logged_in
describe Domain::Fa::Parser::Page do
it "sees_not_logged_in" do
parser = get_parser(
"index_not_logged_in.html",
require_logged_in: false,
@@ -12,26 +10,26 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_nil parser.logged_in_user
end
def test_throws_if_not_logged_in_by_default
it "throws_if_not_logged_in_by_default" do
assert_raises(Domain::Fa::Parser::NotLoggedInError) do
get_parser("index_not_logged_in.html")
end
end
def test_submission_not_found_does_not_throw
it "submission_not_found_does_not_throw" do
parser = get_parser("submission_not_found_in_db.html")
refute parser.logged_in?
assert parser.submission_not_found?
end
def test_sees_logged_in_correctly
it "sees_logged_in_correctly" do
parser = get_parser "index_logged_in_zzreg.html"
assert parser.logged_in?
assert_equal parser.logged_in_user, "zzreg"
refute parser.submission_not_found?
end
def test_user_page_is_correct
it "user_page_is_correct" do
parser = get_parser "user_page_miles_df.html"
assert parser.probably_user_page?
up = parser.user_page
@@ -51,7 +49,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "Jan 12th, 2006 07:52", up.registered_since
end
def test_gallery_is_correct
it "gallery_is_correct" do
parser = get_parser "gallery_page_miles_df.html"
assert parser.probably_listings_page?
@@ -71,7 +69,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal 0, parser_empty.submissions_parsed.length
end
def test_new_gallery_is_correct
it "new_gallery_is_correct" do
parser = get_parser "gallery_page_no_folders_fa_update_1_bradydalton.html"
assert parser.probably_listings_page?
@@ -88,7 +86,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "//t.facdn.net/19067130@300-1455225217.jpg", first.thumb_path
end
def test_new_gallery_is_correct_with_folders
it "new_gallery_is_correct_with_folders" do
parser = get_parser "gallery_page_with_folders_fa_update_1_feretta.html"
assert parser.probably_listings_page?
@@ -111,7 +109,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal({ href: "/gallery/feretta/folder/229520/Marketing", title: "Marketing" }, folders.last)
end
def test_browse_page_update_1
it "browse_page_update_1" do
parser = get_parser "browse_page_fa_update_1.html"
assert parser.probably_listings_page?
assert !parser.probably_submission?
@@ -129,7 +127,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "//t.facdn.net/21840759@200-1480303217.jpg", first.thumb_path
end
def test_gallery_page_is_correct_fa_update_1
it "gallery_page_is_correct_fa_update_1" do
parser = get_parser "gallerey_draconicon_fa_update_1.html"
assert parser.probably_listings_page?
assert !parser.probably_submission?
@@ -146,7 +144,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "//t.facdn.net/21849303@120-1480382449.jpg", first.thumb_path
end
def test_submission_is_correct
it "submission_is_correct" do
parser = get_parser "submission_19143193_miles_df.html"
assert parser.probably_submission?
assert_equal Domain::Fa::Parser::Page::VERSION_0, parser.page_version
@@ -185,7 +183,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal %w[kobold frog hunting male frog mighty hunter], sub.keywords_array
end
def test_updated_fa_1_submission
it "updated_fa_1_submission" do
parser = get_parser("submission_21773129_new_fa_feretta.html")
assert parser.probably_submission?
assert_equal Domain::Fa::Parser::Page::VERSION_1, parser.page_version
@@ -209,7 +207,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal %w[space_vixen vix rocket space], sub.keywords_array
end
def test_old_submission_is_parsed
it "old_submission_is_parsed" do
parser = get_parser("submission_53013_uaykan.html")
assert_equal Domain::Fa::Parser::Page::VERSION_0, parser.page_version
sub = parser.submission
@@ -218,7 +216,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "uaykan", sub.artist
end
def test_folders_work
it "folders_work" do
parser = get_parser "gallery_page_salkitten.html"
folders = parser.submission_folders
assert_equal 11, folders.length
@@ -227,19 +225,19 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal({ href: "/gallery/salkitten/folder/93190/Animations", title: "Animations" }, folders.last)
end
def test_with_no_folders
it "with_no_folders" do
parser = get_parser "gallery_page_spearfrost.html"
assert_equal 0, parser.submission_folders.length
end
def test_works_with_no_myform
it "works_with_no_myform" do
parser = get_parser("submission_19225668_no_comments.html")
sub = parser.submission
assert_equal 19_225_668, sub.id
end
def test_text_only_submission
it "text_only_submission" do
parser = get_parser("submission_10327196_text_only.html")
assert parser.probably_submission?
@@ -250,7 +248,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "//d.facdn.net/art/icyfoxy/stories/1365480658/1365480658.icyfoxy_sonya_the_wereyena.txt", sub.full_res_img
end
def test_poetry_submission
it "poetry_submission" do
parser = get_parser("submission_5971150_no_image.html")
assert parser.probably_submission?
@@ -261,7 +259,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "//d.facdn.net/art/wolfied/poetry/1308624615/1308624615.wolfied_shy_child.txt", sub.full_res_img
end
def test_have_file_url
it "have_file_url" do
parser = get_parser("submission_7762536.html")
sub = parser.submission
@@ -269,7 +267,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "//d.facdn.net/art/musicofluie/1334184376/1334184376.musicofluie_dwragon.png", sub.full_res_img
end
def test_flash_submission
it "flash_submission" do
parser = get_parser("submission_21956745_flash.html")
assert parser.probably_submission?
@@ -282,7 +280,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
assert_equal "Flash", sub.category
end
def test_id_is_correct_20340491
it "id_is_correct_20340491" do
parser = get_parser("submission_20340491_id_changing.html")
assert parser.probably_submission?
sub = parser.submission
@@ -293,7 +291,7 @@ class Domain::Fa::Parser::LegacyPageTest < ActiveSupport::TestCase
def get_parser(file, require_logged_in: true)
path = File.join("domain/fa/parser/legacy", file)
contents = read_fixture_file(path) || raise("Couldn't open #{path}")
contents = SpecUtil.read_fixture_file(path) || raise("Couldn't open #{path}")
Domain::Fa::Parser::Page.new(contents, require_logged_in: require_logged_in)
end
end

38
test/lib/domain/fa/parser/redux_page_test.rb → spec/lib/domain/fa/parser/redux_page_spec.rb
View File

@@ -1,32 +1,32 @@
require "test_helper"
require "rails_helper"
class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
def test_sees_not_logged_in
describe Domain::Fa::Parser::Page do
it "sees_not_logged_in" do
parser = get_parser("login_page.html", require_logged_in: false)
refute parser.logged_in?
assert_nil parser.logged_in_user
end
def test_throws_if_not_logged_in_by_default
it "throws_if_not_logged_in_by_default" do
assert_raises(Domain::Fa::Parser::NotLoggedInError) do
get_parser("login_page.html")
end
end
def test_submission_not_found_does_not_throw
it "submission_not_found_does_not_throw" do
parser = get_parser("submission_not_found_in_db.html")
refute parser.logged_in?
assert parser.submission_not_found?
end
def test_sees_logged_in_correctly
it "sees_logged_in_correctly" do
parser = get_parser("index_logged_in_zzreg.html")
assert parser.logged_in?
assert_equal parser.logged_in_user, "zzreg"
refute parser.submission_not_found?
end
def test_user_page_is_correct
it "user_page_is_correct" do
parser = get_parser "user_page_miles_df.html"
assert parser.logged_in?
assert_page_type parser, :probably_user_page?
@@ -45,7 +45,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal DateTime.new(2006, 1, 12, 7, 52), up.registered_since
end
def test_gallery_is_correct
it "gallery_is_correct" do
parser = get_parser "gallery_page_miles_df.html"
assert parser.logged_in?
assert_page_type parser, :probably_listings_page?
@@ -68,7 +68,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal 0, parser_empty.submissions_parsed.length
end
def test_gallery_is_correct_with_folders
it "gallery_is_correct_with_folders" do
parser = get_parser "gallery_page_with_folders_feretta.html"
assert_page_type parser, :probably_listings_page?
@@ -99,7 +99,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal({ href: "/gallery/feretta/folder/820310/3D", title: "3D" }, folders[13])
end
def test_browse_page
it "browse_page" do
# https://www.furaffinity.net/browse/
parser = get_parser "browse_page.html"
@@ -118,7 +118,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal "//t.furaffinity.net/51067352@400-1676764142.jpg", first.thumb_path
end
def test_adult_submission
it "adult_submission" do
parser = get_parser "submission_51067352_adult.html"
assert_page_type parser, :probably_submission?
sub = parser.submission
@@ -140,7 +140,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert sub.description_html =~ /belongs to Nintendo/
end
def test_submission_is_correct
it "submission_is_correct" do
parser = get_parser "submission_51067333_blauhaher.html"
assert_page_type parser, :probably_submission?
@@ -165,7 +165,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert sub.description_html =~ /really adore at the way this/
end
def test_submission_with_tags
it "submission_with_tags" do
parser = get_parser "submission_19190013_korichi.html"
assert_page_type parser, :probably_submission?
@@ -183,7 +183,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal %w[kobold frog hunting male frog mighty hunter], sub.keywords_array
end
def test_text_only_submission
it "text_only_submission" do
parser = get_parser("submission_10327196_icyfoxy_text_only.html")
assert_page_type parser, :probably_submission?
@@ -208,7 +208,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal DateTime.new(2013, 4, 9, 0, 10), sub.posted_date
end
def test_submission_id_is_correct_20340491
it "submission_id_is_correct_20340491" do
parser = get_parser("submission_20340491.html")
assert_page_type parser, :probably_submission?
@@ -218,7 +218,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal :general, sub.rating
end
def test_flash_submission
it "flash_submission" do
parser = get_parser("submission_21956745_flash.html")
assert_page_type parser, :probably_submission?
@@ -233,7 +233,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal "//d.furaffinity.net/download/art/tasuric/1481384060/1481384060.tasuric_tasflash_camnom.swf", sub.full_res_img
end
def test_submission_id_16750893_category
it "submission_id_16750893_category" do
parser = get_parser("submission_16750893.html")
assert_page_type parser, :probably_submission?
@@ -250,7 +250,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal "//d.furaffinity.net/art/soulhunt3r/1433541729/1433541729.soulhunt3r_image.jpg", sub.full_res_img
end
def test_watchlist_zzreg
it "watchlist_zzreg" do
parser = get_parser("watchlist_zzreg.html", require_logged_in: false)
user_list = parser.user_list
assert_equal 10, user_list.length
@@ -263,7 +263,7 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
def get_parser(file, require_logged_in: true)
path = File.join("domain/fa/parser/redux", file)
contents = read_fixture_file(path) || raise("Couldn't open #{path}")
contents = SpecUtil.read_fixture_file(path) || raise("Couldn't open #{path}")
page = Domain::Fa::Parser::Page.new(contents, require_logged_in: require_logged_in)
assert_equal Domain::Fa::Parser::Page::VERSION_2, page.page_version, "page version mismatch for file #{file}"
page

10
test/lib/fork_future_test.rb → spec/lib/fork_future_spec.rb
View File

@@ -1,7 +1,7 @@
require "test_helper"
require "rails_helper"
class ForkFutureTest < ActiveSupport::TestCase
def test_join_works
describe ForkFuture do
it "join_works" do
captured = "foo"
future1 = ForkFuture.new do
captured + "bar"
@@ -16,7 +16,7 @@ class ForkFutureTest < ActiveSupport::TestCase
assert 0.0 < future2.duration
end
def test_pmap_works
it "pmap_works" do
captured = 2
mapped = ForkFuture.parallel_map(2, 1..10) do |i|
captured * i
@@ -24,7 +24,7 @@ class ForkFutureTest < ActiveSupport::TestCase
assert_equal (1..10).map { |i| i * 2 }, mapped
end
def test_nested_works
it "nested_works" do
captured = 2
mapped = ForkFuture.parallel_map(2, 1..10) do |i|
ForkFuture.new do

35
spec/lib/legacy_import/adaptive_cache_spec.rb Normal file
View File

@@ -0,0 +1,35 @@
require "rails_helper"
describe LegacyImport::AdaptiveCache do
it "works" do
cache = LegacyImport::AdaptiveCache.new(4, 1.0, 0.0)
refute cache.at_capacity?
expect(cache.candidates).to eq([])
# works even when the candidate doesn't exist
cache.reward 1
expect(cache.candidates).to eq([])
cache.insert :a, "a"
cache.insert :b, "b"
cache.insert :c, "c"
cache.insert :d, "d"
assert cache.at_capacity?
expect(cache.candidates).to eq(["d", "c", "b", "a"])
5.times { cache.reward :a }
3.times { cache.reward :b }
1.times { cache.reward :c }
expect(cache.candidates).to eq(["a", "b", "c", "d"])
expect(cache.scores).to eq([5.0, 3.0, 1.0, 0.0])
3.times { cache.reward :c } # 1 => 4
expect(cache.candidates).to eq(["a", "c", "b", "d"])
expect(cache.scores).to eq([5.0, 4.0, 3.0, 0.0])
# new 'e' should bump off 'd' which has a 0 score
cache.insert :e, "e"
expect(cache.candidates).to eq(["a", "c", "b", "e"])
expect(cache.scores).to eq([5.0, 4.0, 3.0, 0.0])
end
end

21
spec/spec_util.rb
View File

@@ -130,10 +130,6 @@ class SpecUtil
end
def self.enqueued_jobs(job_klass = nil)
# unless ::ActiveJob::QueueAdapters::TestAdapter === ::ActiveJob::Base.queue_adapter
# raise StandardError, "set `ActiveJob::Base.queue_adapter = :test`"
# end
GoodJob::Job.order(created_at: :asc).all.map do |job|
{
job: job.job_class.constantize,
@@ -149,26 +145,9 @@ class SpecUtil
true
end
end
# jobs = ActiveJob::Base.queue_adapter.enqueued_jobs
# jobs.filter do |job|
# if job_klass
# job[:job] == job_klass
# else
# true
# end
# end.map do |job|
# job.slice(:job, :queue, :priority).merge({
# args: ::ActiveJob::Arguments.deserialize(job[:args]),
# })
# end
end
def self.shift_jobs(job_klass, by = 1)
# jobs = ActiveJob::Base.queue_adapter.enqueued_jobs
# by.times do
# index = jobs.index { |job| job[:job] == job_klass }
# jobs.delete_at(index)
# end
by.times do
job = enqueued_jobs.find { |job| job[:job] == job_klass }
job[:good_job].destroy if job

35
test/lib/legacy_import/adaptive_cache_test.rb
View File

@@ -1,35 +0,0 @@
require "test_helper"
class AdaptiveCacheTest < ActiveSupport::TestCase
def test_works
cache = LegacyImport::AdaptiveCache.new(4, 1.0, 0.0)
refute cache.at_capacity?
assert_equal [], cache.candidates
# works even when the candidate doesn't exist
cache.reward 1
assert_equal [], cache.candidates
cache.insert :a, "a"
cache.insert :b, "b"
cache.insert :c, "c"
cache.insert :d, "d"
assert cache.at_capacity?
assert_equal ["d", "c", "b", "a"], cache.candidates
5.times { cache.reward :a }
3.times { cache.reward :b }
1.times { cache.reward :c }
assert_equal ["a", "b", "c", "d"], cache.candidates
assert_equal [5.0, 3.0, 1.0, 0.0], cache.scores
3.times { cache.reward :c } # 1 => 4
assert_equal ["a", "c", "b", "d"], cache.candidates
assert_equal [5.0, 4.0, 3.0, 0.0], cache.scores
# new 'e' should bump off 'd' which has a 0 score
cache.insert :e, "e"
assert_equal ["a", "c", "b", "e"], cache.candidates
assert_equal [5.0, 4.0, 3.0, 0.0], cache.scores
end
end