dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

user fav job + refactors

Browse Source
This commit is contained in:
Dylan Knutson
2023-05-18 14:40:56 -07:00
parent c7a2a3481a
commit 5b508060ff
25 changed files with 3147 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
app/jobs/domain/fa/job/base.rb
View File

@@ -7,26 +7,52 @@ class Domain::Fa::Job::Base < Scraper::JobBase
protected
def find_or_create_user_from_args(...)
user = find_or_build_user_from_args(...)
if user.new_record?
user.save!
def init_from_args!(args, require_user_exists: false)
@force_scan = !!args[:force_scan]
@caused_by_entry = args[:caused_by_entry]
@user = find_or_build_user_from_args(args, caused_by_entry: @caused_by_entry)
logger.prefix =
"[user #{(@user.url_name || @user.name).bold} / #{@user.state.bold}]"
if @user.new_record?
if require_user_exists
fatal_error("user must already exist")
else
@user.save!
@created_user = true
end
end
user
end
def find_or_build_user_from_args(args, caused_by_entry: nil)
find_user_from_args(args) || begin
url_name = Domain::Fa::User.name_to_url_name(args[:url_name])
user = Domain::Fa::User.new
user.url_name = url_name
user.name = url_name
user.state_detail ||= {}
user.state_detail["first_seen_entry"] = caused_by_entry.id if caused_by_entry
user
end
end
def find_user_from_args(args)
args[:user] || begin
fatal_error("arg 'url_name' is required if arg 'user' is nil") if args[:url_name].blank?
url_name = Domain::Fa::User.name_to_url_name(args[:url_name])
Domain::Fa::User.find_or_initialize_by(url_name: url_name) do |user|
user.state_detail ||= {}
user.state_detail["first_seen_entry"] = caused_by_entry.id if caused_by_entry
user.name ||= url_name
end
Domain::Fa::User.find_by(url_name: url_name)
end
end
def user_due_for_scan?(scan_type)
unless @user.scan_due?(scan_type)
logger.warn("scanned #{@user.scanned_ago_in_words(scan_type).bold} - skipping")
return false
end
return true
end
ListingsPageScanStats = Struct.new(
:new_seen,
:total_seen,

153
app/jobs/domain/fa/job/favs_job.rb Normal file
View File

@@ -0,0 +1,153 @@
class Domain::Fa::Job::FavsJob < Domain::Fa::Job::Base
include HasMeasureDuration
include HasBulkEnqueueJobs
USERS_PER_FULL_PAGE = Rails.env.test? ? 9 : 190
queue_as :fa_user_favs
ignore_signature_args :caused_by_entry
def perform(args)
@caused_by_entry = args[:caused_by_entry]
@first_job_entry = nil
@force_scan = !!args[:force_scan]
@user = find_user_from_args(args) || begin
Domain::Fa::Job::UserPageJob.perform_later({
url_name: args[:url_name],
caused_by_entry: best_caused_by_entry,
})
raise("user does not exist: #{args}")
end
logger.prefix = "[#{(@user.url_name || @user.name).bold} / #{@user.state.bold}]"
return unless user_due_for_scan?(:favs)
@page_number = 0
@page_id = nil
@total_items_seen = 0
@seen_post_ids = Set.new
while true
break if scan_page == :break
# bail out at 100,000 users
break if @page_number > 500
@page_number += 1
end
to_add = nil
to_remove = nil
measure(proc { |jobs|
"add #{to_add.size.to_s.bold} favs, " +
"remove #{to_remove.size.to_s.bold} favs"
}) do
existing_ids = Set.new(@user.fav_post_joins.pluck(:post_id))
to_remove = existing_ids - @seen_post_ids
to_add = @seen_post_ids - existing_ids
end
measure(proc {
"updated favs list to #{@user.fav_post_joins.count.to_s.bold} posts"
}) do
ReduxApplicationRecord.transaction do
@user.fav_post_joins.where(post_id: to_remove).delete_all
@user.fav_post_joins.insert_all!(to_add.map do |id|
{ post_id: id }
end) unless to_add.empty?
@user.scanned_favs_at = Time.now
@user.save!
end
end
if @created_user
logger.info("user was new record, enqueue page scan job")
Domain::Fa::Job::UserPageJob.perform_later({
user: @user,
caused_by_entry: best_caused_by_entry,
})
end
end
private
def scan_page
ret = nil
url = if @page_id
"https://www.furaffinity.net/favorites/#{@user.url_name}/#{@page_id}/next"
else
"https://www.furaffinity.net/favorites/#{@user.url_name}/"
end
response = http_client.get(url, caused_by_entry: best_caused_by_entry)
@first_job_entry ||= response.log_entry
if response.status_code != 200
fatal_error(
"http #{response.status_code.to_s.red.bold}, " +
"log entry #{response.log_entry.id.to_s.bold}"
)
end
page = Domain::Fa::Parser::Page.new(response.body)
raise("not a listings page") unless page.probably_listings_page?
submissions = page.submissions_parsed
@page_id = page.favorites_next_button_id
ret = :break if @page_id.nil?
@total_items_seen += submissions.length
posts_to_create_hashes = []
followed_user_ids = measure(proc {
"page #{@page_number.to_s.bold} - " +
"#{submissions.length.to_s.bold} posts on page, " +
"created #{posts_to_create_hashes.size.to_s.bold}"
}) do
existing_fa_id_to_post_id = Domain::Fa::Post.where(
fa_id: submissions.map(&:id),
).pluck(:fa_id, :id).to_h
posts_to_create_hashes = submissions.reject do |submission|
existing_fa_id_to_post_id[submission.id]
end.map do |submission|
Domain::Fa::Post.hash_from_submission_parser_helper(
submission,
first_seen_log_entry: response.log_entry,
)
end
created_post_ids = []
created_post_ids = Domain::Fa::Post.insert_all!(
posts_to_create_hashes,
returning: %i[id fa_id],
).map do |row|
row["id"]
end unless posts_to_create_hashes.empty?
enqueue_new_post_scan_jobs(posts_to_create_hashes.map do |hash|
hash[:fa_id]
end)
created_post_ids.each do |id|
@seen_post_ids.add(id)
end
existing_fa_id_to_post_id.values.each do |id|
@seen_post_ids.add(id)
end
end
ret
end
def enqueue_new_post_scan_jobs(fa_ids)
bulk_enqueue_jobs do
fa_ids.each do |fa_id|
Domain::Fa::Job::ScanPostJob.perform_later({
fa_id: fa_id,
caused_by_entry: best_caused_by_entry,
})
end
end
end
def best_caused_by_entry
@first_job_entry || @caused_by_entry
end
end

11
app/jobs/domain/fa/job/user_follows_job.rb
View File

@@ -11,15 +11,8 @@ class Domain::Fa::Job::UserFollowsJob < Domain::Fa::Job::Base
ignore_signature_args :caused_by_entry
def perform(args)
@caused_by_entry = args[:caused_by_entry]
init_from_args!(args)
@first_job_entry = nil
@force_scan = !!args[:force_scan]
@user = find_or_build_user_from_args(args, caused_by_entry: best_caused_by_entry)
is_new_record = @user.new_record?
@user.save!
logger.prefix = "[#{(@user.url_name || @user.name).bold} / #{@user.state.bold}]"
if !@user.due_for_follows_scan? && !@force_scan
logger.warn("scanned #{time_ago_in_words(@user.scanned_follows_at)}, skipping")
return
@@ -61,7 +54,7 @@ class Domain::Fa::Job::UserFollowsJob < Domain::Fa::Job::Base
end
end
if is_new_record
if @created_user
logger.info("user was new record, enqueue page scan job")
Domain::Fa::Job::UserPageJob.perform_later({
user: @user,

5
app/jobs/domain/fa/job/user_gallery_job.rb
View File

@@ -5,10 +5,7 @@ class Domain::Fa::Job::UserGalleryJob < Domain::Fa::Job::Base
MAX_PAGE_NUMBER = 350
def perform(args)
@force_scan = !!args[:force_scan]
@caused_by_entry = args[:caused_by_entry]
@user = find_or_create_user_from_args(args, caused_by_entry: @caused_by_entry)
logger.prefix = "[user #{(@user.url_name || @user.name).bold} / #{@user.state.bold}]"
init_from_args!(args)
if @user.state != "ok" && @user.scanned_gallery_at
logger.warn("state == #{@user.state} and already scanned, skipping")

5
app/jobs/domain/fa/job/user_page_job.rb
View File

@@ -3,10 +3,7 @@ class Domain::Fa::Job::UserPageJob < Domain::Fa::Job::Base
ignore_signature_args :caused_by_entry
def perform(args)
@caused_by_entry = args[:caused_by_entry]
@user = find_or_create_user_from_args(args, caused_by_entry: @caused_by_entry)
@force_scan = !!args[:force_scan]
logger.prefix = "[#{(@user.url_name || @user.name).bold} / #{@user.state.bold}]"
init_from_args!(args)
# buggy (sentinal) user
return if @user.id == 117552 && @user.url_name == "click here"

32
app/lib/domain/fa/parser/page.rb
View File

@@ -66,6 +66,16 @@ class Domain::Fa::Parser::Page < Domain::Fa::Parser::Base
end
end
def favorites_next_button_id
button = @page.css(".gallery-section .pagination a.button.right").first
if button
href = button["href"]
match = /\/favorites\/.+\/(\d+)\/next\/?/.match(href)
raise("invalid favs button uri #{href}") unless match
match[1].to_i
end
end
def submission_folders
@submission_folders ||= @page.css(".folder-list a.dotted").map do |folder_link|
{ href: folder_link["href"], title: folder_link.text }
@@ -77,14 +87,16 @@ class Domain::Fa::Parser::Page < Domain::Fa::Parser::Base
when VERSION_0 then @page.css(".t-image")
when VERSION_1 then @page.css(".submission-list > .gallery > figure")
when VERSION_2
# user gallery pages are under .submission-list
elem = @page.css(".submission-list > .gallery > figure")
if elem.empty?
# /browse/ page is under #gallery-browse
@page.css("#gallery-browse > figure")
else
elem
end
[
# user gallery pages
".submission-list > .gallery > figure",
# browse list
"#gallery-browse > figure",
# favorites list
"#gallery-favorites > figure",
].lazy.map do |css|
@page.css(css)
end.reject(&:empty?).first || []
else unimplemented_version!
end
end
@@ -113,7 +125,9 @@ class Domain::Fa::Parser::Page < Domain::Fa::Parser::Base
when VERSION_0, VERSION_1
@page.css("center.flow").first || @page.css("section.submission-list") ? true : false
when VERSION_2
(@page.css(".submission-list").first || @page.css("#gallery-browse").first) ? true : false
(@page.css(".submission-list").first ||
@page.css("#gallery-browse").first ||
@page.css("#gallery-favorites").first) ? true : false
else unimplemented_version!
end
end

11
app/lib/domain/fa/user_enqueuer.rb
View File

@@ -31,6 +31,10 @@ class Domain::Fa::UserEnqueuer
rows.each do |user|
types = []
if user.state == "ok"
if user.due_for_favs_scan?
Domain::Fa::Job::FavsJob.perform_later({ user: user })
types << "favs"
end
if user.due_for_page_scan?
Domain::Fa::Job::UserPageJob.perform_later({ user: user })
types << "page"
@@ -73,11 +77,12 @@ class Domain::Fa::UserEnqueuer
end
[
"fa_user_follows",
"fa_user_page",
"fa_user_gallery",
"fa_post",
"fa_user_avatar",
"fa_user_favs",
"fa_user_follows",
"fa_user_gallery",
"fa_user_page",
"static_file",
].map do |queue_name|
count_failed_in_queue(queue_name)

8
app/models/domain/fa/fav.rb Normal file
View File

@@ -0,0 +1,8 @@
class Domain::Fa::Fav < ReduxApplicationRecord
self.table_name = "domain_fa_favs"
belongs_to :user,
class_name: "::Domain::Fa::User"
belongs_to :post,
class_name: "::Domain::Fa::Post"
end

27
app/models/domain/fa/post.rb
View File

@@ -38,6 +38,15 @@ class Domain::Fa::Post < ReduxApplicationRecord
optional: :true,
autosave: true
has_many :fav_post_joins,
class_name: "::Domain::Fa::Fav",
inverse_of: :post
has_many :faved_by,
class_name: "::Domain::Fa::User",
through: :fav_post_joins,
source: :user
def file_uri
Addressable::URI.parse(self.file_url_str) if self.file_url_str
end
@@ -221,4 +230,22 @@ class Domain::Fa::Post < ReduxApplicationRecord
nil
end
end
def self.hash_from_submission_parser_helper(submission, first_seen_log_entry: nil)
creator = Domain::Fa::User.find_or_create_by({
url_name: submission.artist_url_name,
}) do |user|
user.name = submission.artist
end
{
fa_id: submission.id,
creator_id: creator.id,
title: submission.title,
state_detail: {
"first_seen_entry" => first_seen_log_entry&.id,
"thumbnail_url_str" => submission.thumb_path,
},
}
end
end

58
app/models/domain/fa/user.rb
View File

@@ -48,6 +48,15 @@ class Domain::Fa::User < ReduxApplicationRecord
through: :followed_joins,
source: :follower
has_many :fav_post_joins,
class_name: "::Domain::Fa::Fav",
inverse_of: :user
has_many :fav_posts,
class_name: "::Domain::Fa::Post",
through: :fav_post_joins,
source: :post
# FA `name` can be up to 30 chars long,
# `url_name` can be longer.
validates_presence_of(:name, :url_name)
@@ -94,6 +103,39 @@ class Domain::Fa::User < ReduxApplicationRecord
throw :abort if posts.any?
end
SCAN_TYPES = {
:page => 1.month,
:gallery => 1.year,
:follows => 1.month,
:favs => 1.month,
}
SCAN_TYPES.keys.each do |scan_type|
define_method(:"due_for_#{scan_type}_scan?") do
scan_due?(scan_type)
end
define_method(:"time_ago_for_#{scan_type}_scan") do
scanned_ago_in_words(scan_type)
end
end
DATE_HELPER = Class.new.extend(ActionView::Helpers::DateHelper)
def scanned_ago_in_words(scan_type)
time = self.send(:"scanned_#{scan_type}_at")
if time.nil?
"never"
else
DATE_HELPER.time_ago_in_words(time) + " ago"
end
end
def scan_due?(scan_type)
duration = SCAN_TYPES[scan_type] || raise("invalid scan type '#{scan_type}'")
timestamp = self.send(:"scanned_#{scan_type}_at")
timestamp.nil? || timestamp < duration.ago
end
def take_posts_from(other_user)
return if other_user == self
other_posts = other_user.posts
@@ -108,18 +150,6 @@ class Domain::Fa::User < ReduxApplicationRecord
end
end
def due_for_page_scan?
timestamp_due?(scanned_page_at, 1.month)
end
def due_for_gallery_scan?
timestamp_due?(scanned_gallery_at, 1.year)
end
def due_for_follows_scan?
timestamp_due?(scanned_follows_at, 1.month)
end
def self.find_or_build_from_legacy(legacy_user)
existing = find_by(url_name: legacy_user.url_name)
return existing if existing
@@ -240,8 +270,4 @@ class Domain::Fa::User < ReduxApplicationRecord
joins(:disco).
merge(disco_query.reselect(:user_id))
end
def timestamp_due?(timestamp, duration)
timestamp.nil? || timestamp < duration.ago
end
end

12
db/redux_migrate/20230503042308_create_domain_fa_favs.rb Normal file
View File

@@ -0,0 +1,12 @@
class CreateDomainFaFavs < ActiveRecord::Migration[7.0]
def change
create_table :domain_fa_favs do |t|
t.references :user, null: false
t.references :post, null: false
end
add_column :domain_fa_users, :scanned_favs_at, :datetime
add_foreign_key :domain_fa_favs, :domain_fa_users, column: :user_id
add_foreign_key :domain_fa_favs, :domain_fa_posts, column: :post_id
end
end

12
db/schema.rb generated
View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 2023_04_07_162751) do
ActiveRecord::Schema[7.0].define(version: 2023_05_03_042308) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_stat_statements"
enable_extension "pg_trgm"
@@ -104,6 +104,13 @@ ActiveRecord::Schema[7.0].define(version: 2023_04_07_162751) do
t.index ["name"], name: "index_domain_e621_tags_on_name", unique: true
end
create_table "domain_fa_favs", force: :cascade do |t|
t.bigint "user_id", null: false
t.bigint "post_id", null: false
t.index ["post_id"], name: "index_domain_fa_favs_on_post_id"
t.index ["user_id"], name: "index_domain_fa_favs_on_user_id"
end
create_table "domain_fa_follows", force: :cascade do |t|
t.bigint "follower_id", null: false
t.bigint "followed_id", null: false
@@ -192,6 +199,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_04_07_162751) do
t.integer "state"
t.jsonb "state_detail"
t.datetime "scanned_follows_at"
t.datetime "scanned_favs_at"
t.index ["name"], name: "index_domain_fa_users_on_name", unique: true
t.index ["url_name"], name: "index_domain_fa_users_on_url_name", unique: true
end
@@ -371,6 +379,8 @@ ActiveRecord::Schema[7.0].define(version: 2023_04_07_162751) do
add_foreign_key "blob_entries", "blob_entries", column: "base_sha256", primary_key: "sha256"
add_foreign_key "domain_e621_post_versions", "domain_e621_posts", column: "item_id"
add_foreign_key "domain_fa_favs", "domain_fa_posts", column: "post_id"
add_foreign_key "domain_fa_favs", "domain_fa_users", column: "user_id"
add_foreign_key "domain_fa_follows", "domain_fa_users", column: "followed_id"
add_foreign_key "domain_fa_follows", "domain_fa_users", column: "follower_id"
add_foreign_key "domain_fa_posts", "domain_fa_users", column: "creator_id"

2
spec/helpers/debug_helpers.rb
View File

@@ -1,4 +1,5 @@
module DebugHelpers
# add `debug_sql: true` to tags
def debug_sql
logger = ActiveRecord::Base.logger
ActiveRecord::Base.logger = Logger.new($stdout)
@@ -7,6 +8,7 @@ module DebugHelpers
ActiveRecord::Base.logger = logger
end
# add `quiet: false` to show ColorLogger output
def quiet_color_logger(&block)
ColorLogger.quiet(&block)
end

22
spec/helpers/twitter_helpers.rb → spec/helpers/spec_helpers.rb
View File

@@ -1,4 +1,24 @@
module TwitterHelpers
module SpecHelpers
# TODO - migrate all the calls to perform_now to use this
def perform_now(params, should_raise: false)
ret = described_class.perform_now(params)
case should_raise
when false
expect(ret).to_not be_a(Exception), proc {
"!> " + ret.message + "\n" + ret.backtrace[0..10].join("\n")
}
when Exception
expect(ret).to be_a(should_raise)
when String, Regexp
expect(ret.message).to match(should_raise)
else
expect(ret).to be_a(Exception)
end
ret
end
def set_up_gallery_dl_mock(mock, sequence)
sequence.each do |seq|
expected = receive(seq[:receive])

55
spec/jobs/domain/fa/job/browse_page_job_spec.rb
View File

@@ -92,14 +92,14 @@ describe Domain::Fa::Job::BrowsePageJob do
expect do
ret = described_class.perform_later({})
expect(ret).not_to be(Exception)
end.to change { GoodJob::Job.count }.by(1)
end.to change(GoodJob::Job, :count).by(1)
end
it "does not enqueue more than one" do
expect do
described_class.perform_later({})
described_class.perform_later({})
end.to change { GoodJob::Job.count }.by(1)
end.to change(GoodJob::Job, :count).by(1)
end
it "can be enqueued in a bulk GoodJob batch" do
@@ -108,7 +108,7 @@ describe Domain::Fa::Job::BrowsePageJob do
described_class.perform_later({})
described_class.perform_later({})
end
end.to change { GoodJob::Job.count }.by(1)
end.to change(GoodJob::Job, :count).by(1)
end
context "with no posts found on page" do
@@ -127,7 +127,7 @@ describe Domain::Fa::Job::BrowsePageJob do
end
it "requests only one page" do
described_class.perform_now({})
perform_now({})
end
end
@@ -155,31 +155,26 @@ describe Domain::Fa::Job::BrowsePageJob do
end
it "creates a new post" do
expect {
ret = described_class.perform_now({})
expect(ret).not_to be(Exception)
}.to change {
Domain::Fa::Post.count
}.by(1)
expect do
perform_now({})
end.to change(Domain::Fa::Post, :count).by(1)
end
it "creates a new user" do
expect {
described_class.perform_now({})
}.to change {
Domain::Fa::User.count
}.by(1)
expect do
perform_now({})
end.to change(Domain::Fa::User, :count).by(1)
end
it "creates a post with the right attributes" do
described_class.perform_now({})
perform_now({})
expect(post.call.state).to eq("ok")
expect(post.call.title).to eq("reminder YCH AUCTION")
expect(post.call.creator).to eq(user.call)
end
it "creates a user with the right attributes" do
described_class.perform_now({})
perform_now({})
expect(user.call.name).to eq("Ruby_69r")
end
end
@@ -208,7 +203,7 @@ describe Domain::Fa::Job::BrowsePageJob do
end
it "enqueues scan post jobs" do
expect(described_class.perform_now({})).to_not be(Exception)
perform_now({})
post1 = Domain::Fa::Post.find_by(fa_id: 51509268)
post2 = Domain::Fa::Post.find_by(fa_id: 51509267)
@@ -261,9 +256,9 @@ describe Domain::Fa::Job::BrowsePageJob do
end
context "and nothing yet scanned" do
before {
expect(described_class.perform_now({})).to_not be(Exception)
}
before do
perform_now({})
end
include_examples "enqueue post scan", true
include_examples "enqueue file scan", false
include_examples "enqueue user page scan", true
@@ -275,7 +270,7 @@ describe Domain::Fa::Job::BrowsePageJob do
found_post = post.call
found_post.file_url_str = "http://www.example.com/img.jpg"
found_post.save!
described_class.perform_now({})
perform_now({})
end
include_examples "enqueue post scan", false
@@ -292,7 +287,7 @@ describe Domain::Fa::Job::BrowsePageJob do
file.save!
found_post.file = file
found_post.save!
described_class.perform_now({})
perform_now({})
end
include_examples "enqueue post scan", false
@@ -306,7 +301,7 @@ describe Domain::Fa::Job::BrowsePageJob do
creator = user.call
creator.scanned_gallery_at = 1.hour.ago
creator.save!
described_class.perform_now({})
perform_now({})
end
include_examples "enqueue post scan", true
@@ -320,7 +315,7 @@ describe Domain::Fa::Job::BrowsePageJob do
creator = user.call
creator.scanned_page_at = 1.hour.ago
creator.save!
described_class.perform_now({})
perform_now({})
end
include_examples "enqueue post scan", true
@@ -344,10 +339,12 @@ describe Domain::Fa::Job::BrowsePageJob do
)
end
it "fails with a fatal error" do
expect(described_class.perform_now({})).to(
be_a(Scraper::JobBase::JobError)
)
it "fails with a JobError" do
perform_now({}, should_raise: Scraper::JobBase::JobError)
end
it "has a useful error message" do
perform_now({}, should_raise: /non 200 response for/)
end
end
end

154
spec/jobs/domain/fa/job/favs_job_spec.rb Normal file
View File

@@ -0,0 +1,154 @@
require "rails_helper"
describe Domain::Fa::Job::FavsJob do
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
let(:client_mock_config) { [] }
before do
Scraper::ClientFactory.http_client_mock = http_client_mock
@log_entries = SpecUtil.init_http_client_mock(
http_client_mock, client_mock_config
)
end
shared_context "user exists" do
let!(:user) { Domain::Fa::User.create!(name: "zzreg", url_name: "zzreg") }
end
context "the user does not yet exist" do
it "fails the job" do
perform_now({ url_name: "zzreg" }, should_raise: /user does not exist/)
expect(Domain::Fa::User.find_by url_name: "zzreg").to be_nil
end
it "enqueues a page scan job" do
perform_now({ url_name: "zzreg" }, should_raise: true)
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob)).to match([
including(args: [including(url_name: "zzreg")]),
])
end
end
context "site indicates no favs" do
include_context "user exists"
let(:client_mock_config) do
[
{
uri: "https://www.furaffinity.net/favorites/zzreg/",
status_code: 200,
content_type: "text/html",
contents: SpecUtil.read_fixture_file("domain/fa/job/favs_zzreg_no_favs.html"),
},
]
end
it "records no favs for the user" do
perform_now({ url_name: "zzreg" })
expect(user.fav_posts).to eq([])
end
end
context "user has had a recent favs scan" do
include_context "user exists"
it "does not re-scan the user" do
user.scanned_favs_at = old_scanned_at = 1.day.ago
user.save!
perform_now({ url_name: "zzreg" })
expect(user.scanned_favs_at).to eq(old_scanned_at)
end
end
context "site indicates favs" do
include_context "user exists"
let(:fa_ids) do
[
52106426,
36755337,
40769488,
20808448,
20585829,
]
end
let(:client_mock_config) do
[
{
uri: "https://www.furaffinity.net/favorites/zzreg/",
status_code: 200,
content_type: "text/html",
contents: SpecUtil.read_fixture_file("domain/fa/job/favs_zzreg_page_0_first.html"),
},
{
uri: "https://www.furaffinity.net/favorites/zzreg/1074627373/next",
status_code: 200,
content_type: "text/html",
contents: SpecUtil.read_fixture_file("domain/fa/job/favs_zzreg_page_1_1074627373.html"),
caused_by_entry_idx: 0,
},
{
uri: "https://www.furaffinity.net/favorites/zzreg/475297391/next",
status_code: 200,
content_type: "text/html",
contents: SpecUtil.read_fixture_file("domain/fa/job/favs_zzreg_page_2_475297391.html"),
caused_by_entry_idx: 0,
},
]
end
it "records favs for the user" do
expect do
perform_now({ url_name: "zzreg" })
end.to change(Domain::Fa::Post, :count).by(5)
posts = Domain::Fa::Post.where(fa_id: fa_ids)
expect(user.fav_posts).to match_array(posts)
end
it "creates missing users" do
expect(Domain::Fa::User.find_by(url_name: "sepulte")).to be_nil
expect do
perform_now({ url_name: "zzreg" })
end.to change(Domain::Fa::User, :count).by(5)
post = Domain::Fa::Post.find_by(fa_id: 52106426)
expect(post).not_to be_nil
expect(post.creator.url_name).to eq("sepulte")
end
it "updates scanned_favs_at" do
perform_now({ url_name: "zzreg" })
user.reload
expect(user.scanned_favs_at).to be_within(1.second).of(Time.now)
end
context "the user model already has favs recorded" do
let(:old_post) { Domain::Fa::Post.create(fa_id: 12345, creator: user) }
before do
user.fav_posts << old_post
end
it "removes favs no longer present" do
perform_now({ url_name: "zzreg" })
user.reload
expect(user.fav_posts).not_to include(old_post)
end
it "adds favs newly present" do
perform_now({ url_name: "zzreg" })
posts = Domain::Fa::Post.where(fa_id: fa_ids)
expect(user.fav_posts).to match_array(posts)
end
it "creates new FA post models and enqueues scans" do
p1 = Domain::Fa::Post.create!(fa_id: fa_ids[0], creator: user)
expect do
perform_now({ url_name: "zzreg" })
end.to change(Domain::Fa::Post, :count).by(4)
user.reload
expect(user.fav_posts).to match_array(Domain::Fa::Post.where(fa_id: fa_ids))
p1.reload
expect(p1.faved_by).to eq([user])
end
end
end
end

11
spec/jobs/domain/fa/job/user_avatar_job_spec.rb
View File

@@ -45,8 +45,7 @@ describe Domain::Fa::Job::UserAvatarJob do
end
it "succeeds" do
ret = described_class.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
user.reload
avatar = user.avatar
@@ -55,8 +54,7 @@ describe Domain::Fa::Job::UserAvatarJob do
expect(HexUtil.bin2hex avatar.file_sha256).to eq("ebbafc07555df0a0656a9b32ec9b95723c62c5246937dc8434924d9241d1b570")
expect(avatar.downloaded_file_at).to be_within(1.seconds).of(Time.now)
ret = described_class.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
user.reload
avatar2 = user.avatar
expect(avatar).to eq(avatar2)
@@ -65,7 +63,7 @@ describe Domain::Fa::Job::UserAvatarJob do
context "the user has not been page scanned yet" do
it "enqueues a user page scan job" do
ret = described_class.perform_now({ user: user })
perform_now({ user: user })
expect(user.avatar.file_uri).to eq(nil)
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob)).to match([
including(args: [{
@@ -97,8 +95,7 @@ describe Domain::Fa::Job::UserAvatarJob do
end
it "succeeds" do
ret = described_class.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
user.reload
avatar = user.avatar
expect(avatar).not_to be_nil

43
spec/jobs/domain/fa/job/user_follows_job_spec.rb
View File

@@ -38,7 +38,7 @@ describe Domain::Fa::Job::UserFollowsJob do
shared_examples "zzreg follow creation" do
it "creates the right follows" do
e = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
perform_now({ user: user })
user.reload
expect(user.follows.length).to eq(FOLLOWS_ON_ZZREG_PAGE)
expect(user.scanned_follows_at).to_not be_nil
@@ -48,13 +48,12 @@ describe Domain::Fa::Job::UserFollowsJob do
context "performed with a user that doesn't exist yet" do
it "creates the scanned user and followed users" do
expect do
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ url_name: "zzreg" })
expect(ret).to_not be_a(Exception)
end.to change { Domain::Fa::User.count }.by(FOLLOWS_ON_ZZREG_PAGE + 1)
perform_now({ url_name: "zzreg" })
end.to change(Domain::Fa::User, :count).by(FOLLOWS_ON_ZZREG_PAGE + 1)
end
it "enqueues a user page job" do
Domain::Fa::Job::UserFollowsJob.perform_now({ url_name: "zzreg" })
perform_now({ url_name: "zzreg" })
zzreg = Domain::Fa::User.find_by(url_name: "zzreg")
expect(zzreg).to_not be_nil
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob).find do |job|
@@ -68,30 +67,27 @@ describe Domain::Fa::Job::UserFollowsJob do
it "can be performed by url_name" do
expect do
Domain::Fa::Job::UserFollowsJob.perform_now({ url_name: "zzreg" })
end.to change { Domain::Fa::User.count }.by(FOLLOWS_ON_ZZREG_PAGE)
perform_now({ url_name: "zzreg" })
end.to change(Domain::Fa::User, :count).by(FOLLOWS_ON_ZZREG_PAGE)
end
it "can be performed by direct post object" do
expect do
Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
end.to change { Domain::Fa::User.count }.by(FOLLOWS_ON_ZZREG_PAGE)
perform_now({ user: user })
end.to change(Domain::Fa::User, :count).by(FOLLOWS_ON_ZZREG_PAGE)
end
it "does not enqueue a user page job" do
Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
perform_now({ user: user })
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob).find do |job|
job[:args][0][:user] == user
end).to be_nil
end
it "can be ran twice to no ill effect" do
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
set_zzreg_http_mock.call
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user, force_scan: true })
expect(ret).to_not be_a(Exception)
perform_now({ user: user, force_scan: true })
end
include_examples "zzreg follow creation"
@@ -113,9 +109,8 @@ describe Domain::Fa::Job::UserFollowsJob do
original_updated_at = followed.updated_at
expect do
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
end.to change { Domain::Fa::User.count }.by(FOLLOWS_ON_ZZREG_PAGE - 1)
perform_now({ user: user })
end.to change(Domain::Fa::User, :count).by(FOLLOWS_ON_ZZREG_PAGE - 1)
followed.reload
expect(followed.num_submissions).to eq(10)
@@ -132,8 +127,7 @@ describe Domain::Fa::Job::UserFollowsJob do
end
it "newly inserted users have a name associated with them" do
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
user = Domain::Fa::User.find_by(url_name: "accelo")
expect(user).to_not be_nil
@@ -162,8 +156,7 @@ describe Domain::Fa::Job::UserFollowsJob do
follow_2 = Domain::Fa::Follow.create!(follower: user, followed: agi_type01_user)
expect(user.follows.length).to eq(2)
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
user.reload
expect(user.follows.length).to eq(FOLLOWS_ON_ZZREG_PAGE)
@@ -186,15 +179,13 @@ describe Domain::Fa::Job::UserFollowsJob do
end
it "does not enqueue a job if the user is not new" do
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob).length).to eq(FOLLOWS_ON_ZZREG_PAGE - 1)
end
it "does not enqueue jobs already in the queue" do
Domain::Fa::Job::UserPageJob.perform_later({ url_name: "accelo" })
ret = Domain::Fa::Job::UserFollowsJob.perform_now({ user: user })
expect(ret).to_not be_a(Exception)
perform_now({ user: user })
expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserPageJob).length).to eq(FOLLOWS_ON_ZZREG_PAGE - 1)
end
end

6
spec/jobs/domain/fa/job/user_page_job_spec.rb
View File

@@ -22,8 +22,7 @@ describe Domain::Fa::Job::UserPageJob do
end
it "succeeds" do
ret = described_class.perform_now({ url_name: "meesh" })
expect(ret).to_not be_a(Exception)
perform_now({ url_name: "meesh" })
user = Domain::Fa::User.find_by(url_name: "meesh")
expect(user).to_not be_nil
expect(user.avatar.file_uri.to_s).to eq("https://a.furaffinity.net/1635789297/meesh.gif")
@@ -51,8 +50,7 @@ describe Domain::Fa::Job::UserPageJob do
end
it "records the right fav count" do
ret = described_class.perform_now({ url_name: "marsdust" })
expect(ret).to_not be_a(Exception)
perform_now({ url_name: "marsdust" })
user = Domain::Fa::User.find_by(url_name: "marsdust")
expect(user).to_not be_nil
expect(user.avatar.file_uri.to_s).to eq("https://a.furaffinity.net/1424255659/marsdust.gif")

12
spec/jobs/domain/fa/twitter/job/user_timeline_tweets_job_spec.rb
View File

@@ -14,10 +14,8 @@ describe Domain::Twitter::Job::UserTimelineTweetsJob do
gallery_dl_user_with_no_tweets_sequence(gallery_dl_client_mock)
expect do
expect(described_class.perform_now({
name: "curtus",
})).to_not be_a(Exception)
end.to change { Domain::Twitter::User.count }.by(1)
perform_now({ name: "curtus" })
end.to change(Domain::Twitter::User, :count).by(1)
user = Domain::Twitter::User.find_by(name: "curtus")
expect(user).to_not be_nil
expect(user.tw_id).to eq(1234567)
@@ -30,10 +28,8 @@ describe Domain::Twitter::Job::UserTimelineTweetsJob do
gallery_dl_user_with_no_tweets_sequence(gallery_dl_client_mock)
user = Domain::Twitter::User.create!(name: "curtus")
expect do
expect(described_class.perform_now({
name: "curtus",
})).to_not be_a(Exception)
end.not_to change { Domain::Twitter::User.count }
perform_now({ name: "curtus" })
end.not_to change(Domain::Twitter::User, :count)
user.reload
expect(user.tw_id).to eq(1234567)

4
spec/spec_helper.rb
View File

@@ -13,11 +13,11 @@
# it.
#
# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
require "./spec/helpers/twitter_helpers"
require "./spec/helpers/spec_helpers"
require "./spec/helpers/debug_helpers"
RSpec.configure do |config|
config.include TwitterHelpers
config.include SpecHelpers
config.include DebugHelpers
# can tag classes with `quiet: false` to make ColorLogger loud

644
test/fixtures/files/domain/fa/job/favs_zzreg_no_favs.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

649
test/fixtures/files/domain/fa/job/favs_zzreg_page_0_first.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

651
test/fixtures/files/domain/fa/job/favs_zzreg_page_1_1074627373.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

647
test/fixtures/files/domain/fa/job/favs_zzreg_page_2_475297391.html vendored Normal file
View File

File diff suppressed because one or more lines are too long