dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Inkbunny post management functionality

Browse Source 
- Introduced a new model for managing Inkbunny posts, including creation, updating, and retrieval of post data.
- Implemented a job system for handling updates to posts and files, ensuring efficient processing of submissions.
- Enhanced the GlobalStatesController to manage Inkbunny credentials, allowing users to set either username/password or session ID.
- Updated routes to support Inkbunny post viewing and management, including parameterized routes for post IDs.
- Created policies to manage access to post details based on user roles, ensuring only authorized users can view sensitive information.
- Improved views for displaying Inkbunny posts, including enhanced layouts and user interaction elements.
- Added comprehensive tests for the new functionality, ensuring robust coverage for post management and credential handling.
This commit is contained in:
Dylan Knutson
2024-12-30 08:07:27 +00:00
parent 44778f6541
commit 97dff5abf9
25 changed files with 1078 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/controllers/domain/inkbunny/posts_controller.rb
View File

@@ -1,9 +1,11 @@
class Domain::Inkbunny::PostsController < ApplicationController
skip_before_action :authenticate_user!, only: [:show]
def index
@posts = Domain::Inkbunny::Post.page(params[:page])
end
def show
@post = Domain::Inkbunny::Post.find(params[:id])
@post = Domain::Inkbunny::Post.find_by!(ib_post_id: params[:ib_post_id])
end
end

41
app/controllers/global_states_controller.rb
View File

@@ -121,10 +121,34 @@ class GlobalStatesController < ApplicationController
authorize GlobalState
begin
params_hash = params.require(:ib_cookies).permit(*IB_COOKIE_KEYS).to_h
has_credentials =
params_hash["inkbunny-username"].present? ||
params_hash["inkbunny-password"].present?
has_sid = params_hash["inkbunny-sid"].present?
if has_credentials && has_sid
raise ArgumentError,
"Cannot set both credentials and session ID at the same time"
end
if !has_credentials && !has_sid
raise ArgumentError, "Must set either credentials or session ID"
end
ActiveRecord::Base.transaction do
ib_cookies_params.each do |key, value|
state = GlobalState.find_or_initialize_by(key: key)
state.value = value
if has_credentials
# Update username and password
%w[inkbunny-username inkbunny-password].each do |key|
state = GlobalState.find_or_initialize_by(key: key)
state.value = params_hash[key]
state.value_type = :string
state.save!
end
else
# Update SID
state = GlobalState.find_or_initialize_by(key: "inkbunny-sid")
state.value = params_hash["inkbunny-sid"]
state.value_type = :string
state.save!
end
@@ -132,6 +156,17 @@ class GlobalStatesController < ApplicationController
redirect_to ib_cookies_global_states_path,
notice: "Inkbunny credentials were successfully updated."
rescue ArgumentError => e
@ib_cookies =
IB_COOKIE_KEYS
.reject { |key| key == "inkbunny-sid" }
.map do |key|
GlobalState.find_by(key: key) ||
GlobalState.new(key: key, value_type: :string)
end
@ib_sid = GlobalState.find_by(key: "inkbunny-sid")
flash.now[:alert] = "Error updating Inkbunny credentials: #{e.message}"
render :edit_ib_cookies, status: :unprocessable_entity
rescue ActiveRecord::RecordInvalid => e
@ib_cookies =
IB_COOKIE_KEYS

4
app/helpers/indexable_posts_helper.rb
View File

@@ -10,6 +10,10 @@ module IndexablePostsHelper
Rails.application.routes.url_helpers.domain_e621_post_path(
indexed_post.postable,
)
when "Domain::Inkbunny::Post"
Rails.application.routes.url_helpers.domain_inkbunny_post_path(
indexed_post.postable,
)
else
raise("Unsupported postable type: #{indexed_post.postable_type}")
end

3
app/jobs/domain/inkbunny/job/file_job.rb
View File

@@ -1,5 +1,7 @@
module Domain::Inkbunny::Job
class FileJob < Base
queue_as :static_file
def perform(args)
file = args[:file] || fatal_error("file is required")
caused_by_entry = args[:caused_by_entry]
@@ -41,6 +43,7 @@ module Domain::Inkbunny::Job
file.state = :ok
file.log_entry = response.log_entry
file.blob_entry = response.log_entry.response
file.state_detail.delete("error")
file.save!
logger.info "downloaded file"
end

29
app/jobs/domain/inkbunny/job/job_helper.rb Normal file
View File

@@ -0,0 +1,29 @@
module Domain::Inkbunny::Job::JobHelper
def self.find_or_create_post_from_submission_json(submission_json)
ib_post_id = submission_json["submission_id"]&.to_i
raise "ib_post_id is blank" if ib_post_id.blank?
post =
Domain::Inkbunny::Post.includes(:creator).find_or_initialize_by(
ib_post_id: ib_post_id,
)
creator = find_or_create_user_from_submission_json(submission_json)
if post.creator && post.creator.ib_user_id != creator.ib_user_id
raise "post.creator.ib_user_id != creator.ib_user_id"
end
post.creator = creator
post.save! if post.changed?
post
end
def self.find_or_create_user_from_submission_json(submission_json)
ib_user_id = submission_json["user_id"]&.to_i
raise "ib_user_id is blank" if ib_user_id.blank?
user = Domain::Inkbunny::User.find_or_initialize_by(ib_user_id: ib_user_id)
user.name = submission_json["username"]
user.avatar_url_str = submission_json["user_icon_url_large"]
user.save! if user.changed?
user
end
end

199
app/jobs/domain/inkbunny/job/latest_posts_job.rb
View File

@@ -1,172 +1,61 @@
module Domain::Inkbunny::Job
class LatestPostsJob < Base
API_SEARCH_URL =
"https://inkbunny.net/api_search.php?orderby=create_datetime&keywords=no&title=no&description=no"
def perform(args)
url =
"https://inkbunny.net/api_search.php?orderby=create_datetime&keywords=no&title=no&description=no"
caused_by_entry = args[:caused_by_entry]
@api_search_response =
http_client.post(
url,
caused_by_entry: @first_browse_page_entry || @caused_by_entry
)
http_client.post(API_SEARCH_URL, caused_by_entry: caused_by_entry)
if @api_search_response.status_code != 200
fatal_error("api_search failed: #{@api_search_response.status_code}")
end
api_search_json = JSON.parse(@api_search_response.body)
handle_search_response(api_search_json)
ib_submission_jsons = api_search_json["submissions"]
@need_deep_update_ib_post_ids = []
ib_submission_jsons.each do |submission_json|
shallow_update_post!(submission_json)
end
if @need_deep_update_ib_post_ids.any?
defer_job(
Domain::Inkbunny::Job::UpdatePostsJob,
{
ib_post_ids: @need_deep_update_ib_post_ids,
caused_by_entry: @api_search_response.log_entry,
},
)
end
end
private
def handle_search_response(api_search_json)
ib_submission_jsons = api_search_json["submissions"]
ib_submission_ids =
ib_submission_jsons.map { |j| j["submission_id"]&.to_i }
@ib_post_id_to_model =
Domain::Inkbunny::Post
.where(ib_post_id: ib_submission_ids)
.includes(:files, :creator)
.index_by(&:ib_post_id)
new_posts = []
users = []
ib_submission_jsons.each do |submission_json|
ib_post_id = submission_json["submission_id"]&.to_i
unless @ib_post_id_to_model[ib_post_id]
post = Domain::Inkbunny::Post.new({ ib_post_id: ib_post_id })
user =
Domain::Inkbunny::User.find_or_initialize_by(
{ ib_user_id: submission_json["user_id"].to_i }
) { |user| user.name = submission_json["username"] }
user.save!
post.creator = user
new_posts << post
@ib_post_id_to_model[ib_post_id] = post
end
end
Domain::Inkbunny::Post.transaction do
users.select { |user| user.new_record? || user.changed? }.each(&:save!)
new_posts.each(&:save!)
end
# do shallow updates of all posts
needs_deep_update_posts = []
Domain::Inkbunny::Post.transaction do
ib_submission_jsons.each do |submission_json|
needs_deep_update, post =
shallow_update_post_from_submission_json(submission_json)
needs_deep_update_posts << post if needs_deep_update
end
end
# TODO - check condition for needing a deep update
# Such as:
# - Never been deep updated before
# - Number of files changed
# - Latest file updated timestamp changed
# - Don't have a user avatar yet
if needs_deep_update_posts.any?
ids_list = needs_deep_update_posts.map(&:ib_post_id).join(",")
url =
"https://inkbunny.net/api_submissions.php?" +
"submission_ids=#{ids_list}" +
"&show_description=yes&show_writing=yes&show_pools=yes"
@api_submissions_response =
http_client.get(url, caused_by_entry: @api_search_response.log_entry)
if @api_submissions_response.status_code != 200
fatal_error(
"api_submissions failed: #{@api_submissions_response.status_code}"
)
end
api_submissions_json = JSON.parse(@api_submissions_response.body)
submissions = api_submissions_json["submissions"]
logger.info("api_submissions page has #{submissions.size} posts")
submissions.each do |submission_json|
Domain::Inkbunny::Post.transaction do
deep_update_post_from_submission_json(submission_json)
end
end
end
end
def shallow_update_post_from_submission_json(json)
post = post_for_json(json)
post.shallow_updated_at = Time.now
post.title = json["title"]
post.posted_at = Time.parse json["create_datetime"]
post.last_file_updated_at = Time.parse json["last_file_update_datetime"]
post.num_files = json["pagecount"]&.to_i
post.rating = json["rating_id"]&.to_i
post.submission_type = json["submission_type_id"]&.to_i
post.ib_detail_raw = json
needs_deep_update =
post.last_file_updated_at_changed? || post.num_files_changed? ||
post.files.count != post.num_files
post.save!
[needs_deep_update, post]
end
def deep_update_post_from_submission_json(submission_json)
post = post_for_json(submission_json)
logger.info "deep update post #{post.ib_post_id.to_s.bold}"
post.deep_updated_at = Time.now
post.description = submission_json["description"]
# TODO - enqueue avatar download job if needed
if submission_json["user_icon_url_large"]
post.creator.avatar_url_str = submission_json["user_icon_url_large"]
post.creator.save! if post.creator.changed?
end
post_files_by_md5 = post.files.index_by(&:md5_initial)
file_jsons = submission_json["files"] || fatal_error("no files[] array")
file_jsons.each do |file_json|
md5_initial = file_json["initial_file_md5"]
next if post_files_by_md5[md5_initial]
md5_full = file_json["full_file_md5"]
file =
post.files.create!(
{
ib_file_id: file_json["file_id"]&.to_i,
ib_created_at: Time.parse(file_json["create_datetime"]),
file_order: file_json["submission_file_order"]&.to_i,
ib_detail_raw: file_json,
file_name: file_json["file_name"],
url_str: file_json["file_url_full"],
md5_initial: md5_initial,
md5_full: md5_full,
md5s: {
initial_file_md5: md5_initial,
full_file_md5: file_json["full_file_md5"],
large_file_md5: file_json["large_file_md5"],
small_file_md5: file_json["small_file_md5"],
thumbnail_md5: file_json["thumbnail_md5"]
}
}
)
logger.info "[ib_post_id #{post.ib_post_id.to_s.bold}] " +
"new file #{file.ib_file_id.to_s.bold} - #{file.file_name.black.bold}"
defer_job(
Domain::Inkbunny::Job::FileJob,
{ file: file, caused_by_entry: @api_submissions_response.log_entry }
def shallow_update_post!(submission_json)
post =
Domain::Inkbunny::Job::JobHelper.find_or_create_post_from_submission_json(
submission_json,
)
end
post.save!
end
post.shallow_updated_at = Time.now
post.title = submission_json["title"]
post.posted_at = Time.parse submission_json["create_datetime"]
post.last_file_updated_at =
Time.parse submission_json["last_file_update_datetime"]
post.num_files = submission_json["pagecount"]&.to_i
post.rating = submission_json["rating_id"]&.to_i
post.submission_type = submission_json["submission_type_id"]&.to_i
post.ib_detail_raw["submission_json"] = submission_json
def post_for_json(submission_json)
post_id =
submission_json["submission_id"]&.to_i ||
fatal_error(
"submission_id not found in submission_json: #{submission_json.keys.join(", ")}"
)
@ib_post_id_to_model[post_id] ||
fatal_error("post not found for ib_post_id #{post_id}")
if post.last_file_updated_at_changed? || post.num_files_changed? ||
post.files.count != post.num_files ||
post.creator.avatar_url_str.blank?
@need_deep_update_ib_post_ids << post.ib_post_id
end
post.save!
end
end
end

94
app/jobs/domain/inkbunny/job/update_posts_job.rb Normal file
View File

@@ -0,0 +1,94 @@
module Domain::Inkbunny::Job
class UpdatePostsJob < Base
def perform(args)
@caused_by_entry = args[:caused_by_entry]
@ib_post_ids = args[:ib_post_ids]
@ib_posts =
Domain::Inkbunny::Post
.where(ib_post_id: @ib_post_ids)
.includes(:files, :creator)
.index_by(&:ib_post_id)
ids_list = @ib_posts.keys.join(",")
url =
"https://inkbunny.net/api_submissions.php?" +
"submission_ids=#{ids_list}" +
"&show_description=yes&show_writing=yes&show_pools=yes"
@api_submissions_response =
http_client.get(url, caused_by_entry: @caused_by_entry)
if @api_submissions_response.status_code != 200
fatal_error(
"api_submissions failed: #{@api_submissions_response.status_code}",
)
end
api_submissions_json = JSON.parse(@api_submissions_response.body)
submissions = api_submissions_json["submissions"]
logger.info("api_submissions page has #{submissions.size} posts")
submissions.each do |submission_json|
Domain::Inkbunny::Post.transaction do
deep_update_post_from_submission_json(submission_json)
end
end
end
def deep_update_post_from_submission_json(submission_json)
post =
Domain::Inkbunny::Job::JobHelper.find_or_create_post_from_submission_json(
submission_json,
)
logger.info "deep update post #{post.ib_post_id.to_s.bold}"
post.deep_updated_at = Time.now
post.description = submission_json["description"]
post.writing = submission_json["writing"]
post.rating = submission_json["rating"]
post.submission_type = submission_json["submission_type"]
post.num_views = submission_json["views"]
post.num_files = submission_json["pagecount"]
post.last_file_updated_at =
Time.parse(submission_json["last_file_update_datetime"])
# TODO - enqueue avatar download job if needed
if submission_json["user_icon_url_large"]
post.creator.avatar_url_str = submission_json["user_icon_url_large"]
post.creator.save! if post.creator.changed?
end
post_files_by_md5 = post.files.index_by(&:md5_initial)
file_jsons = submission_json["files"] || fatal_error("no files[] array")
file_jsons.each do |file_json|
md5_initial = file_json["initial_file_md5"]
next if post_files_by_md5[md5_initial]
file =
post.files.create!(
{
ib_file_id: file_json["file_id"]&.to_i,
ib_created_at: Time.parse(file_json["create_datetime"]),
file_order: file_json["submission_file_order"]&.to_i,
ib_detail_raw: file_json,
file_name: file_json["file_name"],
url_str: file_json["file_url_full"],
md5_initial: md5_initial,
md5_full: file_json["full_file_md5"],
md5s: {
initial_file_md5: md5_initial,
full_file_md5: file_json["full_file_md5"],
large_file_md5: file_json["large_file_md5"],
small_file_md5: file_json["small_file_md5"],
thumbnail_md5: file_json["thumbnail_md5"],
},
},
)
logger.info "[ib_post_id #{post.ib_post_id.to_s.bold}] " +
"new file #{file.ib_file_id.to_s.bold} - #{file.file_name.black.bold}"
defer_job(
Domain::Inkbunny::Job::FileJob,
{ file: file, caused_by_entry: @api_submissions_response.log_entry },
)
end
post.save!
end
end
end

5
app/models/domain/inkbunny/post.rb
View File

@@ -32,5 +32,10 @@ class Domain::Inkbunny::Post < ReduxApplicationRecord
after_initialize do
self.state ||= :ok
self.state_detail ||= {}
self.ib_detail_raw ||= {}
end
def to_param
ib_post_id.to_s
end
end

2
app/models/indexed_post.rb
View File

@@ -38,6 +38,8 @@ class IndexedPost < ReduxApplicationRecord
postable&.title || "FA Post #{postable&.fa_id}"
when "Domain::E621::Post"
"E621 Post #{postable&.e621_id}"
when "Domain::Inkbunny::Post"
postable&.title || "IB Post #{postable&.ib_post_id}"
else
raise("Unsupported postable type: #{postable_type}")
end

19
app/policies/domain/inkbunny/post_policy.rb Normal file
View File

@@ -0,0 +1,19 @@
class Domain::Inkbunny::PostPolicy < ApplicationPolicy
def show?
true # Anyone can view the basic post info
end
def view_file?
user&.admin?
end
def view_scraper_metadata?
user&.admin?
end
class Scope < Scope
def resolve
scope.all # All users can see posts exist in lists
end
end
end

98
app/views/domain/inkbunny/posts/show.html.erb
View File

@@ -1,29 +1,77 @@
<div class='max-w-5xl mx-auto w-full px-6 sm:px-8'>
<%= link_to "← Back to Posts", domain_inkbunny_posts_path, class: "text-blue-600 hover:underline mb-4 inline-block" %>
<div class="text-center">
<h1 class='text-2xl'><%= @post.title %></h1>
<div class='text-stone-500 text-sm mt-2'>
by <%= link_to @post.creator.name, @post.creator, class: 'hover:underline' %>
<div
id="<%= dom_id @post %>"
class="mx-auto mt-4 flex w-full max-w-2xl flex-col gap-4 pb-4"
>
<section class="rounded-md border border-slate-300 bg-slate-50 p-4">
<div class="flex items-center justify-between gap-4">
<div class="flex min-w-0 items-center gap-4">
<div class="flex min-w-0 items-center gap-2">
<span class="truncate text-lg font-medium">
<%= link_to @post.title,
"https://inkbunny.net/s/#{@post.ib_post_id}",
class: "text-blue-600 hover:underline",
target: "_blank" %>
</span>
<i class="fa-solid fa-arrow-up-right-from-square text-slate-400"></i>
</div>
<div class="flex items-center gap-2 whitespace-nowrap text-slate-600">
by
<%= link_to @post.creator.name, @post.creator, class: "hover:underline" %>
</div>
</div>
</div>
</div>
</div>
<div class='mx-auto mt-4'>
<div class='text-stone-600 mb-4'>
<div>Post ID: <%= link_to "https://inkbunny.net/s/#{@post.ib_post_id}", target: "_blank", class: "text-blue-600 hover:underline inline-flex items-center" do %>
<%= @post.ib_post_id %>
<%= render partial: "shared/icons/external_link", locals: { class_name: "w-4 h-4 ml-1" } %>
<% end %></div>
<div>Type: <%= @post.submission_type.titleize %></div>
<div>Rating: <%= @post.rating.titleize %></div>
<div>Posted: <%= @post.posted_at.strftime("%B %d, %Y at %I:%M %p") %> (<%= time_ago_in_words(@post.posted_at) %> ago)</div>
<div>Scanned: <%= @post.created_at.strftime("%B %d, %Y at %I:%M %p") %> (<%= time_ago_in_words(@post.created_at) %> ago)</div>
</div>
<div class='flex flex-row gap-6 flex-wrap justify-center p-4'>
<% @post.files.each do |file| %>
<% img_src_path = contents_blob_path(HexUtil.bin2hex(file.blob_entry_sha256), format: "jpg") %>
<div class="rounded-lg overflow-hidden shadow-lg">
<img class='max-w-[400px] w-full h-auto' alt='<%= @post.title %>' src='<%= img_src_path %>' />
<div class="mt-2 flex flex-wrap gap-x-4 text-sm text-slate-600">
<span>
<i class="fa-regular fa-calendar mr-1"></i>
Posted: <%= @post.posted_at&.strftime("%Y-%m-%d") %>
(<%= time_ago_in_words(@post.posted_at) if @post.posted_at %> ago)
</span>
<span>
<i class="fa-solid fa-tag mr-1"></i>
Type: <%= @post.submission_type&.titleize || "Unknown" %>
</span>
<span>
<i class="fa-solid fa-shield mr-1"></i>
Rating: <%= @post.rating&.titleize || "Unknown" %>
</span>
</div>
<% if policy(@post).view_scraper_metadata? %>
<div class="mt-2 text-sm text-slate-500">
Scanned: <%= @post.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
(<%= time_ago_in_words(@post.created_at) %> ago)
</div>
<% end %>
</div>
</section>
<% if policy(@post).view_file? %>
<section>
<div class="flex flex-col gap-4">
<% @post.files.each do |file| %>
<% if file.blob_entry %>
<div class="overflow-hidden rounded-lg shadow-lg">
<img
class="h-auto w-full"
alt="<%= @post.title %>"
src="<%= contents_blob_path(HexUtil.bin2hex(file.blob_entry_sha256), format: "jpg") %>"
/>
</div>
<% else %>
<div class="text-center text-slate-600">
File #<%= file.ib_file_id %> not yet downloaded
</div>
<% end %>
<% end %>
</div>
</section>
<% else %>
<section class="sky-section">
<%= link_to "https://inkbunny.net/s/#{@post.ib_post_id}",
target: "_blank",
rel: "noopener noreferrer",
class: "section-header flex items-center gap-2 hover:text-slate-600" do %>
<span>View Post on Inkbunny</span>
<i class="fa-solid fa-arrow-up-right-from-square"></i>
<% end %>
</section>
<% end %>
</div>

129
app/views/global_states/edit_ib_cookies.html.erb
View File

@@ -5,7 +5,8 @@
Edit Inkbunny Credentials
</h1>
<p class="mt-2 text-sm text-slate-700">
Update the credentials used for Inkbunny authentication.
Update the credentials used for Inkbunny authentication. You can either
set username and password, or set a session ID directly.
</p>
</div>
<div class="mt-4 sm:ml-16 sm:mt-0 sm:flex-none">
@@ -18,56 +19,88 @@
<div class="mt-6 overflow-hidden bg-white shadow sm:rounded-lg">
<div class="p-3 sm:p-4">
<%= form_tag ib_cookies_global_states_path, method: :patch do %>
<table class="min-w-full divide-y divide-slate-300">
<thead>
<tr>
<th class="pb-2 text-left text-sm font-semibold text-slate-900">
Field
</th>
<th class="pb-2 text-left text-sm font-semibold text-slate-900">
Value
</th>
</tr>
</thead>
<tbody class="divide-y divide-slate-200">
<% @ib_cookies.each do |cookie| %>
<div class="mb-8">
<h2 class="mb-4 text-lg font-medium text-slate-900">
Login with Username and Password
</h2>
<%= form_tag ib_cookies_global_states_path, method: :patch do %>
<table class="min-w-full divide-y divide-slate-300">
<thead>
<tr>
<td class="py-2 pr-4 text-sm font-medium text-slate-900">
<%= cookie.key.sub("inkbunny-", "").titleize %>
</td>
<td class="py-2 pr-4">
<%= text_field_tag "ib_cookies[#{cookie.key}]",
cookie.value,
type: cookie.key == "inkbunny-password" ? "password" : "text",
class:
"block w-full rounded-md border-slate-300 shadow-sm focus:border-sky-500 focus:ring-sky-500 sm:text-sm" %>
</td>
<th class="pb-2 text-left text-sm font-semibold text-slate-900">
Field
</th>
<th class="pb-2 text-left text-sm font-semibold text-slate-900">
Value
</th>
</tr>
<% end %>
<% if @ib_sid %>
<tr>
<td class="py-2 pr-4 text-sm font-medium text-slate-900">
Session ID
</td>
<td class="py-2 pr-4 text-sm text-slate-500">
<%= @ib_sid.value.present? ? @ib_sid.value : "(not set)" %>
</td>
</tr>
<% end %>
</tbody>
</table>
</thead>
<tbody class="divide-y divide-slate-200">
<% @ib_cookies.each do |cookie| %>
<tr>
<td class="py-2 pr-4 text-sm font-medium text-slate-900">
<%= cookie.key.sub("inkbunny-", "").titleize %>
</td>
<td class="py-2 pr-4">
<%= text_field_tag "ib_cookies[#{cookie.key}]",
cookie.value,
type: cookie.key == "inkbunny-password" ? "password" : "text",
class:
"block w-full rounded-md border-slate-300 shadow-sm focus:border-sky-500 focus:ring-sky-500 sm:text-sm" %>
</td>
</tr>
<% end %>
</tbody>
</table>
<div class="mt-4 flex justify-end space-x-3">
<%= link_to "Cancel",
ib_cookies_global_states_path,
class:
"rounded-md border border-slate-300 bg-white py-2 px-4 text-sm font-medium text-slate-700 shadow-sm hover:bg-slate-50 focus:outline-none focus:ring-2 focus:ring-sky-500 focus:ring-offset-2" %>
<%= submit_tag "Save",
class:
"inline-flex justify-center rounded-md border border-transparent bg-sky-600 py-2 px-4 text-sm font-medium text-white shadow-sm hover:bg-sky-700 focus:outline-none focus:ring-2 focus:ring-sky-500 focus:ring-offset-2" %>
</div>
<% end %>
<div class="mt-4 flex justify-end space-x-3">
<%= link_to "Cancel",
ib_cookies_global_states_path,
class:
"rounded-md border border-slate-300 bg-white py-2 px-4 text-sm font-medium text-slate-700 shadow-sm hover:bg-slate-50 focus:outline-none focus:ring-2 focus:ring-sky-500 focus:ring-offset-2" %>
<%= submit_tag "Save Credentials",
class:
"inline-flex justify-center rounded-md border border-transparent bg-sky-600 py-2 px-4 text-sm font-medium text-white shadow-sm hover:bg-sky-700 focus:outline-none focus:ring-2 focus:ring-sky-500 focus:ring-offset-2" %>
</div>
<% end %>
</div>
<div class="border-t border-slate-200 pt-8">
<h2 class="mb-4 text-lg font-medium text-slate-900">
Set Session ID Directly
</h2>
<%= form_tag ib_cookies_global_states_path, method: :patch do %>
<div class="space-y-4">
<div>
<label
for="ib_cookies[inkbunny-sid]"
class="block text-sm font-medium text-slate-700"
>Session ID</label
>
<div class="mt-1">
<%= text_field_tag "ib_cookies[inkbunny-sid]",
@ib_sid&.value,
class:
"block w-full rounded-md border-slate-300 shadow-sm focus:border-sky-500 focus:ring-sky-500 sm:text-sm" %>
</div>
<p class="mt-2 text-sm text-slate-500">
Current session ID:
<%= @ib_sid&.value.present? ? @ib_sid.value : "(not set)" %>
</p>
</div>
<div class="flex justify-end space-x-3">
<%= link_to "Cancel",
ib_cookies_global_states_path,
class:
"rounded-md border border-slate-300 bg-white py-2 px-4 text-sm font-medium text-slate-700 shadow-sm hover:bg-slate-50 focus:outline-none focus:ring-2 focus:ring-sky-500 focus:ring-offset-2" %>
<%= submit_tag "Save Session ID",
class:
"inline-flex justify-center rounded-md border border-transparent bg-sky-600 py-2 px-4 text-sm font-medium text-white shadow-sm hover:bg-sky-700 focus:outline-none focus:ring-2 focus:ring-sky-500 focus:ring-offset-2" %>
</div>
</div>
<% end %>
</div>
</div>
</div>
</div>

5
app/views/indexed_posts/_inline_postable_domain_link.html.erb
View File

@@ -9,6 +9,11 @@
<% icon_title = "E621" %>
<% external_url = "https://e621.net/posts/#{post.postable.e621_id}" %>
<% link_text = "E621 ##{post.postable.e621_id}" %>
<% when "Domain::Inkbunny::Post" %>
<% domain_icon = asset_path("domain-icons/inkbunny.png") %>
<% icon_title = "Inkbunny" %>
<% external_url = "https://inkbunny.net/post/#{post.postable.ib_post_id}" %>
<% link_text = "IB ##{post.postable.ib_post_id}" %>
<% else %>
<% domain_icon = nil %>
<% external_url = nil %>

2
config/routes.rb
View File

@@ -37,7 +37,7 @@ Rails.application.routes.draw do
end
namespace :inkbunny, path: "ib" do
resources :posts, only: %i[show]
resources :posts, param: :ib_post_id, only: %i[show]
resources :users, param: :name, only: [:show]
end
end

40
db/migrate/20241230060212_add_fks_to_inkbunny_table.rb Normal file
View File

@@ -0,0 +1,40 @@
class AddFksToInkbunnyTable < ActiveRecord::Migration[7.2]
def change
add_foreign_key :domain_inkbunny_posts,
:domain_inkbunny_users,
column: :creator_id,
validate: true
add_foreign_key :domain_inkbunny_files,
:domain_inkbunny_posts,
column: :post_id,
validate: true
add_foreign_key :domain_inkbunny_favs,
:domain_inkbunny_users,
column: :user_id,
validate: true
add_foreign_key :domain_inkbunny_favs,
:domain_inkbunny_posts,
column: :post_id,
validate: true
add_foreign_key :domain_inkbunny_files,
:http_log_entries,
column: :log_entry_id,
validate: true
add_foreign_key :domain_inkbunny_follows,
:domain_inkbunny_users,
column: :follower_id,
validate: true
add_foreign_key :domain_inkbunny_follows,
:domain_inkbunny_users,
column: :followed_id,
validate: true
add_foreign_key :domain_inkbunny_pool_joins,
:domain_inkbunny_posts,
column: :post_id,
validate: true
add_foreign_key :domain_inkbunny_pool_joins,
:domain_inkbunny_pools,
column: :pool_id,
validate: true
end
end

4
db/schema.rb generated
View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.2].define(version: 2024_12_30_005956) do
ActiveRecord::Schema[7.2].define(version: 2024_12_30_060212) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_prewarm"
enable_extension "pg_stat_statements"
@@ -1824,10 +1824,12 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_30_005956) do
add_foreign_key "domain_inkbunny_favs", "domain_inkbunny_posts", column: "post_id"
add_foreign_key "domain_inkbunny_favs", "domain_inkbunny_users", column: "user_id"
add_foreign_key "domain_inkbunny_files", "domain_inkbunny_posts", column: "post_id"
add_foreign_key "domain_inkbunny_files", "http_log_entries", column: "log_entry_id"
add_foreign_key "domain_inkbunny_follows", "domain_inkbunny_users", column: "followed_id"
add_foreign_key "domain_inkbunny_follows", "domain_inkbunny_users", column: "follower_id"
add_foreign_key "domain_inkbunny_pool_joins", "domain_inkbunny_pools", column: "pool_id"
add_foreign_key "domain_inkbunny_pool_joins", "domain_inkbunny_posts", column: "post_id"
add_foreign_key "domain_inkbunny_posts", "domain_inkbunny_users", column: "creator_id"
add_foreign_key "domain_twitter_medias", "domain_twitter_tweets", column: "tweet_id"
add_foreign_key "domain_twitter_medias", "http_log_entries", column: "file_id"
add_foreign_key "domain_twitter_tweets", "domain_twitter_users", column: "author_id", primary_key: "tw_id", name: "on_author_id"

44
er_id Normal file
View File

@@ -0,0 +1,44 @@
=> {"submission_id"=>"3104200",
"hidden"=>"f",
"username"=>"Seff",
"user_id"=>"229331",
"create_datetime"=>"2023-08-27 21:30:59.308046+02",
"create_datetime_usertime"=>"27 Aug 2023 21:30 CEST",
"last_file_update_datetime"=>"2023-08-27 21:26:14.049+02",
"last_file_update_datetime_usertime"=>"27 Aug 2023 21:26 CEST",
"thumbnail_url_huge_noncustom"=>"https://us.ib.metapix.net/files/preview/4652/4652528_Seff_aug23sketches7-1.jpg",
"thumbnail_url_large_noncustom"=>"https://us.ib.metapix.net/thumbnails/large/4652/4652528_Seff_aug23sketches7-1_noncustom.jpg",
"thumbnail_url_medium_noncustom"=>"https://us.ib.metapix.net/thumbnails/medium/4652/4652528_Seff_aug23sketches7-1_noncustom.jpg",
"thumb_medium_noncustom_x"=>"96",
"thumb_medium_noncustom_y"=>"120",
"thumb_large_noncustom_x"=>"160",
"thumb_large_noncustom_y"=>"200",
"thumb_huge_noncustom_x"=>"240",
"thumb_huge_noncustom_y"=>"300",
"file_name"=>"4652528_Seff_aug23sketches7-1.png",
"title"=>"Camp Pines Sketch Dump (Aug 2023)",
"deleted"=>"f",
"public"=>"t",
"mimetype"=>"image/png",
"pagecount"=>"4",
"rating_id"=>"2",
"rating_name"=>"Adult",
"file_url_full"=>"https://us.ib.metapix.net/files/full/4652/4652528_Seff_aug23sketches7-1.png",
"file_url_screen"=>"https://us.ib.metapix.net/files/screen/4652/4652528_Seff_aug23sketches7-1.png",
"file_url_preview"=>"https://us.ib.metapix.net/files/preview/4652/4652528_Seff_aug23sketches7-1.jpg",
"submission_type_id"=>"1",
"type_name"=>"Picture/Pinup",
"friends_only"=>"f",
"guest_block"=>"t",
"scraps"=>"t",
"latest_file_name"=>"4652535_Seff_aug23sketches6-1.png",
"latest_mimetype"=>"image/png",
"latest_thumbnail_url_huge_noncustom"=>"https://us.ib.metapix.net/files/preview/4652/4652535_Seff_aug23sketches6-1.jpg",
"latest_thumbnail_url_large_noncustom"=>"https://us.ib.metapix.net/thumbnails/large/4652/4652535_Seff_aug23sketches6-1_noncustom.jpg",
"latest_thumbnail_url_medium_noncustom"=>"https://us.ib.metapix.net/thumbnails/medium/4652/4652535_Seff_aug23sketches6-1_noncustom.jpg",
"latest_thumb_medium_noncustom_x"=>"80",
"latest_thumb_medium_noncustom_y"=>"120",
"latest_thumb_large_noncustom_x"=>"133",
"latest_thumb_large_noncustom_y"=>"200",
"latest_thumb_huge_noncustom_x"=>"200",
"latest_thumb_huge_noncustom_y"=>"300"}

32
spec/controllers/domain/inkbunny/posts_controller_spec.rb Normal file
View File

@@ -0,0 +1,32 @@
require "rails_helper"
RSpec.describe Domain::Inkbunny::PostsController, type: :controller do
describe "GET #show" do
let(:post) { create(:domain_inkbunny_post) }
let(:file) { create(:domain_inkbunny_file, post: post) }
context "when user is not logged in" do
it "shows post details but not file content or scraper metadata" do
file # Create the file
get :show, params: { ib_post_id: post.ib_post_id }
expect(response).to be_successful
expect(response).to render_template(:show)
end
end
context "when user is an admin" do
let(:user) { create(:user, :admin) }
before do
sign_in user
file # Create the file
end
it "shows file content and scraper metadata" do
get :show, params: { ib_post_id: post.ib_post_id }
expect(response).to be_successful
expect(response).to render_template(:show)
end
end
end
end

80
spec/controllers/global_states_controller_spec.rb
View File

@@ -393,6 +393,86 @@ RSpec.describe GlobalStatesController, type: :controller do
)
end
end
context "when trying to set both credentials and session ID" do
let(:invalid_params) do
{
ib_cookies: {
"inkbunny-username" => "user",
"inkbunny-password" => "pass",
"inkbunny-sid" => "sid123",
},
}
end
it "does not update any credentials" do
expect {
patch :update_ib_cookies, params: invalid_params
}.not_to change(GlobalState, :count)
end
it "renders the edit template" do
patch :update_ib_cookies, params: invalid_params
expect(response).to render_template(:edit_ib_cookies)
end
it "sets an appropriate error message" do
patch :update_ib_cookies, params: invalid_params
expect(flash.now[:alert]).to match(
/Cannot set both credentials and session ID at the same time/,
)
end
end
context "when setting only session ID" do
let(:sid_params) { { ib_cookies: { "inkbunny-sid" => "newsid123" } } }
it "updates the session ID and preserves credentials" do
username = create(:global_state, :inkbunny_username, value: "olduser")
password = create(:global_state, :inkbunny_password, value: "oldpass")
patch :update_ib_cookies, params: sid_params
expect(GlobalState.find_by(key: "inkbunny-sid").value).to eq(
"newsid123",
)
expect(username.reload.value).to eq("olduser")
expect(password.reload.value).to eq("oldpass")
end
it "redirects to the credentials page" do
patch :update_ib_cookies, params: sid_params
expect(response).to redirect_to(ib_cookies_global_states_path)
end
end
context "when setting only credentials" do
let(:credentials_params) do
{
ib_cookies: {
"inkbunny-username" => "newuser",
"inkbunny-password" => "newpass",
},
}
end
it "updates the credentials and preserves session ID" do
sid = create(:global_state, :inkbunny_sid, value: "oldsid")
patch :update_ib_cookies, params: credentials_params
username = GlobalState.find_by(key: "inkbunny-username")
password = GlobalState.find_by(key: "inkbunny-password")
expect(username.value).to eq("newuser")
expect(password.value).to eq("newpass")
expect(sid.reload.value).to eq("oldsid")
end
it "redirects to the credentials page" do
patch :update_ib_cookies, params: credentials_params
expect(response).to redirect_to(ib_cookies_global_states_path)
end
end
end
end
end

41
spec/factories/inkbunny.rb Normal file
View File

@@ -0,0 +1,41 @@
FactoryBot.define do
factory :domain_inkbunny_user, class: "Domain::Inkbunny::User" do
sequence(:name) { |n| "user#{n}" }
sequence(:ib_user_id) { |n| n }
state { :ok }
state_detail { {} }
end
factory :domain_inkbunny_post, class: "Domain::Inkbunny::Post" do
sequence(:ib_post_id) { |n| n }
state { :ok }
state_detail { {} }
title { "Test Post" }
description { "Test Description" }
writing { "" }
rating { :general }
submission_type { :picture_pinup }
num_views { 0 }
num_files { 1 }
ib_detail_raw { {} }
association :creator, factory: :domain_inkbunny_user
end
factory :domain_inkbunny_file, class: "Domain::Inkbunny::File" do
state { :ok }
state_detail { {} }
file_order { 0 }
sequence(:ib_file_id) { |n| n }
ib_detail_raw { {} }
file_name { "test.jpg" }
url_str { "https://example.com/test.jpg" }
ib_created_at { Time.current }
md5_initial { "abc123" }
md5_full { "def456" }
md5s { { initial: "abc123", full: "def456" } }
association :post, factory: :domain_inkbunny_post
after(:build) do |file|
file.blob_entry_sha256 = Digest::SHA256.digest(SecureRandom.hex(32))
end
end
end

6
spec/jobs/domain/fa/twitter/job/user_timeline_tweets_job_spec.rb
View File

@@ -1,3 +1,5 @@
require "rails_helper"
describe Domain::Twitter::Job::UserTimelineTweetsJob do
GDLClient = Scraper::GalleryDlClient
@@ -15,7 +17,7 @@ describe Domain::Twitter::Job::UserTimelineTweetsJob do
expect do perform_now({ name: "curtus" }) end.to change(
Domain::Twitter::User,
:count
:count,
).by(1)
user = Domain::Twitter::User.find_by(name: "curtus")
expect(user).to_not be_nil
@@ -30,7 +32,7 @@ describe Domain::Twitter::Job::UserTimelineTweetsJob do
user = Domain::Twitter::User.create!(name: "curtus")
expect do perform_now({ name: "curtus" }) end.not_to change(
Domain::Twitter::User,
:count
:count,
)
user.reload

39
spec/jobs/domain/inkbunny/file_job_spec.rb
View File

@@ -153,5 +153,44 @@ describe Domain::Inkbunny::Job::FileJob do
expect(file.state).to eq("error")
expect(file.blob_entry).to be_nil
end
it "fails if file argument is missing" do
expect { perform_now({}) }.to raise_error(/file is required/)
end
it "retries a file in error state that hasn't hit retry limit" do
SpecUtil.init_http_client_mock(
http_client_mock,
[
# First attempt fails
{
uri: FileJobSpec::AN_IMAGE_URL,
status_code: 500,
content_type: "text/html",
contents: "error",
},
{
uri: FileJobSpec::AN_IMAGE_URL,
status_code: 200,
content_type: "image/png",
contents: SpecUtil.read_fixture_file(FileJobSpec::AN_IMAGE_PATH),
},
],
)
perform_now({ file: file }, should_raise: true)
file.reload
expect(file.state).to eq("error")
expect(file.state_detail["error"]["retry_count"]).to eq(1)
# Second attempt succeeds
perform_now({ file: file })
file.reload
expect(file.state).to eq("ok")
expect(file.blob_entry).not_to be_nil
expect(file.blob_entry.sha256_hex).to eq(FileJobSpec::AN_IMAGE_SHA256)
expect(file.state_detail["error"]).to be_nil
expect(file.state_detail).not_to have_key("retry_count")
end
end
end

237
spec/jobs/domain/inkbunny/latest_posts_job_spec.rb
View File

@@ -23,36 +23,35 @@ describe Domain::Inkbunny::Job::LatestPostsJob do
uri: api_search_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file("domain/inkbunny/job/api_search.json")
SpecUtil.read_fixture_file("domain/inkbunny/job/api_search.json"),
},
{
method: :get,
uri: api_submissions_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions.json"
),
caused_by_entry_idx: 0
},
# same as the first, should not update or touch any posts
{
method: :post,
uri: api_search_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file("domain/inkbunny/job/api_search.json")
}
]
# {
# method: :get,
# uri: api_submissions_url,
# content_type: "application/json",
# contents:
# SpecUtil.read_fixture_file(
# "domain/inkbunny/job/api_submissions.json",
# ),
# caused_by_entry_idx: 0,
# },
# # same as the first, should not update or touch any posts
# {
# method: :post,
# uri: api_search_url,
# content_type: "application/json",
# contents:
# SpecUtil.read_fixture_file("domain/inkbunny/job/api_search.json"),
# },
],
)
end
it "creates posts" do
expect { perform_now({}) }.to(
change(Domain::Inkbunny::Post, :count)
.by(3)
.and(change(Domain::Inkbunny::File, :count).by(6))
.and(change(Domain::Inkbunny::User, :count).by(3))
change(Domain::Inkbunny::Post, :count).by(3).and(
change(Domain::Inkbunny::User, :count).by(3),
),
)
user_thendyart = Domain::Inkbunny::User.find_by!(ib_user_id: 941_565)
@@ -60,66 +59,78 @@ describe Domain::Inkbunny::Job::LatestPostsJob do
user_seff = Domain::Inkbunny::User.find_by!(ib_user_id: 229_331)
expect(user_seff.name).to eq("Seff")
expect(user_seff.avatar_url_str).to eq(
"https://us.ib.metapix.net/usericons/large/176/176443_Seff_seffpfp.png"
)
expect(user_seff.avatar_url_str).to be_nil
# this gets populated in the update_posts job
# expect(user_seff.avatar_url_str).to eq(
# "https://us.ib.metapix.net/usericons/large/176/176443_Seff_seffpfp.png",
# )
post_3104202 = Domain::Inkbunny::Post.find_by!(ib_post_id: 3_104_202)
expect(post_3104202.title).to eq("Phantom Touch - Page 25")
expect(post_3104202.posted_at).to eq(
Time.parse("2023-08-27 21:31:40.365597+02")
Time.parse("2023-08-27 21:31:40.365597+02"),
)
expect(post_3104202.creator).to eq(user_thendyart)
expect(post_3104202.last_file_updated_at).to eq(
Time.parse("2023-08-27 21:30:06.222262+02")
Time.parse("2023-08-27 21:30:06.222262+02"),
)
expect(post_3104202.num_files).to eq(1)
expect(post_3104202.rating).to eq("adult")
expect(post_3104202.submission_type).to eq("comic")
expect(post_3104202.shallow_updated_at).to be_within(1.second).of(
Time.now
Time.now,
)
expect(post_3104202.deep_updated_at).to be_within(1.second).of(Time.now)
expect(post_3104202.deep_updated_at).to be_nil
expect(post_3104202.files.count).to eq(1)
file_4652537 = post_3104202.files.first
expect(file_4652537.ib_file_id).to eq(4_652_537)
expect(file_4652537.file_order).to eq(0)
expect(file_4652537.md5_initial).to eq("fbeb553c483a346108beeada93d90086")
expect(file_4652537.md5_full).to eq("15eea2648c8afaee1fef970befb28b24")
expect(file_4652537.url_str).to eq(
"https://us.ib.metapix.net/files/full/4652/4652537_ThendyArt_pt_pg_25.jpg"
update_post_jobs =
SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::UpdatePostsJob)
expect(update_post_jobs.size).to eq(1)
expect(update_post_jobs[0][:args][0][:ib_post_ids]).to eq(
[3_104_202, 3_104_200, 3_104_197],
)
expect(update_post_jobs[0][:args][0][:caused_by_entry]).to eq(
log_entries[0],
)
post_3104200 = Domain::Inkbunny::Post.find_by!(ib_post_id: 3_104_200)
expect(post_3104200.creator).to eq(user_seff)
expect(post_3104200.title).to eq("Camp Pines Sketch Dump (Aug 2023)")
expect(post_3104200.description).to match(/Not sure how canon/)
expect(post_3104200.num_files).to eq(4)
# expect(post_3104202.files.count).to eq(1)
# file_4652537 = post_3104202.files.first
# expect(file_4652537.ib_file_id).to eq(4_652_537)
# expect(file_4652537.file_order).to eq(0)
# expect(file_4652537.md5_initial).to eq("fbeb553c483a346108beeada93d90086")
# expect(file_4652537.md5_full).to eq("15eea2648c8afaee1fef970befb28b24")
# expect(file_4652537.url_str).to eq(
# "https://us.ib.metapix.net/files/full/4652/4652537_ThendyArt_pt_pg_25.jpg",
# )
# should enqueue file download jobs as all are new
file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
expect(file_jobs.length).to eq(6)
expect(
file_jobs.map { |job| job[:args][0][:file].ib_file_id }.sort
).to eq(
[4_652_528, 4_652_530, 4_652_531, 4_652_534, 4_652_535, 4_652_537]
)
file_jobs.each do |job|
expect(job[:args][0][:caused_by_entry]).to eq(log_entries[1])
end
# post_3104200 = Domain::Inkbunny::Post.find_by!(ib_post_id: 3_104_200)
# expect(post_3104200.creator).to eq(user_seff)
# expect(post_3104200.title).to eq("Camp Pines Sketch Dump (Aug 2023)")
# expect(post_3104200.description).to match(/Not sure how canon/)
# expect(post_3104200.num_files).to eq(4)
# perform another scan, nothing should change
expect { perform_now({}) }.to(
change(Domain::Inkbunny::Post, :count)
.by(0)
.and(change(Domain::Inkbunny::File, :count).by(0))
.and(change(Domain::Inkbunny::User, :count).by(0))
)
# # should enqueue file download jobs as all are new
# file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
# expect(file_jobs.length).to eq(6)
# expect(
# file_jobs.map { |job| job[:args][0][:file].ib_file_id }.sort,
# ).to eq(
# [4_652_528, 4_652_530, 4_652_531, 4_652_534, 4_652_535, 4_652_537],
# )
# file_jobs.each do |job|
# expect(job[:args][0][:caused_by_entry]).to eq(log_entries[1])
# end
expect(
SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob).length
).to eq(file_jobs.length)
# # perform another scan, nothing should change
# expect { perform_now({}) }.to(
# change(Domain::Inkbunny::Post, :count)
# .by(0)
# .and(change(Domain::Inkbunny::File, :count).by(0))
# .and(change(Domain::Inkbunny::User, :count).by(0)),
# )
# expect(
# SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob).length,
# ).to eq(file_jobs.length)
end
end
@@ -134,70 +145,82 @@ describe Domain::Inkbunny::Job::LatestPostsJob do
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_search_1047334_before.json"
)
},
{
method: :get,
uri: api_submissions_1047334_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions_1047334_before.json"
"domain/inkbunny/job/api_search_1047334_before.json",
),
caused_by_entry_idx: 0
},
# {
# method: :get,
# uri: api_submissions_1047334_url,
# content_type: "application/json",
# contents:
# SpecUtil.read_fixture_file(
# "domain/inkbunny/job/api_submissions_1047334_before.json",
# ),
# caused_by_entry_idx: 0,
# },
{
method: :post,
uri: api_search_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_search_1047334_after.json"
)
},
{
method: :get,
uri: api_submissions_1047334_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions_1047334_after.json"
"domain/inkbunny/job/api_search_1047334_after.json",
),
caused_by_entry_idx: 2
}
]
},
# {
# method: :get,
# uri: api_submissions_1047334_url,
# content_type: "application/json",
# contents:
# SpecUtil.read_fixture_file(
# "domain/inkbunny/job/api_submissions_1047334_after.json",
# ),
# caused_by_entry_idx: 1,
# },
],
)
end
it "updates posts and files" do
perform_now({})
expect { perform_now({}) }.to(
change(Domain::Inkbunny::Post, :count).by(1).and(
change(Domain::Inkbunny::User, :count).by(1),
),
)
post_1047334 = Domain::Inkbunny::Post.find_by!(ib_post_id: 1_047_334)
file_1445274 = post_1047334.files.find_by!(ib_file_id: 1_445_274)
expect(file_1445274.md5_initial).to eq("0127e88651e73140718f3b8f7f2037d5")
expect(file_1445274.md5_full).to eq("aa0e22f86a9c345ead2bd711a1c91986")
expect(post_1047334.title).to eq("New Submission")
# file_1445274 = post_1047334.files.find_by!(ib_file_id: 1_445_274)
# expect(file_1445274.md5_initial).to eq("0127e88651e73140718f3b8f7f2037d5")
# expect(file_1445274.md5_full).to eq("aa0e22f86a9c345ead2bd711a1c91986")
file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
expect(file_jobs.size).to eq(1)
SpecUtil.clear_enqueued_jobs!(Domain::Inkbunny::Job::FileJob)
# file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
# expect(file_jobs.size).to eq(1)
# SpecUtil.clear_enqueued_jobs!(Domain::Inkbunny::Job::FileJob)
update_post_jobs =
SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::UpdatePostsJob)
expect(update_post_jobs.size).to eq(1)
expect(update_post_jobs[0][:args][0][:ib_post_ids]).to eq([1_047_334])
SpecUtil.clear_enqueued_jobs!(Domain::Inkbunny::Job::UpdatePostsJob)
# second perform should create the new file
expect { perform_now({}) }.to(
change(Domain::Inkbunny::Post, :count)
.by(0)
.and(change(Domain::Inkbunny::File, :count).by(1))
.and(change(Domain::Inkbunny::User, :count).by(0))
change(Domain::Inkbunny::Post, :count).by(0).and(
change(Domain::Inkbunny::User, :count).by(0),
),
)
post_1047334.reload
expect(post_1047334.files.count).to eq(2)
file_4680214 = post_1047334.files.find_by!(ib_file_id: 4_680_214)
expect(file_4680214.ib_file_id).to eq(4_680_214)
expect(file_4680214.md5_initial).to eq("9fbfbdf3cc6d8b3538b7edbfe36bde8c")
expect(file_4680214.md5_full).to eq("d2e30d953f4785e22c3d9c722249a974")
# expect(post_1047334.files.count).to eq(2)
# file_4680214 = post_1047334.files.find_by!(ib_file_id: 4_680_214)
# expect(file_4680214.ib_file_id).to eq(4_680_214)
# expect(file_4680214.md5_initial).to eq("9fbfbdf3cc6d8b3538b7edbfe36bde8c")
# expect(file_4680214.md5_full).to eq("d2e30d953f4785e22c3d9c722249a974")
file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
expect(file_jobs.size).to eq(1)
update_post_jobs =
SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::UpdatePostsJob)
expect(update_post_jobs.size).to eq(1)
expect(update_post_jobs[0][:args][0][:ib_post_ids]).to eq([1_047_334])
end
end
end

230
spec/jobs/domain/inkbunny/update_posts_job_spec.rb Normal file
View File

@@ -0,0 +1,230 @@
require "rails_helper"
describe Domain::Inkbunny::Job::UpdatePostsJob do
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
before { Scraper::ClientFactory.http_client_mock = http_client_mock }
context "when updating multiple posts" do
let(:api_submissions_url) do
"https://inkbunny.net/api_submissions.php?submission_ids=3104202,3104200,3104197&show_description=yes&show_writing=yes&show_pools=yes"
end
let! :log_entries do
SpecUtil.init_http_client_mock(
http_client_mock,
[
{
method: :get,
uri: api_submissions_url,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions.json",
),
},
],
)
end
let!(:user_thendyart) do
Domain::Inkbunny::User.create!(ib_user_id: 941_565, name: "ThendyArt")
end
let!(:user_seff) do
Domain::Inkbunny::User.create!(ib_user_id: 229_331, name: "Seff")
end
let!(:user_soulcentinel) do
Domain::Inkbunny::User.create!(ib_user_id: 349_747, name: "SoulCentinel")
end
let!(:post_3104202) do
Domain::Inkbunny::Post.create!(
ib_post_id: 3_104_202,
creator: user_thendyart,
title: "Phantom Touch - Page 25",
posted_at: Time.parse("2023-08-27 21:31:40.365597+02"),
last_file_updated_at: Time.parse("2023-08-27 21:30:06.222262+02"),
num_files: 1,
rating: "adult",
submission_type: "comic",
)
end
let!(:post_3104200) do
Domain::Inkbunny::Post.create!(
ib_post_id: 3_104_200,
creator: user_seff,
title: "Camp Pines Sketch Dump (Aug 2023)",
posted_at: Time.parse("2023-08-27 21:30:59.308046+02"),
last_file_updated_at: Time.parse("2023-08-27 21:26:14.049+02"),
num_files: 4,
rating: "adult",
submission_type: "picture_pinup",
)
end
let!(:post_3104197) do
Domain::Inkbunny::Post.create!(
ib_post_id: 3_104_197,
creator: user_soulcentinel,
title: "Comm - BJ bird",
posted_at: Time.parse("2023-08-27 21:29:37.995264+02"),
last_file_updated_at: Time.parse("2023-08-27 21:24:23.653306+02"),
num_files: 1,
rating: "adult",
submission_type: "picture_pinup",
)
end
it "updates posts with detailed information" do
perform_now(
{
ib_post_ids: [3_104_202, 3_104_200, 3_104_197],
caused_by_entry: nil,
},
)
# Check post details were updated
post_3104202.reload
expect(post_3104202.description).to match(/Vulk grabs Lyra/)
expect(post_3104202.writing).to eq("")
expect(post_3104202.num_views).to eq(202)
expect(post_3104202.deep_updated_at).to be_within(1.second).of(Time.now)
post_3104200.reload
expect(post_3104200.description).to match(/Some requests from my Patreon/)
expect(post_3104200.writing).to eq("")
expect(post_3104200.num_views).to eq(690)
expect(post_3104200.deep_updated_at).to be_within(1.second).of(Time.now)
# Check user details were updated
user_seff.reload
expect(user_seff.avatar_url_str).to eq(
"https://us.ib.metapix.net/usericons/large/176/176443_Seff_seffpfp.png",
)
# Check files were created
expect(post_3104200.files.count).to eq(4)
file_4652528 = post_3104200.files.find_by!(ib_file_id: 4_652_528)
expect(file_4652528.file_order).to eq(0)
expect(file_4652528.file_name).to eq("4652528_Seff_aug23sketches7-1.png")
expect(file_4652528.url_str).to eq(
"https://us.ib.metapix.net/files/full/4652/4652528_Seff_aug23sketches7-1.png",
)
expect(file_4652528.md5_initial).to eq("e1cf8388a8aa03e4f33dc442e8984e5b")
expect(file_4652528.md5_full).to eq("07946e8d485664704b316cb218805367")
# Check file jobs were enqueued
file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
expect(file_jobs.length).to eq(6)
expect(
file_jobs.map { |job| job[:args][0][:file].ib_file_id }.sort,
).to eq(
[4_652_528, 4_652_530, 4_652_531, 4_652_534, 4_652_535, 4_652_537],
)
file_jobs.each do |job|
expect(job[:args][0][:caused_by_entry]).to eq(log_entries[0])
end
end
end
context "when a post's files change" do
let(:api_submissions_url_before) do
"https://inkbunny.net/api_submissions.php?submission_ids=1047334&show_description=yes&show_writing=yes&show_pools=yes"
end
let(:api_submissions_url_after) { api_submissions_url_before }
let! :log_entries do
SpecUtil.init_http_client_mock(
http_client_mock,
[
{
method: :get,
uri: api_submissions_url_before,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions_1047334_before.json",
),
},
{
method: :get,
uri: api_submissions_url_after,
content_type: "application/json",
contents:
SpecUtil.read_fixture_file(
"domain/inkbunny/job/api_submissions_1047334_after.json",
),
},
],
)
end
let!(:user_zzreg) do
Domain::Inkbunny::User.create!(ib_user_id: 110_036, name: "zzreg")
end
let!(:post_1047334) do
Domain::Inkbunny::Post.create!(
ib_post_id: 1_047_334,
creator: user_zzreg,
title: "New Submission",
posted_at: Time.parse("2016-03-13 22:18:52.32319+01"),
last_file_updated_at: Time.parse("2016-03-13 22:18:52.32319+01"),
num_files: 1,
rating: "general",
submission_type: "picture_pinup",
)
end
it "updates post with new file information" do
# First update - initial state
perform_now({ ib_post_ids: [1_047_334], caused_by_entry: nil })
post_1047334.reload
expect(post_1047334.files.count).to eq(1)
file_1445274 = post_1047334.files.find_by!(ib_file_id: 1_445_274)
expect(file_1445274.md5_initial).to eq("0127e88651e73140718f3b8f7f2037d5")
expect(file_1445274.md5_full).to eq("aa0e22f86a9c345ead2bd711a1c91986")
expect(file_1445274.file_name).to eq(
"1445274_zzreg_sname-yellow-small-border.png",
)
# Second update - file has changed
perform_now({ ib_post_ids: [1_047_334], caused_by_entry: nil })
post_1047334.reload
expect(post_1047334.files.count).to eq(2)
expect(post_1047334.last_file_updated_at).to eq(
Time.parse("2023-09-14 19:07:45.735562+02"),
)
# Old file should still exist
expect(file_1445274.reload.attributes).to include(
"md5_initial" => "0127e88651e73140718f3b8f7f2037d5",
"md5_full" => "aa0e22f86a9c345ead2bd711a1c91986",
"file_name" => "1445274_zzreg_sname-yellow-small-border.png",
)
# New file should be created
file_4680214 = post_1047334.files.find_by!(ib_file_id: 4_680_214)
expect(file_4680214.attributes).to include(
"md5_initial" => "9fbfbdf3cc6d8b3538b7edbfe36bde8c",
"md5_full" => "d2e30d953f4785e22c3d9c722249a974",
"file_name" =>
"4680214_zzreg_how-to-photograph-snakes-15-1200x900-cropped.jpg",
)
# Check file jobs were enqueued for both updates
file_jobs = SpecUtil.enqueued_jobs(Domain::Inkbunny::Job::FileJob)
expect(file_jobs.length).to eq(2)
expect(
file_jobs.map { |job| job[:args][0][:file].ib_file_id }.sort,
).to eq([1_445_274, 4_680_214])
expect(file_jobs[0][:args][0][:caused_by_entry]).to eq(log_entries[0])
expect(file_jobs[1][:args][0][:caused_by_entry]).to eq(log_entries[1])
end
end
end

34
spec/policies/domain/inkbunny/post_policy_spec.rb Normal file
View File

@@ -0,0 +1,34 @@
require "rails_helper"
RSpec.describe Domain::Inkbunny::PostPolicy, type: :policy do
subject { described_class }
let(:post) { Domain::Inkbunny::Post.new }
context "for a visitor" do
let(:user) { nil }
let(:policy) { described_class.new(user, post) }
it { expect(policy).to permit_action(:show) }
it { expect(policy).to forbid_action(:view_file) }
it { expect(policy).to forbid_action(:view_scraper_metadata) }
end
context "for an admin" do
let(:user) { build(:user, :admin) }
let(:policy) { described_class.new(user, post) }
it { expect(policy).to permit_action(:show) }
it { expect(policy).to permit_action(:view_file) }
it { expect(policy).to permit_action(:view_scraper_metadata) }
end
context "for a regular user" do
let(:user) { build(:user) }
let(:policy) { described_class.new(user, post) }
it { expect(policy).to permit_action(:show) }
it { expect(policy).to forbid_action(:view_file) }
it { expect(policy).to forbid_action(:view_scraper_metadata) }
end
end