user scripts improvements
This commit is contained in:
Dylan Knutson
1
.gitignore
vendored
1
.gitignore
vendored
@@ -11,6 +11,7 @@ core
|
||||
lib/xdiff
|
||||
ext/xdiff/Makefile
|
||||
ext/xdiff/xdiff
|
||||
user_scripts/dist
|
||||
|
||||
# use yarn to manage node_modules
|
||||
package-lock.json
|
||||
|
||||
4
TODO.md
4
TODO.md
@@ -27,3 +27,7 @@
|
||||
- [x] Parse E621 source url for fa users
|
||||
- [ ] Parse BBCode in post descriptions
|
||||
- example post with bbcode: https://refurrer.com/posts/ib/3452498
|
||||
- [ ] Show tags on fa posts, ib posts
|
||||
- [ ] Sofurry implmentation
|
||||
- [ ] Make unified Static file job
|
||||
- [ ] Make unified Avatar file job
|
||||
|
||||
@@ -6,7 +6,8 @@ class Domain::Fa::ApiController < ApplicationController
|
||||
skip_before_action :verify_authenticity_token,
|
||||
only: %i[enqueue_objects object_statuses]
|
||||
|
||||
skip_before_action :validate_api_token!, only: %i[search_user_names]
|
||||
skip_before_action :validate_api_token!,
|
||||
only: %i[search_user_names object_statuses]
|
||||
|
||||
def search_user_names
|
||||
name = params[:name]
|
||||
@@ -20,28 +21,20 @@ class Domain::Fa::ApiController < ApplicationController
|
||||
end
|
||||
|
||||
def object_statuses
|
||||
fa_ids = (params[:fa_ids] || []).map(&:to_i)
|
||||
url_names = (params[:url_names] || [])
|
||||
fa_ids = (params[:fa_ids] || []).reject(&:blank?).map(&:to_i)
|
||||
url_names = (params[:url_names] || []).reject(&:blank?)
|
||||
|
||||
jobs_async =
|
||||
GoodJob::Job
|
||||
.select(:id, :queue_name, :serialized_params)
|
||||
.where(queue_name: "manual", finished_at: nil)
|
||||
.where(
|
||||
[
|
||||
"(serialized_params->'exception_executions' = '{}')",
|
||||
"(serialized_params->'exception_executions' is null)",
|
||||
].join(" OR "),
|
||||
)
|
||||
.load_async
|
||||
|
||||
users_async = Domain::Fa::User.where(url_name: url_names).load_async
|
||||
url_name_to_user =
|
||||
Domain::User::FaUser
|
||||
.where(url_name: url_names)
|
||||
.map { |user| [T.must(user.url_name), user] }
|
||||
.to_h
|
||||
|
||||
fa_id_to_post =
|
||||
Domain::Fa::Post
|
||||
Domain::Post::FaPost
|
||||
.includes(:file)
|
||||
.where(fa_id: fa_ids)
|
||||
.map { |post| [post.fa_id, post] }
|
||||
.map { |post| [T.must(post.fa_id), post] }
|
||||
.to_h
|
||||
|
||||
posts_response = {}
|
||||
@@ -50,96 +43,64 @@ class Domain::Fa::ApiController < ApplicationController
|
||||
fa_ids.each do |fa_id|
|
||||
post = fa_id_to_post[fa_id]
|
||||
|
||||
post_response =
|
||||
T.let(
|
||||
{
|
||||
terminal_state: false,
|
||||
seen_at: time_ago_or_never(post&.created_at),
|
||||
scanned_at: "never",
|
||||
downloaded_at: "never",
|
||||
},
|
||||
T::Hash[Symbol, T.untyped],
|
||||
)
|
||||
|
||||
if post
|
||||
post_response[:info_url] = domain_fa_post_url(fa_id: post.fa_id)
|
||||
post_response[:scanned_at] = time_ago_or_never(post.scanned_at)
|
||||
post_state =
|
||||
if post.file.present?
|
||||
"have_file"
|
||||
elsif post.scanned_at?
|
||||
"scanned_post"
|
||||
else
|
||||
post.state
|
||||
end
|
||||
|
||||
if post.file.present?
|
||||
post_response[:downloaded_at] = time_ago_or_never(
|
||||
post.file&.created_at,
|
||||
)
|
||||
post_response[:state] = "have_file"
|
||||
post_response[:terminal_state] = true
|
||||
elsif post.scanned?
|
||||
post_response[:state] = "scanned_post"
|
||||
else
|
||||
post_response[:state] = post.state
|
||||
end
|
||||
post_response = {
|
||||
state: post_state,
|
||||
seen_at: time_ago_or_never(post.created_at),
|
||||
object_url: request.base_url + helpers.domain_post_path(post),
|
||||
post_scan: {
|
||||
last_at: time_ago_or_never(post.scanned_at),
|
||||
due_for_scan: !post.scanned_at?,
|
||||
},
|
||||
file_scan: {
|
||||
last_at: time_ago_or_never(post.file&.created_at),
|
||||
due_for_scan: !post.file&.created_at?,
|
||||
},
|
||||
}
|
||||
else
|
||||
post_response[:state] = "not_seen"
|
||||
post_response = { state: "not_seen" }
|
||||
end
|
||||
|
||||
posts_response[fa_id] = post_response
|
||||
end
|
||||
|
||||
url_name_to_user = users_async.map { |user| [user.url_name, user] }.to_h
|
||||
|
||||
url_names.each do |url_name|
|
||||
user = url_name_to_user[url_name]
|
||||
|
||||
if user
|
||||
user_response = {
|
||||
created_at: time_ago_or_never(user.created_at),
|
||||
scanned_gallery_at: time_ago_or_never(user.scanned_gallery_at),
|
||||
scanned_page_at: time_ago_or_never(user.scanned_page_at),
|
||||
state: user.state,
|
||||
object_url: request.base_url + helpers.domain_user_path(user),
|
||||
page_scan: {
|
||||
last_at: time_ago_or_never(user.scanned_page_at),
|
||||
due_for_scan: user.due_for_page_scan?,
|
||||
},
|
||||
gallery_scan: {
|
||||
last_at: time_ago_or_never(user.scanned_gallery_at),
|
||||
due_for_scan: user.due_for_gallery_scan?,
|
||||
},
|
||||
favs_scan: {
|
||||
last_at: time_ago_or_never(user.scanned_favs_at),
|
||||
due_for_scan: user.due_for_favs_scan?,
|
||||
},
|
||||
}
|
||||
states = []
|
||||
states << "page" unless user.due_for_page_scan?
|
||||
states << "gallery" unless user.due_for_gallery_scan?
|
||||
states << "seen" if states.empty?
|
||||
|
||||
user_response[:state] = states.join(",")
|
||||
|
||||
if user.scanned_gallery_at && user.scanned_page_at
|
||||
user_response[:terminal_state] = true
|
||||
end
|
||||
else
|
||||
user_response = { state: "not_seen", terminal_state: false }
|
||||
user_response = { state: "not_seen" }
|
||||
end
|
||||
users_response[url_name] = user_response
|
||||
end
|
||||
|
||||
queue_depths = Hash.new { |hash, key| hash[key] = 0 }
|
||||
|
||||
jobs_async.each do |job|
|
||||
queue_depths[job.serialized_params["job_class"]] += 1
|
||||
end
|
||||
|
||||
queue_depths =
|
||||
queue_depths
|
||||
.map do |key, value|
|
||||
[
|
||||
key
|
||||
.delete_prefix("Domain::Fa::Job::")
|
||||
.split("::")
|
||||
.last
|
||||
.underscore
|
||||
.delete_suffix("_job")
|
||||
.gsub("_", " "),
|
||||
value,
|
||||
]
|
||||
end
|
||||
.to_h
|
||||
|
||||
render json: {
|
||||
posts: posts_response,
|
||||
users: users_response,
|
||||
queues: {
|
||||
total_depth: queue_depths.values.sum,
|
||||
depths: queue_depths,
|
||||
},
|
||||
}
|
||||
render json: { posts: posts_response, users: users_response }
|
||||
end
|
||||
|
||||
def enqueue_objects
|
||||
|
||||
@@ -36,8 +36,8 @@ module Domain::DescriptionsHelper
|
||||
|
||||
sig { params(text: String, url: String).returns(T::Boolean) }
|
||||
def text_same_as_url?(text, url)
|
||||
text = text.strip
|
||||
url = url.strip
|
||||
text = text.strip.downcase
|
||||
url = url.strip.downcase
|
||||
["", "http://", "https://"].any? { |prefix| "#{prefix}#{text}" == url }
|
||||
end
|
||||
|
||||
@@ -48,11 +48,12 @@ module Domain::DescriptionsHelper
|
||||
|
||||
# profiles often contain bbcode, so first re-parse that
|
||||
# for some reason, lots of duplicate <br> tags appear as well
|
||||
html =
|
||||
T.cast(
|
||||
T.unsafe(html.gsub("<br>", "").strip).bbcode_to_html(false),
|
||||
String,
|
||||
)
|
||||
html = html.gsub("<br>", "").strip
|
||||
begin
|
||||
html = T.cast(T.unsafe(html).bbcode_to_html(false), String)
|
||||
rescue RuntimeError
|
||||
# if the bbcode is invalid, skip parsing it
|
||||
end
|
||||
|
||||
replacements = {}
|
||||
|
||||
@@ -99,7 +100,7 @@ module Domain::DescriptionsHelper
|
||||
if ALLOWED_EXTERNAL_LINK_DOMAINS.any? { |domain|
|
||||
url_matches_domain?(domain, url.host)
|
||||
}
|
||||
if text_same_as_url?(node.text, url.to_s)
|
||||
if node.text.blank? || text_same_as_url?(node.text, url.to_s)
|
||||
title = title_for_url(url.to_s)
|
||||
else
|
||||
title = node.text
|
||||
@@ -111,7 +112,7 @@ module Domain::DescriptionsHelper
|
||||
"domain/has_description_html/inline_link_external",
|
||||
locals: {
|
||||
url: url.to_s,
|
||||
text: title,
|
||||
title:,
|
||||
icon_path: icon_path_for_domain(url.host),
|
||||
},
|
||||
),
|
||||
@@ -150,5 +151,8 @@ module Domain::DescriptionsHelper
|
||||
sanitizer.node!(fragment)
|
||||
replacements.each { |node, replacement| node.replace(replacement) }
|
||||
raw fragment.to_html(preserve_newline: true)
|
||||
rescue StandardError
|
||||
# if anything goes wrong, bail out and don't display anything
|
||||
"(error generating description)"
|
||||
end
|
||||
end
|
||||
|
||||
@@ -32,6 +32,7 @@ module Domain::DomainsHelper
|
||||
behance.net
|
||||
gumroad.com
|
||||
bigcartel.com
|
||||
furaffinity.net
|
||||
].freeze
|
||||
|
||||
DOMAIN_TO_ICON_PATH =
|
||||
@@ -67,6 +68,7 @@ module Domain::DomainsHelper
|
||||
[%r{://(.*\.)?x.com/([^/]+)}, ->(match) { match[2] }],
|
||||
[%r{://(.*\.)?twitter.com/([^/]+)}, ->(match) { match[2] }],
|
||||
[%r{://(.*\.)?patreon.com/([^/]+)}, ->(match) { match[2] }],
|
||||
[%r{://(.*\.)?furaffinity.net/user/([^/]+)}, ->(match) { match[2] }],
|
||||
],
|
||||
T::Array[[Regexp, T.proc.params(match: MatchData).returns(String)]],
|
||||
)
|
||||
|
||||
@@ -7,5 +7,5 @@
|
||||
<% if local_assigns[:icon_path] %>
|
||||
<%= image_tag local_assigns[:icon_path], class: "w-4 h-4 inline-block rounded-sm" %>
|
||||
<% end %>
|
||||
<span><%= text %></span>
|
||||
<span><%= title %></span>
|
||||
<% end %>
|
||||
|
||||
@@ -15,6 +15,7 @@ Rails.application.routes.draw do
|
||||
namespace :fa do
|
||||
get :similar_users, to: "/domain/fa/api#similar_users"
|
||||
get :search_user_names, to: "/domain/fa/api#search_user_names"
|
||||
get :object_statuses, to: "/domain/fa/api#object_statuses"
|
||||
end
|
||||
end
|
||||
|
||||
@@ -144,7 +145,6 @@ Rails.application.routes.draw do
|
||||
|
||||
namespace :fa do
|
||||
post :enqueue_objects, to: "/domain/fa/api#enqueue_objects"
|
||||
post :object_statuses, to: "/domain/fa/api#object_statuses"
|
||||
end
|
||||
namespace :twitter do
|
||||
post :enqueue_objects, to: "/domain/twitter/api#enqueue_objects"
|
||||
|
||||
3
justfile
3
justfile
@@ -24,3 +24,6 @@ tc *args:
|
||||
|
||||
tapioca *args:
|
||||
bundle _2.5.6_ exec tapioca {{args}}
|
||||
|
||||
user_scripts:
|
||||
yarn run tsc --project user_scripts/tsconfig.json
|
||||
|
||||
@@ -1,878 +0,0 @@
|
||||
// ==UserScript==
|
||||
// @name Redux Enquerer
|
||||
// @namespace http://tampermonkey.net/
|
||||
// @version 1.0
|
||||
// @description Enqeue webpages into Redux for archiving
|
||||
// @author You
|
||||
// @match https://www.furaffinity.net/*
|
||||
// @match https://twitter.com/*
|
||||
// @icon https://www.google.com/s2/favicons?sz=64&domain=furaffinity.net
|
||||
// @grant GM_xmlhttpRequest
|
||||
// @grant unsafeWindow
|
||||
// @connect scraper
|
||||
// @connect localhost
|
||||
// ==/UserScript==
|
||||
"use strict";
|
||||
const HOST = "scraper:3000";
|
||||
// const HOST = "localhost:3000";
|
||||
|
||||
function fa() {
|
||||
function setupNavbar() {
|
||||
const navbarStatusNode = document.createElement("li");
|
||||
navbarStatusNode.classList = "lileft";
|
||||
navbarStatusNode.style.height = "100%";
|
||||
navbarStatusNode.style.display = "flex";
|
||||
|
||||
const navbar = document.querySelector("nav");
|
||||
if (navbar != null) {
|
||||
navbar.querySelector("ul > div").after(navbarStatusNode);
|
||||
return navbarStatusNode;
|
||||
} else {
|
||||
// watch pages, etc don't have a navbar - create one at the top of the page
|
||||
const center = document.body.querySelector("div[align=center]");
|
||||
center.prepend(navbarStatusNode);
|
||||
return navbarStatusNode;
|
||||
}
|
||||
}
|
||||
|
||||
function makeLargeStatusNode(opts = {}) {
|
||||
if (opts.type == null) {
|
||||
opts.type = "span";
|
||||
}
|
||||
if (opts.smaller == null) {
|
||||
opts.smaller = true;
|
||||
}
|
||||
if (opts.style == null) {
|
||||
opts.style = {};
|
||||
}
|
||||
|
||||
const statusNode = document.createElement(opts.type);
|
||||
statusNode.style.cssText =
|
||||
"margin-left: 5px; color: #b7b7b7!important; display: inline";
|
||||
if (opts.smaller) {
|
||||
statusNode.style.fontSize = "80%";
|
||||
}
|
||||
|
||||
for (const [property, value] of Object.entries(opts.style)) {
|
||||
statusNode.style.setProperty(property, value);
|
||||
}
|
||||
|
||||
statusNode.innerHTML = "(...)";
|
||||
return statusNode;
|
||||
}
|
||||
|
||||
function galleryFigureElements() {
|
||||
return [
|
||||
...document.querySelectorAll("#gallery-search-results figure"),
|
||||
...document.querySelectorAll("#gallery-browse figure"),
|
||||
...document.querySelectorAll("#gallery-favorites figure"),
|
||||
...document.querySelectorAll("#gallery-gallery figure"),
|
||||
...document.querySelectorAll("#gallery-latest-favorites figure"),
|
||||
...document.querySelectorAll("#gallery-latest-submissions figure"),
|
||||
...document.querySelectorAll("section.gallery.messagecenter figure"),
|
||||
];
|
||||
}
|
||||
|
||||
function gatherPostElements() {
|
||||
// if on a gallery page, links to the list of submissions
|
||||
const galleryFigures = galleryFigureElements().map((figure) => {
|
||||
const faId = parseInt(figure.id.split("-")[1]);
|
||||
const statusNode = document.createElement("p");
|
||||
statusNode.innerHTML = "(...)";
|
||||
const captionLabel = figure.querySelector("figcaption label p");
|
||||
const caption = figure.querySelector("figcaption");
|
||||
// gallery pages will include a figcaption
|
||||
if (captionLabel) {
|
||||
figure.style.height = `calc(${figure.style.height} + 20px)`;
|
||||
figure.querySelector("b").style.height = "auto";
|
||||
statusNode.style.cssText = "position:relative;bottom:2px;";
|
||||
captionLabel.after(statusNode);
|
||||
} else if (caption) {
|
||||
statusNode.style.cssText = "position:relative;bottom:2px;";
|
||||
caption.appendChild(statusNode);
|
||||
} else {
|
||||
const figcaption = document.createElement("div");
|
||||
figcaption.style.cssText =
|
||||
"display: block !important; position: absolute; bottom: 0px; font-size: 10px; width:100%";
|
||||
figcaption.appendChild(statusNode);
|
||||
figure.appendChild(figcaption);
|
||||
figure.style.height = `calc(${figure.style.height} + 20px)`;
|
||||
}
|
||||
|
||||
return { faId, statusNode };
|
||||
});
|
||||
|
||||
const featuredElem = [
|
||||
...document.querySelectorAll(".userpage-featured-title h2"),
|
||||
]
|
||||
.map((elem) => {
|
||||
// skip if it's a dynamically changing preview, `submissionDataElems` handles those
|
||||
if (elem.classList.contains("preview_title")) {
|
||||
return null;
|
||||
}
|
||||
const link = elem.querySelector("a");
|
||||
if (!link) {
|
||||
return null;
|
||||
}
|
||||
const faId = faIdFromViewHref(link.href);
|
||||
if (!faId) {
|
||||
return null;
|
||||
}
|
||||
const statusNode = makeLargeStatusNode();
|
||||
elem.appendChild(statusNode);
|
||||
|
||||
return {
|
||||
faId,
|
||||
statusNode,
|
||||
};
|
||||
})
|
||||
.filter(isNotNull);
|
||||
|
||||
let submissionDataElems = [];
|
||||
if (unsafeWindow.submission_data) {
|
||||
submissionDataElems = Object.entries(unsafeWindow.submission_data).map(
|
||||
([fa_id, _]) => {
|
||||
const statusNode = makeLargeStatusNode();
|
||||
return {
|
||||
faId: fa_id,
|
||||
statusNode,
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// watch for changes to preview user and swap out status node when that happens
|
||||
[...document.querySelectorAll("h2.preview_title")].forEach(
|
||||
(previewTitleNode) => {
|
||||
const swapOutNode = document.createElement("span");
|
||||
swapOutNode.classList = "swapper";
|
||||
previewTitleNode.appendChild(swapOutNode);
|
||||
const previewTitleLink = previewTitleNode.querySelector("a");
|
||||
|
||||
const observerCb = () => {
|
||||
const previewFaId = faIdFromViewHref(previewTitleLink.href);
|
||||
swapOutNode.innerHTML = "";
|
||||
const currentSubmissionElem = submissionDataElems.find(
|
||||
({ faId }) => faId == previewFaId
|
||||
);
|
||||
if (currentSubmissionElem) {
|
||||
swapOutNode.appendChild(currentSubmissionElem.statusNode);
|
||||
}
|
||||
};
|
||||
const observer = new MutationObserver(observerCb);
|
||||
observer.observe(previewTitleLink, { childList: true, subtree: true });
|
||||
observerCb();
|
||||
}
|
||||
);
|
||||
|
||||
// /view/<faId>/ page elements
|
||||
let primaryViewPageElems = [];
|
||||
let faId = faIdFromViewHref(window.location.href);
|
||||
if (faId) {
|
||||
primaryViewPageElems = [
|
||||
...document.querySelectorAll(".submission-title h2"),
|
||||
].map((elem) => {
|
||||
elem.querySelector("p").style.display = "inline";
|
||||
const statusNode = makeLargeStatusNode({ type: "h3" });
|
||||
elem.appendChild(statusNode);
|
||||
return {
|
||||
faId,
|
||||
statusNode,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
return [
|
||||
...galleryFigures,
|
||||
...submissionDataElems,
|
||||
...featuredElem,
|
||||
...primaryViewPageElems,
|
||||
];
|
||||
}
|
||||
|
||||
function gatherUserElements() {
|
||||
// if on a gallery / browse page, the creator links from those posts
|
||||
const userSubmissionLinks = galleryFigureElements()
|
||||
.map((figure) => {
|
||||
const userLinkElem = [...figure.querySelectorAll("figcaption a")]
|
||||
.map((elem) => ({
|
||||
elem: elem,
|
||||
urlName: urlNameFromUserHref(elem.href),
|
||||
}))
|
||||
.filter(({ elem, urlName }) => urlName != null)[0];
|
||||
|
||||
if (userLinkElem == null) {
|
||||
return null;
|
||||
}
|
||||
figure.querySelector("u").style.cssText = "display: block";
|
||||
|
||||
const statusNode = document.createElement("span");
|
||||
statusNode.style.cssText = "margin-left: 2px; color: #c1c1c1";
|
||||
statusNode.innerHTML = "(...)";
|
||||
userLinkElem.elem.after(statusNode);
|
||||
|
||||
return {
|
||||
urlName: userLinkElem.urlName,
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
})
|
||||
.filter(isNotNull);
|
||||
|
||||
// if on a /user/ page, the primary username element
|
||||
const userPageUrlName = urlNameFromUserHref(window.location.href);
|
||||
let userPageMain = userPageUrlName
|
||||
? [...document.querySelectorAll("h1 username")].map((elem) => {
|
||||
const statusNode = document.createElement("span");
|
||||
statusNode.innerHTML = "(...)";
|
||||
elem.after(statusNode);
|
||||
return {
|
||||
urlName: userPageUrlName,
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
})
|
||||
: [];
|
||||
|
||||
// comments made by users on posts or user pages
|
||||
let userComments = [...document.querySelectorAll("comment-username a")].map(
|
||||
(elem) => {
|
||||
const statusNode = makeLargeStatusNode();
|
||||
elem.after(statusNode);
|
||||
return {
|
||||
urlName: urlNameFromUserHref(elem.href),
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
// users mentioned in post descriptions or user bios
|
||||
let iconUsernames = [
|
||||
...document.querySelectorAll("a.iconusername, a.linkusername"),
|
||||
].map((elem) => {
|
||||
const statusNode = makeLargeStatusNode();
|
||||
elem.after(statusNode);
|
||||
return {
|
||||
urlName: urlNameFromUserHref(elem.href),
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
});
|
||||
|
||||
// watcher / watching lists on user page
|
||||
let watchersAndWatchList = [
|
||||
...document.querySelectorAll("a > span.artist_name"),
|
||||
].map((elem) => {
|
||||
const link = elem.parentNode;
|
||||
const statusNode = makeLargeStatusNode();
|
||||
link.after(statusNode);
|
||||
return {
|
||||
urlName: urlNameFromUserHref(link.href),
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
});
|
||||
|
||||
// users mentioned in the "hero" sections on a url page
|
||||
let submissionDataElems = [];
|
||||
|
||||
if (unsafeWindow.submission_data) {
|
||||
submissionDataElems = Object.entries(unsafeWindow.submission_data).map(
|
||||
([_, { lower }]) => {
|
||||
const statusNode = makeLargeStatusNode();
|
||||
return {
|
||||
urlName: lower,
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// watch for changes to preview user and swap out status node when that happens
|
||||
const previewUser = document.querySelector("a.preview_user");
|
||||
if (previewUser) {
|
||||
const swapOutNode = document.createElement("span");
|
||||
swapOutNode.classList = "swapper";
|
||||
previewUser.after(swapOutNode);
|
||||
|
||||
const observerCb = () => {
|
||||
const previewUrlName = urlNameFromUserHref(previewUser.href);
|
||||
swapOutNode.innerHTML = "";
|
||||
const currentSubmissionElem = submissionDataElems.find(
|
||||
({ urlName }) => urlName == previewUrlName
|
||||
);
|
||||
if (currentSubmissionElem) {
|
||||
swapOutNode.appendChild(currentSubmissionElem.statusNode);
|
||||
}
|
||||
};
|
||||
const observer = new MutationObserver(observerCb);
|
||||
observer.observe(previewUser, { childList: true, subtree: true });
|
||||
observerCb();
|
||||
}
|
||||
|
||||
// on a /view/ page, the name of the user
|
||||
const submissionContainerUserLinks = [
|
||||
...document.querySelectorAll(".submission-id-sub-container a"),
|
||||
].map((elem) => {
|
||||
const statusNode = makeLargeStatusNode({ smaller: false });
|
||||
elem.after(statusNode);
|
||||
return {
|
||||
urlName: urlNameFromUserHref(elem.href),
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
});
|
||||
|
||||
// on a /watchlist/by/ or /watchlist/to page
|
||||
const watchListUserLinks = [
|
||||
...document.querySelectorAll(".watch-list-items.watch-row a"),
|
||||
].map((elem) => {
|
||||
const statusNode = makeLargeStatusNode({
|
||||
smaller: false,
|
||||
style: {
|
||||
display: "block",
|
||||
"margin-bottom": "5px",
|
||||
"font-size": "50%",
|
||||
},
|
||||
});
|
||||
elem.parentNode.appendChild(statusNode);
|
||||
return {
|
||||
urlName: urlNameFromUserHref(elem.href),
|
||||
shouldEnqueue: true,
|
||||
statusNode,
|
||||
};
|
||||
});
|
||||
|
||||
return [
|
||||
...userSubmissionLinks,
|
||||
...userPageMain,
|
||||
...userComments,
|
||||
...iconUsernames,
|
||||
...watchersAndWatchList,
|
||||
...submissionDataElems,
|
||||
...submissionContainerUserLinks,
|
||||
...watchListUserLinks,
|
||||
];
|
||||
}
|
||||
|
||||
function isNotNull(val) {
|
||||
return val != null;
|
||||
}
|
||||
|
||||
function urlNameFromUserHref(href) {
|
||||
const url = new URL(href);
|
||||
if (url.host != "www.furaffinity.net") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const userPageRegex = /^\/(user|gallery|scraps|favorites|journals)\/.+/;
|
||||
const match = url.pathname.match(userPageRegex);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return url.pathname.split("/")[2];
|
||||
}
|
||||
|
||||
function faIdFromViewHref(href) {
|
||||
const viewPageRegex = /\/(view|full)\/(\d+)/;
|
||||
const match = href.match(viewPageRegex);
|
||||
const faId = (match && match[2]) || null;
|
||||
if (faId) {
|
||||
return parseInt(faId);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function renderTable(stats, styleOpts = {}) {
|
||||
if (stats.length == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (styleOpts["border-collapse"] == null) {
|
||||
styleOpts["border-collapse"] = "collapse";
|
||||
}
|
||||
|
||||
let table = document.createElement("table");
|
||||
for (const [property, value] of Object.entries(styleOpts)) {
|
||||
table.style.setProperty(property, value);
|
||||
}
|
||||
|
||||
let tbody = document.createElement("tbody");
|
||||
table.appendChild(tbody);
|
||||
|
||||
stats.each(({ name, value, sep, nameAlign, valueAlign }) => {
|
||||
if (name == "" && value == "") {
|
||||
tbody.innerHTML += `<tr><td>---</td><td>---</td></tr>`;
|
||||
} else {
|
||||
if (sep == null) {
|
||||
sep = ":";
|
||||
}
|
||||
if (nameAlign == null) {
|
||||
nameAlign = "right";
|
||||
}
|
||||
if (valueAlign == null) {
|
||||
valueAlign = "right";
|
||||
}
|
||||
|
||||
tbody.innerHTML += `<tr>
|
||||
<td style="text-align:${nameAlign};padding-right:5px">${name}${sep}</td>
|
||||
<td style="text-align:${valueAlign}">${value}</td>
|
||||
</tr>`;
|
||||
}
|
||||
});
|
||||
return table;
|
||||
}
|
||||
|
||||
function optsForNumRows(numRows) {
|
||||
switch (numRows) {
|
||||
case 0:
|
||||
case 1:
|
||||
case 2:
|
||||
case 3:
|
||||
return { "line-height": "1.0em", "font-size": "1.0em" };
|
||||
case 4:
|
||||
return { "line-height": "0.9em", "font-size": "0.8em" };
|
||||
default:
|
||||
return { "line-height": "0.9em", "font-size": "0.6em" };
|
||||
}
|
||||
}
|
||||
|
||||
const navbarNode = setupNavbar();
|
||||
const navbarPageStatsNode = document.createElement("div");
|
||||
const navbarEnqueueNode = document.createElement("div");
|
||||
const navbarLiveQueueNode = document.createElement("div");
|
||||
const navbarLiveEntityNode = document.createElement("div");
|
||||
|
||||
navbarPageStatsNode.innerHTML = "querying...";
|
||||
navbarEnqueueNode.innerHTML = "enqueueing...";
|
||||
navbarLiveQueueNode.innerHTML = "queue loading...";
|
||||
navbarLiveEntityNode.innerHTML = "entity loading...";
|
||||
|
||||
[
|
||||
navbarPageStatsNode,
|
||||
navbarEnqueueNode,
|
||||
navbarLiveQueueNode,
|
||||
navbarLiveEntityNode,
|
||||
].forEach((node) => {
|
||||
node.style.display = "flex";
|
||||
node.style.marginRight = "5px";
|
||||
});
|
||||
|
||||
[navbarPageStatsNode, navbarEnqueueNode, navbarLiveQueueNode].forEach(
|
||||
(node) => {
|
||||
node.style.paddingRight = "5px";
|
||||
node.style.borderRight = "1px solid #d7d7d7";
|
||||
}
|
||||
);
|
||||
|
||||
navbarNode.append(navbarPageStatsNode);
|
||||
navbarNode.append(navbarEnqueueNode);
|
||||
navbarNode.append(navbarLiveQueueNode);
|
||||
navbarNode.append(navbarLiveEntityNode);
|
||||
|
||||
const userElements = gatherUserElements();
|
||||
const urlNames = [...new Set(userElements.map(({ urlName }) => urlName))];
|
||||
const urlNamesToEnqueue = [
|
||||
...new Set(
|
||||
userElements
|
||||
.filter(({ shouldEnqueue }) => shouldEnqueue)
|
||||
.map(({ urlName }) => urlName)
|
||||
),
|
||||
];
|
||||
|
||||
const postElements = gatherPostElements();
|
||||
const faIds = [...new Set(postElements.map(({ faId }) => faId))];
|
||||
|
||||
function renderLiveQueueStats({ livePostsStats, liveQueueStats }) {
|
||||
let elemsCountsNode = document.createElement("div");
|
||||
elemsCountsNode.style.width = "100%";
|
||||
elemsCountsNode.style.height = "100%";
|
||||
elemsCountsNode.style.display = "flex";
|
||||
elemsCountsNode.style.flexDirection = "row";
|
||||
elemsCountsNode.style.gap = "1em";
|
||||
|
||||
navbarLiveQueueNode.innerHTML = "";
|
||||
navbarLiveQueueNode.appendChild(elemsCountsNode);
|
||||
|
||||
const postsStatsTable = renderTable(livePostsStats, {
|
||||
...optsForNumRows(livePostsStats.length),
|
||||
width: "auto",
|
||||
});
|
||||
|
||||
const queueStatsTable = renderTable(liveQueueStats, {
|
||||
...optsForNumRows(liveQueueStats.length),
|
||||
width: "auto",
|
||||
});
|
||||
|
||||
postsStatsTable && elemsCountsNode.appendChild(postsStatsTable);
|
||||
queueStatsTable && elemsCountsNode.appendChild(queueStatsTable);
|
||||
}
|
||||
|
||||
function renderLiveEntityStats(liveEntityStats) {
|
||||
const liveEntityStatsTable = renderTable(liveEntityStats, {
|
||||
...optsForNumRows(liveEntityStats.length),
|
||||
width: "auto",
|
||||
});
|
||||
navbarLiveEntityNode.innerHTML = "";
|
||||
liveEntityStatsTable &&
|
||||
navbarLiveEntityNode.appendChild(liveEntityStatsTable);
|
||||
}
|
||||
|
||||
let completedEnqueue = false;
|
||||
|
||||
GM_xmlhttpRequest({
|
||||
url: `http://${HOST}/api/fa/enqueue_objects`,
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: JSON.stringify({
|
||||
fa_ids: faIds,
|
||||
url_names: urlNames,
|
||||
url_names_to_enqueue: urlNamesToEnqueue,
|
||||
}),
|
||||
onload: (response) => {
|
||||
console.log("response: ", response);
|
||||
completedEnqueue = true;
|
||||
|
||||
if (response.status === 200) {
|
||||
const jsonResponse = JSON.parse(response.response);
|
||||
console.log("json: ", jsonResponse);
|
||||
handleEnqueueResponse(jsonResponse);
|
||||
} else {
|
||||
navbarLiveQueueNode.innerHTML = `<b>${response.status} enqueing</b>`;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
function handleEnqueueResponse(jsonResponse) {
|
||||
navbarEnqueueNode.innerHTML = "";
|
||||
|
||||
const enqueueStats = Object.entries(jsonResponse).map(([name, value]) => ({
|
||||
name: name.split("_").join(" "),
|
||||
value,
|
||||
}));
|
||||
const enqueueStatsTable = renderTable(enqueueStats, {
|
||||
...optsForNumRows(enqueueStats.length),
|
||||
width: "auto",
|
||||
});
|
||||
|
||||
enqueueStatsTable && navbarEnqueueNode.append(enqueueStatsTable);
|
||||
}
|
||||
|
||||
function pollLiveStats() {
|
||||
GM_xmlhttpRequest({
|
||||
url: `http://${HOST}/api/fa/object_statuses`,
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: JSON.stringify({
|
||||
fa_ids: faIds,
|
||||
url_names: urlNames,
|
||||
url_names_to_enqueue: urlNamesToEnqueue,
|
||||
}),
|
||||
onload: (response) => {
|
||||
console.log("response: ", response);
|
||||
if (response.status === 200) {
|
||||
const jsonResponse = JSON.parse(response.response);
|
||||
console.log("json: ", jsonResponse);
|
||||
const keepPolling = handleLiveStatsResponse(jsonResponse);
|
||||
if (!completedEnqueue || keepPolling) {
|
||||
setTimeout(() => pollLiveStats(), 2500);
|
||||
} else {
|
||||
console.log("reached terminal state");
|
||||
}
|
||||
} else {
|
||||
navbarStatusNode.innerHTML = `<b>${response.status} from scraper</b>`;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function handleLiveStatsResponse(jsonResponse) {
|
||||
let allTerminalState = true;
|
||||
|
||||
let numNotSeenPosts = 0;
|
||||
let numOkPosts = 0;
|
||||
let numScannedPosts = 0;
|
||||
let numHaveFile = 0;
|
||||
|
||||
for (const [gotFaId, postInfo] of Object.entries(jsonResponse.posts)) {
|
||||
allTerminalState = allTerminalState && postInfo.terminal_state;
|
||||
postElements
|
||||
.filter(({ faId }) => faId == gotFaId)
|
||||
.forEach(({ statusNode }) => {
|
||||
statusNode.innerHTML = postInfo.state;
|
||||
|
||||
switch (postInfo.state) {
|
||||
case "not_seen":
|
||||
numNotSeenPosts += 1;
|
||||
break;
|
||||
case "ok":
|
||||
numOkPosts += 1;
|
||||
break;
|
||||
case "scanned_post":
|
||||
numScannedPosts += 1;
|
||||
break;
|
||||
case "have_file":
|
||||
numHaveFile += 1;
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const [gotUrlName, userInfo] of Object.entries(jsonResponse.users)) {
|
||||
allTerminalState = allTerminalState && userInfo.terminal_state;
|
||||
userElements
|
||||
.filter(({ urlName }) => urlName == gotUrlName)
|
||||
.forEach(({ statusNode }) => {
|
||||
statusNode.innerHTML = userInfo.state;
|
||||
});
|
||||
}
|
||||
|
||||
const livePostsStats = [
|
||||
{ name: "not seen", value: numNotSeenPosts },
|
||||
{ name: "ok", value: numOkPosts },
|
||||
{ name: "scanned", value: numScannedPosts },
|
||||
{ name: "have file", value: numHaveFile },
|
||||
];
|
||||
|
||||
let liveQueueStats = Object.entries(jsonResponse.queues.depths).map(
|
||||
([queue, depth]) => ({ name: queue, value: depth })
|
||||
);
|
||||
|
||||
liveQueueStats = [
|
||||
{ name: "total depth", value: `${jsonResponse.queues.total_depth}` },
|
||||
...liveQueueStats,
|
||||
];
|
||||
|
||||
while (liveQueueStats.length < 4) {
|
||||
liveQueueStats.push({ name: "", value: "" });
|
||||
}
|
||||
|
||||
allTerminalState &&= jsonResponse.queues.total_depth == 0;
|
||||
|
||||
let liveEntityStats = [{ name: "no entity", value: "", sep: "" }];
|
||||
|
||||
const thisPageFaId = faIdFromViewHref(window.location.href);
|
||||
const pssCommon = { sep: "", valueAlign: "left" };
|
||||
if (thisPageFaId != null) {
|
||||
const postData = jsonResponse.posts[thisPageFaId];
|
||||
liveEntityStats = [
|
||||
{
|
||||
name: "link",
|
||||
value: `<a target="_blank" href="${postData.info_url}" style="text-decoration: underline dotted">${thisPageFaId}</a>`,
|
||||
...pssCommon,
|
||||
},
|
||||
{ name: `seen`, value: postData.seen_at, ...pssCommon },
|
||||
{ name: `scanned`, value: postData.scanned_at, ...pssCommon },
|
||||
{ name: `downloaded`, value: postData.downloaded_at, ...pssCommon },
|
||||
];
|
||||
}
|
||||
|
||||
const thisPageUrlName = urlNameFromUserHref(window.location.href);
|
||||
if (thisPageUrlName != null) {
|
||||
const userData = jsonResponse.users[thisPageUrlName];
|
||||
liveEntityStats = [
|
||||
{ name: "", value: thisPageUrlName, ...pssCommon },
|
||||
{ name: "first seen", value: userData.created_at, ...pssCommon },
|
||||
{ name: "page scan", value: userData.scanned_page_at, ...pssCommon },
|
||||
{
|
||||
name: "gallery scan",
|
||||
value: userData.scanned_gallery_at,
|
||||
...pssCommon,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
renderLiveQueueStats({
|
||||
livePostsStats,
|
||||
liveQueueStats,
|
||||
});
|
||||
renderLiveEntityStats(liveEntityStats);
|
||||
|
||||
return !allTerminalState;
|
||||
}
|
||||
|
||||
// right off, can render the page stats table
|
||||
const pageStatsTable = renderTable(
|
||||
[
|
||||
{ name: "page users", value: urlNames.length },
|
||||
{ name: "page posts", value: faIds.length },
|
||||
],
|
||||
{ ...optsForNumRows(2), width: "auto" }
|
||||
);
|
||||
navbarPageStatsNode.innerHTML = "";
|
||||
navbarPageStatsNode.append(pageStatsTable);
|
||||
|
||||
renderLiveQueueStats({
|
||||
livePostsStats: [
|
||||
{ name: "not seen", value: "---" },
|
||||
{ name: "ok", value: "---" },
|
||||
{ name: "scanned", value: "---" },
|
||||
{ name: "have file", value: "---" },
|
||||
],
|
||||
liveQueueStats: [
|
||||
{ name: "queue depths", value: "---" },
|
||||
{ name: "", value: "" },
|
||||
{ name: "", value: "" },
|
||||
{ name: "", value: "" },
|
||||
],
|
||||
});
|
||||
|
||||
pollLiveStats();
|
||||
}
|
||||
|
||||
function twitter(mainNode) {
|
||||
function onlyUnique(value, index, array) {
|
||||
return array.indexOf(value) === index;
|
||||
}
|
||||
|
||||
let seenTweets = new Set();
|
||||
let ignoreUsers = new Set(["elonmusk"]);
|
||||
let nameToStatusNodes = {};
|
||||
let nameToStatus = { Mangoyena: "yes!" };
|
||||
|
||||
function observerCallback() {
|
||||
// update status nodes for all the user links
|
||||
function parentN(node, times) {
|
||||
for (let i = 0; i < times; i++) {
|
||||
node = node.parentNode;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
[
|
||||
...document.querySelectorAll(
|
||||
"[data-testid=primaryColumn] a[role=link][tabindex='-1']:not(aria-hidden)"
|
||||
),
|
||||
]
|
||||
.filter(
|
||||
(link) =>
|
||||
link.hostname == "twitter.com" &&
|
||||
!link.pathname.startsWith("/i/") &&
|
||||
!link.querySelector("img")
|
||||
)
|
||||
.map((link) => ({
|
||||
elem: link,
|
||||
name: link.pathname.split("/")[1],
|
||||
header: parentN(link, 7),
|
||||
}))
|
||||
.forEach(({ elem, name, header }) => {
|
||||
if (nameToStatusNodes[name] == null) {
|
||||
nameToStatusNodes[name] = [];
|
||||
}
|
||||
if (header.querySelector(".reduxStatusNode") == null) {
|
||||
const statusNode = document.createElement("span");
|
||||
statusNode.style.cssText = "color:white; float: right";
|
||||
statusNode.classList = ["reduxStatusNode"];
|
||||
header.insertBefore(statusNode, header.children[1]);
|
||||
nameToStatusNodes[name].push(statusNode);
|
||||
}
|
||||
|
||||
const status = nameToStatus[name];
|
||||
nameToStatusNodes[name].forEach((statusNode) => {
|
||||
if (status != null) {
|
||||
statusNode.innerHTML = status;
|
||||
} else {
|
||||
statusNode.innerHTML = "(loading...)";
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const tweets = [...document.querySelectorAll("a[role=link]")]
|
||||
.filter(
|
||||
(link) =>
|
||||
link.hostname == "twitter.com" && link.pathname.includes("/status/")
|
||||
)
|
||||
.map((link) => {
|
||||
const parts = link.pathname.split("/");
|
||||
return {
|
||||
name: parts[1],
|
||||
tweetId: parts[3],
|
||||
};
|
||||
})
|
||||
.filter(({ name }) => !ignoreUsers.has(name))
|
||||
.map(JSON.stringify)
|
||||
.filter(onlyUnique);
|
||||
|
||||
let newTweets = [...tweets].filter((n) => !seenTweets.has(n));
|
||||
if (newTweets.length > 0) {
|
||||
newTweets
|
||||
.map(JSON.parse)
|
||||
.forEach(({ name, tweetId }) =>
|
||||
console.log("new tweet", `${name} / ${tweetId}`)
|
||||
);
|
||||
|
||||
newTweets.forEach((tweet) => seenTweets.add(tweet));
|
||||
enqueueNewTweets(newTweets);
|
||||
}
|
||||
}
|
||||
|
||||
const enqueuedTweets = new Set();
|
||||
let enqueueInProgress = false;
|
||||
function enqueueNewTweets(newTweets) {
|
||||
newTweets.forEach((tweet) => enqueuedTweets.add(tweet));
|
||||
|
||||
if (!enqueueInProgress && enqueuedTweets.size > 0) {
|
||||
enqueueInProgress = true;
|
||||
sendEnqueue([...enqueuedTweets].map(JSON.parse));
|
||||
enqueuedTweets.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// const enqueuedNames = new Set()
|
||||
function sendEnqueue(tweets) {
|
||||
// const newNames = tweets.map(({ name }) => name).filter(name => !enqueuedNames.has(name));
|
||||
console.log("enqueue tweets: ", tweets);
|
||||
|
||||
GM_xmlhttpRequest({
|
||||
url: `http://${HOST}/api/twitter/enqueue_objects`,
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: JSON.stringify({ names: tweets.map(({ name }) => name) }),
|
||||
onload: (response) => {
|
||||
if (response.status === 200) {
|
||||
const jsonResponse = JSON.parse(response.response);
|
||||
console.log("redux json: ", jsonResponse);
|
||||
|
||||
enqueueInProgress = false;
|
||||
enqueueNewTweets([]);
|
||||
} else {
|
||||
console.error("redux error: ", response.status, response);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function startObserving() {
|
||||
const observer = new MutationObserver(() => {
|
||||
observer.disconnect();
|
||||
observerCallback();
|
||||
startObserving();
|
||||
});
|
||||
|
||||
observer.observe(mainNode, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
}
|
||||
startObserving();
|
||||
}
|
||||
|
||||
(function () {
|
||||
if (window.location.hostname == "www.furaffinity.net") {
|
||||
fa();
|
||||
} else if (window.location.hostname == "twitter.com") {
|
||||
twitter(document.querySelector("body"));
|
||||
} else {
|
||||
console.log("unhandled domain ", window.location.hostname);
|
||||
}
|
||||
})();
|
||||
1035
user_scripts/enqueuer.user.ts
Normal file
1035
user_scripts/enqueuer.user.ts
Normal file
File diff suppressed because it is too large
Load Diff
15
user_scripts/tsconfig.json
Normal file
15
user_scripts/tsconfig.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2015",
|
||||
"module": "CommonJS",
|
||||
"lib": ["dom", "es6", "dom.iterable"],
|
||||
"outDir": "./dist",
|
||||
"sourceMap": true,
|
||||
"noImplicitAny": true,
|
||||
"baseUrl": ".",
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["*.ts"],
|
||||
"exclude": ["../app", "../node_modules", "dist"],
|
||||
"compileOnSave": true
|
||||
}
|
||||
Reference in New Issue
Block a user