dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

user scripts improvements

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-25 00:13:57 +00:00
parent 2faa485a35
commit ccd5404a10
11 changed files with 339 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@@ -12,3 +12,4 @@ launch.json
settings.json
*.export
.devcontainer
user_scripts/dist

3
Dockerfile
View File

@@ -81,6 +81,9 @@ COPY . .
RUN RAILS_ENV=production bin/rails assets:precompile
RUN mkdir -p tmp/pids
# build user scripts
RUN yarn build:user-scripts
# create user with id=1000 gid=1000
RUN groupadd -g 1000 app && \
useradd -m -d /home/app -s /bin/bash -u 1000 -g 1000 app

34
app/controllers/domain/fa/api_controller.rb
View File

@@ -147,7 +147,7 @@ class Domain::Fa::ApiController < ApplicationController
url_name = params[:url_name]
exclude_url_name = params[:exclude_url_name]
user = Domain::Fa::User.find_by(url_name: url_name)
user = Domain::User::FaUser.find_by(url_name: url_name)
if user.nil?
render status: 404,
json: {
@@ -174,21 +174,23 @@ class Domain::Fa::ApiController < ApplicationController
not_followed_similar_users = nil
if exclude_url_name
exclude_folowed_by_user =
Domain::Fa::User.find_by(url_name: exclude_url_name)
Domain::User::FaUser.find_by(url_name: exclude_url_name)
not_followed_similar_users =
if exclude_folowed_by_user.nil?
# TODO - enqueue a manual UserFollowsJob for this user and have client
# re-try the request later
{
error: "user '#{exclude_url_name}' not found",
error_type: "exclude_user_not_found",
}
render status: 500,
json: {
error: "user '#{exclude_url_name}' not found",
error_type: "exclude_user_not_found",
}
elsif exclude_folowed_by_user.scanned_follows_at.nil?
{
error:
"user '#{exclude_url_name}' followers list hasn't been scanned",
error_type: "exclude_user_not_scanned",
}
render status: 500,
json: {
error:
"user '#{exclude_url_name}' followers list hasn't been scanned",
error_type: "exclude_user_not_scanned",
}
else
users_list_to_similar_list(
helpers.similar_users_by_followed(
@@ -327,9 +329,14 @@ class Domain::Fa::ApiController < ApplicationController
end
end
sig do
params(users_list: T::Array[Domain::User::FaUser]).returns(
T::Array[T::Hash[Symbol, T.untyped]],
)
end
def users_list_to_similar_list(users_list)
users_list.map do |user|
profile_thumb_url = user.avatar&.file_uri&.to_s
profile_thumb_url = user.avatar&.log_entry&.uri_str
profile_thumb_url ||
begin
profile_page_response = get_best_user_page_http_log_entry_for(user)
@@ -360,7 +367,8 @@ class Domain::Fa::ApiController < ApplicationController
name: user.name,
url_name: user.url_name,
profile_thumb_url: profile_thumb_url,
url: "https://www.furaffinity.net/user/#{user.url_name}/",
external_url: "https://www.furaffinity.net/user/#{user.url_name}/",
refurrer_url: request.base_url + helpers.domain_user_path(user),
}
end
end

16
app/controllers/user_scripts_controller.rb
View File

@@ -2,20 +2,22 @@
class UserScriptsController < ApplicationController
skip_before_action :authenticate_user!, only: [:get]
ALLOWED_SCRIPTS = %w[object_statuses.user.js furecs.user.js].freeze
def get
expires_in 1.hour, public: true
response.cache_control[:public] = false
response.cache_control[:private] = true
script = params[:script]
case script
when "furecs.user.js"
send_file(
Rails.root.join("user_scripts/furecs.user.js"),
type: "application/json",
)
else
unless ALLOWED_SCRIPTS.include?(script)
render status: 404, text: "not found"
return
end
send_file(
Rails.root.join("user_scripts/dist/#{script}"),
type: "application/javascript",
)
end
end

32
app/helpers/domain/fa/users_helper.rb
View File

@@ -86,14 +86,38 @@ module Domain::Fa::UsersHelper
)
end
# TODO - remove this once we've migrated similarity scores to new user model
def similar_users_by_followed(user, limit: 10, exclude_followed_by: nil)
if user.disco.nil?
old_user = Domain::Fa::User.find_by(url_name: user.url_name)
old_exclude_user =
(
if exclude_followed_by
Domain::Fa::User.find_by(url_name: exclude_followed_by.url_name)
else
nil
end
)
return nil if old_user.nil?
if old_user.disco.nil?
nil
else
ReduxApplicationRecord.connection.execute("SET ivfflat.probes = 32")
user.similar_users_by_followed(
exclude_followed_by: exclude_followed_by,
).limit(limit)
old_users =
old_user.similar_users_by_followed(
exclude_followed_by: old_exclude_user,
).limit(limit)
old_user_url_names = old_users.map(&:url_name)
new_users = Domain::User::FaUser.where(url_name: old_user_url_names).to_a
# return in same order as old_users
old_users
.map do |old_user|
new_users.find { |new_user| new_user.url_name == old_user.url_name }
end
.compact
end
end

2
justfile
View File

@@ -26,4 +26,4 @@ tapioca *args:
bundle _2.5.6_ exec tapioca {{args}}
user_scripts:
yarn run tsc --project user_scripts/tsconfig.json
yarn build:user-scripts

3
package.json
View File

@@ -64,6 +64,7 @@
},
"scripts": {
"build:css": "tailwindcss -c ./config/tailwind.config.js -i ./app/assets/stylesheets/application.tailwind.css -o ./app/assets/builds/tailwind.css --minify",
"build:css[debug]": "tailwindcss -c ./config/tailwind.config.js -i ./app/assets/stylesheets/application.tailwind.css -o ./app/assets/builds/tailwind.css"
"build:css[debug]": "tailwindcss -c ./config/tailwind.config.js -i ./app/assets/stylesheets/application.tailwind.css -o ./app/assets/builds/tailwind.css",
"build:user-scripts": "tsc --project user_scripts/tsconfig.json"
}
}

229
user_scripts/furecs.user.js
View File

@@ -1,229 +0,0 @@
// ==UserScript==
// @name FuRecs
// @namespace https://twitter.com/DeltaNoises
// @version 1.3
// @description FurAffinity User Recommender
// @author https://twitter.com/DeltaNoises
// @match https://www.furaffinity.net/user/*
// @icon https://www.google.com/s2/favicons?sz=64&domain=furaffinity.net
// @grant GM_xmlhttpRequest
// @grant GM.getValue
// @grant GM.setValue
// @connect refurrer.com
// @updateURL https://refurrer.com/us/furecs.user.js
// @downloadURL https://refurrer.com/us/furecs.user.js
// ==/UserScript==
"use strict";
const API_HOST = "https://refurrer.com";
function urlNameFromUserHref(href) {
const url = new URL(href);
if (url.host != "www.furaffinity.net") {
return null;
}
const userPageRegex = /^\/(user|gallery|scraps|favorites|journals)\/.+/;
const match = url.pathname.match(userPageRegex);
if (!match) {
return null;
}
return url.pathname.split("/")[2];
}
async function fa() {
if (!window.location.pathname.startsWith("/user/")) {
return null;
}
const urlName = urlNameFromUserHref(window.location.href);
if (!urlName) {
return;
}
// if the user is logged in, get their user account, so we can
// also provide a list of recommendations that don't overlap with their
// current watchlist
let loggedInUrlName = null;
const loggedInLink = document.querySelector(
".mobile-nav-content-container h2 a"
);
if (loggedInLink) {
loggedInUrlName = urlNameFromUserHref(loggedInLink.href);
}
function buildRecommendedContainer() {
let leftColumn = document.querySelector(
".userpage-layout-left-col-content"
);
if (!leftColumn) {
console.log("didn't find left column container, bailing");
return [];
}
let section = document.createElement("section");
section.classList = "userpage-left-column gallery_container";
section.innerHTML = `
<div class="userpage-section-left">
<div class="section-header" style='display:flex;align-items:center'>
<h2 style='flex-grow:1'>Similar Users</h2>
</div>
<div class="section-body">
</div>
</div>`;
leftColumn.prepend(section);
return [
section.querySelector(".section-body"),
section.querySelector(".section-header"),
];
}
let [container, header] = buildRecommendedContainer();
if (!container) {
return;
}
console.log(`getting recommended follows for ${urlName}`);
function addApiTokenSetForm() {
const form = document.createElement("form");
form.innerHTML = `
<input style='height:32px;min-width:20em' required type=text placeholder="API Token" name="api_token" />
<input class='button' type=submit value="Save" />
`;
header.appendChild(form);
const input = form.querySelector("input[type=text]");
form.onsubmit = async () => {
apiToken = input.value;
await GM.setValue("apiToken", apiToken);
header.removeChild(form);
addApiTokenClearButton();
run();
};
}
function addApiTokenClearButton() {
const button = document.createElement("button");
const apiTokenShort =
apiToken.length > 10 ? apiToken.slice(0, 7) + "..." : apiToken;
button.innerHTML = `Clear Token (${apiTokenShort})`;
header.appendChild(button);
button.onclick = async () => {
apiToken = null;
await GM.setValue("apiToken", null);
header.removeChild(button);
addApiTokenSetForm();
};
}
let apiToken = await GM.getValue("apiToken", null);
if (apiToken == null) {
container.innerHTML = `<h2 class='aligncenter'>Please set API token</h2>`;
addApiTokenSetForm();
} else {
addApiTokenClearButton();
run();
}
function run() {
container.innerHTML = `<h2 class='aligncenter'>(Loading...)</h2>`;
let url = `${API_HOST}/api/fa/similar_users`;
url += `?url_name=${encodeURIComponent(urlName)}`;
if (loggedInUrlName) {
url += `&exclude_url_name=${encodeURIComponent(loggedInUrlName)}`;
}
url += `&api_token=${encodeURIComponent(apiToken)}`;
function setContainerError(error) {
container.innerHTML = `Error: <pre>${error}</pre>`;
}
GM_xmlhttpRequest({
url,
method: "GET",
headers: {
Accept: "application/json",
"Content-Type": "application/json",
},
onload: (response) => {
const json_error_codes = [403, 404, 500];
if (response.status == 200) {
let json = JSON.parse(response.response);
populateRecommendations(json);
} else if (json_error_codes.includes(response.status)) {
let json = JSON.parse(response.response);
setContainerError(json.error);
} else {
setContainerError(response.response);
}
},
});
}
function populateRecommendations(recommendations) {
if (recommendations.not_followed) {
container.innerHTML = buildTwoColumns(recommendations);
} else {
container.innerHTML = buildOneColumn(recommendations.all);
}
}
function buildTwoColumns({ all, not_followed }) {
let content = ``;
content += `<table>`;
content += ` <tr>`;
content += ` <th>Similar to ${urlName}</th>`;
content += ` <th>Users you don't follow</th>`;
content += ` </tr>`;
content += ` <tr>`;
content += ` <td>`;
content += ` ` + buildOneColumn(all);
content += ` </td>`;
content += ` <td>`;
if (not_followed.error != null) {
content +=
` Error getting recommended users ` +
`you don't already follow: ${not_followed.error}`;
} else {
content += ` ` + buildOneColumn(not_followed);
}
content += ` </td>`;
content += ` </tr>`;
content += `</table>`;
return content;
}
function buildOneColumn(userList) {
let content = ``;
content += '<ol class="user-submitted-links">';
userList.forEach(({ name, url, profile_thumb_url }) => {
content += `<li>`;
content += `<a
href="${url}"
style="display: flex; align-items: center; border: dotted 1px rgba(255,255,255,0.5); padding: 4px"
>`;
if (profile_thumb_url) {
content += `<img
alt="${name} thumbnail"
title="${name}"
src="${profile_thumb_url}"
style="max-height: 40px; padding-right: 10px"
/>`;
}
content += `<span>${name}</span>`;
content += `</a>`;
content += "</li>";
});
content += "</ol>";
return content;
}
}
(async function () {
if (window.location.hostname == "www.furaffinity.net") {
await fa();
} else {
console.log("unhandled domain ", window.location.hostname);
}
})();

238
user_scripts/furecs.user.ts Normal file
View File

@@ -0,0 +1,238 @@
// ==UserScript==
// @name FuRecs
// @namespace https://twitter.com/DeltaNoises
// @version 1.4
// @description FurAffinity User Recommender
// @author https://twitter.com/DeltaNoises
// @match https://www.furaffinity.net/user/*
// @icon https://www.google.com/s2/favicons?sz=64&domain=furaffinity.net
// @grant GM_xmlhttpRequest
// @grant GM.getValue
// @grant GM.setValue
// @connect refurrer.com
// @connect localhost:3001
// @connect localhost
// @updateURL https://refurrer.com/us/furecs.user.js
// @downloadURL https://refurrer.com/us/furecs.user.js
// ==/UserScript==
'use strict';
// const HOST = 'https://refurrer.com';
const HOST = 'http://localhost:3001';
interface SimilarUserList {
name: string;
url_name: string;
profile_thumb_url: string;
external_url: string;
refurrer_url: string;
}
type SimilarUsersErrorType =
| 'recs_not_computed'
| 'user_not_found'
| 'exclude_user_not_found'
| 'exclude_user_not_scanned';
type SimilarUsersResponse =
| {
all: SimilarUserList[];
not_followed: SimilarUserList[] | null;
}
| {
error_type: SimilarUsersErrorType;
error: string;
};
function urlNameFromUserHref(href: string) {
const url = new URL(href);
if (url.host != 'www.furaffinity.net') {
return null;
}
const userPageRegex = /^\/(user|gallery|scraps|favorites|journals)\/.+/;
const match = url.pathname.match(userPageRegex);
if (!match) {
return null;
}
return url.pathname.split('/')[2];
}
async function faUserRecs(): Promise<void> {
if (!window.location.pathname.startsWith('/user/')) {
return null;
}
const urlName = urlNameFromUserHref(window.location.href);
if (!urlName) {
return;
}
// if the user is logged in, get their user account, so we can
// also provide a list of recommendations that don't overlap with their
// current watchlist
let loggedInUrlName: string | null = null;
const loggedInLink = document.querySelector(
'.mobile-nav-content-container h2 a',
) as HTMLAnchorElement | null;
if (loggedInLink) {
loggedInUrlName = urlNameFromUserHref(loggedInLink.href);
}
function buildRecommendedContainer(): HTMLElement | null {
let leftColumn = document.querySelector(
'.userpage-layout-left-col-content',
) as HTMLElement | null;
if (!leftColumn) {
console.error("didn't find left column container, bailing");
return null;
}
let section = document.createElement('section');
section.classList.add('userpage-left-column');
section.classList.add('gallery_container');
section.innerHTML = `
<div class="userpage-section-left">
<div class="section-header" style='display:flex;align-items:center'>
<h2 style='flex-grow:1'>Similar Users</h2>
</div>
<div class="section-body">
</div>
</div>`;
leftColumn.prepend(section);
return section.querySelector('.section-body');
}
const container = buildRecommendedContainer();
if (!container) {
console.error('failed to build recommended container, bailing');
return;
}
function setContainerError(error: string) {
container.innerHTML = `Error: <pre>${error}</pre>`;
}
async function requestSimilarUsers(): Promise<SimilarUsersResponse> {
return new Promise((resolve, reject) => {
container.innerHTML = `<h2 class='aligncenter'>(Loading...)</h2>`;
let url = `${HOST}/api/fa/similar_users`;
url += `?url_name=${encodeURIComponent(urlName)}`;
if (loggedInUrlName) {
url += `&exclude_url_name=${encodeURIComponent(loggedInUrlName)}`;
}
GM_xmlhttpRequest({
url,
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
onabort: () => {
reject(new Error('request aborted'));
},
ontimeout: () => {
reject(new Error('request timed out'));
},
onerror: (response) => {
reject(new Error('server error: ' + response.responseText));
},
onload: (response) => {
const json_error_codes = [403, 404, 500];
if (response.status == 200) {
let json = JSON.parse(response.responseText);
resolve(json);
} else if (json_error_codes.includes(response.status)) {
let json = JSON.parse(response.responseText);
reject(new Error(json.error));
} else {
reject(new Error(response.responseText));
}
},
});
});
}
function populateRecommendations(recommendations: SimilarUsersResponse) {
if ('error' in recommendations) {
setContainerError(recommendations.error);
} else {
container.innerHTML = buildTwoColumns({
all: recommendations.all,
not_followed: recommendations.not_followed,
});
}
}
function buildTwoColumns({
all,
not_followed,
}: {
all: SimilarUserList[];
not_followed: SimilarUserList[] | null;
}) {
const width = not_followed ? '50%' : '100%';
let content = ``;
content += `<table>`;
content += ` <tr>`;
content += ` <th>Similar to ${urlName}</th>`;
if (not_followed) {
content += ` <th>Similar users you don't follow</th>`;
}
content += ` </tr>`;
content += ` <tr>`;
content += ` <td style='width:${width}'>`;
content += ` ` + buildOneColumn(all);
content += ` </td>`;
if (not_followed) {
content += ` <td style='width:${width}'>`;
content += ` ` + buildOneColumn(not_followed);
content += ` </td>`;
}
content += ` </tr>`;
content += `</table>`;
return content;
}
function buildOneColumn(userList: SimilarUserList[]) {
let content = ``;
content += '<ol class="user-submitted-links">';
userList.forEach(({ name, external_url, profile_thumb_url }) => {
content += `<li>`;
content += `<a
href="${external_url}"
target="_blank"
style="display: flex; align-items: center; border: dotted 1px rgba(255,255,255,0.5); padding: 4px"
>`;
if (profile_thumb_url) {
content += `<img
alt="${name} thumbnail"
title="${name}"
src="${profile_thumb_url}"
style="max-height: 40px; padding-right: 10px"
/>`;
}
content += `<span>${name}</span>`;
content += `</a>`;
content += '</li>';
});
content += '</ol>';
return content;
}
await requestSimilarUsers()
.then(populateRecommendations)
.catch((error) => {
setContainerError(error.message);
});
}
(async function () {
if (window.location.hostname == 'www.furaffinity.net') {
await faUserRecs();
} else {
console.log('unhandled domain ', window.location.hostname);
}
})();

38
user_scripts/enqueuer.user.ts → user_scripts/object_statuses.user.ts
View File

@@ -1,7 +1,7 @@
// ==UserScript==
// @name Refurrer Object Statuses
// @namespace http://tampermonkey.net/
// @version 1.0
// @version 2.0
// @description Show FA object statuses in Refurrer
// @author You
// @match https://www.furaffinity.net/*
@@ -12,32 +12,11 @@
// @connect refurrer.com
// @connect localhost:3001
// @connect localhost
// @updateURL https://refurrer.com/us/object_statuses.user.js
// @downloadURL https://refurrer.com/us/object_statuses.user.js
// ==/UserScript==
// This file is generated from a Typescript file, do not edit the js file directly.
'use strict';
declare const unsafeWindow: Window &
typeof globalThis & {
submission_data: Record<string, any>;
};
interface GMResponse {
readyState: number;
responseHeaders: string;
responseText: string;
status: number;
statusText: string;
}
declare const GM_xmlhttpRequest: (details: {
url: string;
method: string;
headers?: Record<string, string>;
data?: string;
onload: (response: GMResponse) => void;
onabort: (response: GMResponse) => void;
ontimeout: (response: GMResponse) => void;
onerror: (response: GMResponse) => void;
}) => void;
type PostObjectState =
| 'not_seen'
@@ -90,12 +69,11 @@ interface LiveEntityStatTable {
}[];
}
async function fa() {
async function faObjectStatuses() {
function isNotNull<T>(val: T | null | undefined): val is T {
return val != null;
}
// const HOST = 'refurrer.com';
const HOST = 'refurrer.com';
const LIGHT_BG_COLOR = '#353b45';
const DARK_BG_COLOR = '#20242a';
@@ -805,7 +783,6 @@ async function fa() {
function fetchObjectStatuses(): Promise<ObjectStatusesResponse> {
return new Promise((resolve, reject) => {
console.log('fetching object statuses');
let url = `http://${HOST}/api/fa/object_statuses`;
let params = new URLSearchParams();
faIds.forEach((id) => params.append('fa_ids[]', id.toString()));
@@ -834,7 +811,6 @@ async function fa() {
);
},
onload: (response) => {
console.log('response: ', response);
if (response.status === 200) {
const jsonResponse = JSON.parse(
response.responseText,
@@ -1023,13 +999,15 @@ async function fa() {
.then(handleObjectStatusesResponse)
.catch((err) => {
console.error('error: ', err);
navbarLiveEntityStatTablesNode.innerHTML =
'<div style="color: red;">Error: ' + err.message + '</div>';
});
}
(async function () {
if (window.location.hostname == 'www.furaffinity.net') {
await fa();
await faObjectStatuses();
} else {
console.log('unhandled domain ', window.location.hostname);
console.error('unhandled domain ', window.location.hostname);
}
})();

28
user_scripts/userscript_api.ts Normal file
View File

@@ -0,0 +1,28 @@
declare const unsafeWindow: Window &
typeof globalThis & {
submission_data: Record<string, any>;
};
interface GMResponse {
readyState: number;
responseHeaders: string;
responseText: string;
status: number;
statusText: string;
}
declare const GM_xmlhttpRequest: (details: {
url: string;
method: string;
headers?: Record<string, string>;
data?: string;
onload: (response: GMResponse) => void;
onabort: (response: GMResponse) => void;
ontimeout: (response: GMResponse) => void;
onerror: (response: GMResponse) => void;
}) => void;
declare const GM: {
getValue: (key: string, defaultValue?: string) => Promise<string | null>;
setValue: (key: string, value: string) => Promise<void>;
};