user scripts improvements

.gitignore

@@ -11,6 +11,7 @@ core
 lib/xdiff
 ext/xdiff/Makefile
 ext/xdiff/xdiff
+user_scripts/dist
 
 # use yarn to manage node_modules
 package-lock.json

TODO.md

@@ -27,3 +27,7 @@
 - [x] Parse E621 source url for fa users
 - [ ] Parse BBCode in post descriptions
   - example post with bbcode: https://refurrer.com/posts/ib/3452498
+- [ ] Show tags on fa posts, ib posts
+- [ ] Sofurry implmentation
+- [ ] Make unified Static file job
+- [ ] Make unified Avatar file job

app/controllers/domain/fa/api_controller.rb

@@ -6,7 +6,8 @@ class Domain::Fa::ApiController < ApplicationController
   skip_before_action :verify_authenticity_token,
                      only: %i[enqueue_objects object_statuses]
 
-  skip_before_action :validate_api_token!, only: %i[search_user_names]
+  skip_before_action :validate_api_token!,
+                     only: %i[search_user_names object_statuses]
 
   def search_user_names
     name = params[:name]
@@ -20,28 +21,20 @@ class Domain::Fa::ApiController < ApplicationController
   end
 
   def object_statuses
-    fa_ids = (params[:fa_ids] || []).map(&:to_i)
+    fa_ids = (params[:fa_ids] || []).reject(&:blank?).map(&:to_i)
-    url_names = (params[:url_names] || [])
+    url_names = (params[:url_names] || []).reject(&:blank?)
 
-    jobs_async =
+    url_name_to_user =
-      GoodJob::Job
+      Domain::User::FaUser
-        .select(:id, :queue_name, :serialized_params)
+        .where(url_name: url_names)
-        .where(queue_name: "manual", finished_at: nil)
+        .map { |user| [T.must(user.url_name), user] }
-        .where(
+        .to_h
-          [
-            "(serialized_params->'exception_executions' = '{}')",
-            "(serialized_params->'exception_executions' is null)",
-          ].join(" OR "),
-        )
-        .load_async
-
-    users_async = Domain::Fa::User.where(url_name: url_names).load_async
 
     fa_id_to_post =
-      Domain::Fa::Post
+      Domain::Post::FaPost
         .includes(:file)
         .where(fa_id: fa_ids)
-        .map { |post| [post.fa_id, post] }
+        .map { |post| [T.must(post.fa_id), post] }
         .to_h
 
     posts_response = {}
@@ -50,96 +43,64 @@ class Domain::Fa::ApiController < ApplicationController
     fa_ids.each do |fa_id|
       post = fa_id_to_post[fa_id]
 
-      post_response =
-        T.let(
-          {
-            terminal_state: false,
-            seen_at: time_ago_or_never(post&.created_at),
-            scanned_at: "never",
-            downloaded_at: "never",
-          },
-          T::Hash[Symbol, T.untyped],
-        )
-
       if post
-        post_response[:info_url] = domain_fa_post_url(fa_id: post.fa_id)
+        post_state =
-        post_response[:scanned_at] = time_ago_or_never(post.scanned_at)
+          if post.file.present?
+            "have_file"
+          elsif post.scanned_at?
+            "scanned_post"
+          else
+            post.state
+          end
 
-        if post.file.present?
+        post_response = {
-          post_response[:downloaded_at] = time_ago_or_never(
+          state: post_state,
-            post.file&.created_at,
+          seen_at: time_ago_or_never(post.created_at),
-          )
+          object_url: request.base_url + helpers.domain_post_path(post),
-          post_response[:state] = "have_file"
+          post_scan: {
-          post_response[:terminal_state] = true
+            last_at: time_ago_or_never(post.scanned_at),
-        elsif post.scanned?
+            due_for_scan: !post.scanned_at?,
-          post_response[:state] = "scanned_post"
+          },
-        else
+          file_scan: {
-          post_response[:state] = post.state
+            last_at: time_ago_or_never(post.file&.created_at),
-        end
+            due_for_scan: !post.file&.created_at?,
+          },
+        }
       else
-        post_response[:state] = "not_seen"
+        post_response = { state: "not_seen" }
       end
 
       posts_response[fa_id] = post_response
     end
 
-    url_name_to_user = users_async.map { |user| [user.url_name, user] }.to_h
-
     url_names.each do |url_name|
       user = url_name_to_user[url_name]
 
       if user
         user_response = {
           created_at: time_ago_or_never(user.created_at),
-          scanned_gallery_at: time_ago_or_never(user.scanned_gallery_at),
+          state: user.state,
-          scanned_page_at: time_ago_or_never(user.scanned_page_at),
+          object_url: request.base_url + helpers.domain_user_path(user),
+          page_scan: {
+            last_at: time_ago_or_never(user.scanned_page_at),
+            due_for_scan: user.due_for_page_scan?,
+          },
+          gallery_scan: {
+            last_at: time_ago_or_never(user.scanned_gallery_at),
+            due_for_scan: user.due_for_gallery_scan?,
+          },
+          favs_scan: {
+            last_at: time_ago_or_never(user.scanned_favs_at),
+            due_for_scan: user.due_for_favs_scan?,
+          },
         }
-        states = []
-        states << "page" unless user.due_for_page_scan?
-        states << "gallery" unless user.due_for_gallery_scan?
-        states << "seen" if states.empty?
-
-        user_response[:state] = states.join(",")
-
-        if user.scanned_gallery_at && user.scanned_page_at
-          user_response[:terminal_state] = true
-        end
       else
-        user_response = { state: "not_seen", terminal_state: false }
+        user_response = { state: "not_seen" }
       end
       users_response[url_name] = user_response
     end
 
-    queue_depths = Hash.new { |hash, key| hash[key] = 0 }
+    render json: { posts: posts_response, users: users_response }
-
-    jobs_async.each do |job|
-      queue_depths[job.serialized_params["job_class"]] += 1
-    end
-
-    queue_depths =
-      queue_depths
-        .map do |key, value|
-          [
-            key
-              .delete_prefix("Domain::Fa::Job::")
-              .split("::")
-              .last
-              .underscore
-              .delete_suffix("_job")
-              .gsub("_", " "),
-            value,
-          ]
-        end
-        .to_h
-
-    render json: {
-             posts: posts_response,
-             users: users_response,
-             queues: {
-               total_depth: queue_depths.values.sum,
-               depths: queue_depths,
-             },
-           }
   end
 
   def enqueue_objects

app/helpers/domain/descriptions_helper.rb

@@ -36,8 +36,8 @@ module Domain::DescriptionsHelper
 
   sig { params(text: String, url: String).returns(T::Boolean) }
   def text_same_as_url?(text, url)
-    text = text.strip
+    text = text.strip.downcase
-    url = url.strip
+    url = url.strip.downcase
     ["", "http://", "https://"].any? { |prefix| "#{prefix}#{text}" == url }
   end
 
@@ -48,11 +48,12 @@ module Domain::DescriptionsHelper
 
     # profiles often contain bbcode, so first re-parse that
     # for some reason, lots of duplicate <br> tags appear as well
-    html =
+    html = html.gsub("<br>", "").strip
-      T.cast(
+    begin
-        T.unsafe(html.gsub("<br>", "").strip).bbcode_to_html(false),
+      html = T.cast(T.unsafe(html).bbcode_to_html(false), String)
-        String,
+    rescue RuntimeError
-      )
+      # if the bbcode is invalid, skip parsing it
+    end
 
     replacements = {}
 
@@ -99,7 +100,7 @@ module Domain::DescriptionsHelper
                 if ALLOWED_EXTERNAL_LINK_DOMAINS.any? { |domain|
                      url_matches_domain?(domain, url.host)
                    }
-                  if text_same_as_url?(node.text, url.to_s)
+                  if node.text.blank? || text_same_as_url?(node.text, url.to_s)
                     title = title_for_url(url.to_s)
                   else
                     title = node.text
@@ -111,7 +112,7 @@ module Domain::DescriptionsHelper
                         "domain/has_description_html/inline_link_external",
                       locals: {
                         url: url.to_s,
-                        text: title,
+                        title:,
                         icon_path: icon_path_for_domain(url.host),
                       },
                     ),
@@ -150,5 +151,8 @@ module Domain::DescriptionsHelper
     sanitizer.node!(fragment)
     replacements.each { |node, replacement| node.replace(replacement) }
     raw fragment.to_html(preserve_newline: true)
+  rescue StandardError
+    # if anything goes wrong, bail out and don't display anything
+    "(error generating description)"
   end
 end

app/helpers/domain/domains_helper.rb

@@ -32,6 +32,7 @@ module Domain::DomainsHelper
     behance.net
     gumroad.com
     bigcartel.com
+    furaffinity.net
   ].freeze
 
   DOMAIN_TO_ICON_PATH =
@@ -67,6 +68,7 @@ module Domain::DomainsHelper
         [%r{://(.*\.)?x.com/([^/]+)}, ->(match) { match[2] }],
         [%r{://(.*\.)?twitter.com/([^/]+)}, ->(match) { match[2] }],
         [%r{://(.*\.)?patreon.com/([^/]+)}, ->(match) { match[2] }],
+        [%r{://(.*\.)?furaffinity.net/user/([^/]+)}, ->(match) { match[2] }],
       ],
       T::Array[[Regexp, T.proc.params(match: MatchData).returns(String)]],
     )

app/views/domain/has_description_html/_inline_link_external.html.erb

@@ -7,5 +7,5 @@
   <% if local_assigns[:icon_path] %>
     <%= image_tag local_assigns[:icon_path], class: "w-4 h-4 inline-block rounded-sm" %>
   <% end %>
-  <span><%= text %></span>
+  <span><%= title %></span>
 <% end %>

config/routes.rb

@@ -15,6 +15,7 @@ Rails.application.routes.draw do
     namespace :fa do
       get :similar_users, to: "/domain/fa/api#similar_users"
       get :search_user_names, to: "/domain/fa/api#search_user_names"
+      get :object_statuses, to: "/domain/fa/api#object_statuses"
     end
   end
 
@@ -144,7 +145,6 @@ Rails.application.routes.draw do
 
       namespace :fa do
         post :enqueue_objects, to: "/domain/fa/api#enqueue_objects"
-        post :object_statuses, to: "/domain/fa/api#object_statuses"
       end
       namespace :twitter do
         post :enqueue_objects, to: "/domain/twitter/api#enqueue_objects"

justfile

@@ -24,3 +24,6 @@ tc *args:
 
 tapioca *args:
     bundle _2.5.6_ exec tapioca {{args}}
+
+user_scripts:
+    yarn run tsc --project user_scripts/tsconfig.json

user_scripts/enqueuer.user.js

@@ -1,878 +0,0 @@
-// ==UserScript==
-// @name         Redux Enquerer
-// @namespace    http://tampermonkey.net/
-// @version      1.0
-// @description  Enqeue webpages into Redux for archiving
-// @author       You
-// @match        https://www.furaffinity.net/*
-// @match        https://twitter.com/*
-// @icon         https://www.google.com/s2/favicons?sz=64&domain=furaffinity.net
-// @grant        GM_xmlhttpRequest
-// @grant        unsafeWindow
-// @connect      scraper
-// @connect      localhost
-// ==/UserScript==
-"use strict";
-const HOST = "scraper:3000";
-// const HOST = "localhost:3000";
-
-function fa() {
-  function setupNavbar() {
-    const navbarStatusNode = document.createElement("li");
-    navbarStatusNode.classList = "lileft";
-    navbarStatusNode.style.height = "100%";
-    navbarStatusNode.style.display = "flex";
-
-    const navbar = document.querySelector("nav");
-    if (navbar != null) {
-      navbar.querySelector("ul > div").after(navbarStatusNode);
-      return navbarStatusNode;
-    } else {
-      // watch pages, etc don't have a navbar - create one at the top of the page
-      const center = document.body.querySelector("div[align=center]");
-      center.prepend(navbarStatusNode);
-      return navbarStatusNode;
-    }
-  }
-
-  function makeLargeStatusNode(opts = {}) {
-    if (opts.type == null) {
-      opts.type = "span";
-    }
-    if (opts.smaller == null) {
-      opts.smaller = true;
-    }
-    if (opts.style == null) {
-      opts.style = {};
-    }
-
-    const statusNode = document.createElement(opts.type);
-    statusNode.style.cssText =
-      "margin-left: 5px; color: #b7b7b7!important; display: inline";
-    if (opts.smaller) {
-      statusNode.style.fontSize = "80%";
-    }
-
-    for (const [property, value] of Object.entries(opts.style)) {
-      statusNode.style.setProperty(property, value);
-    }
-
-    statusNode.innerHTML = "(...)";
-    return statusNode;
-  }
-
-  function galleryFigureElements() {
-    return [
-      ...document.querySelectorAll("#gallery-search-results figure"),
-      ...document.querySelectorAll("#gallery-browse figure"),
-      ...document.querySelectorAll("#gallery-favorites figure"),
-      ...document.querySelectorAll("#gallery-gallery figure"),
-      ...document.querySelectorAll("#gallery-latest-favorites figure"),
-      ...document.querySelectorAll("#gallery-latest-submissions figure"),
-      ...document.querySelectorAll("section.gallery.messagecenter figure"),
-    ];
-  }
-
-  function gatherPostElements() {
-    // if on a gallery page, links to the list of submissions
-    const galleryFigures = galleryFigureElements().map((figure) => {
-      const faId = parseInt(figure.id.split("-")[1]);
-      const statusNode = document.createElement("p");
-      statusNode.innerHTML = "(...)";
-      const captionLabel = figure.querySelector("figcaption label p");
-      const caption = figure.querySelector("figcaption");
-      // gallery pages will include a figcaption
-      if (captionLabel) {
-        figure.style.height = `calc(${figure.style.height} + 20px)`;
-        figure.querySelector("b").style.height = "auto";
-        statusNode.style.cssText = "position:relative;bottom:2px;";
-        captionLabel.after(statusNode);
-      } else if (caption) {
-        statusNode.style.cssText = "position:relative;bottom:2px;";
-        caption.appendChild(statusNode);
-      } else {
-        const figcaption = document.createElement("div");
-        figcaption.style.cssText =
-          "display: block !important; position: absolute; bottom: 0px; font-size: 10px; width:100%";
-        figcaption.appendChild(statusNode);
-        figure.appendChild(figcaption);
-        figure.style.height = `calc(${figure.style.height} + 20px)`;
-      }
-
-      return { faId, statusNode };
-    });
-
-    const featuredElem = [
-      ...document.querySelectorAll(".userpage-featured-title h2"),
-    ]
-      .map((elem) => {
-        // skip if it's a dynamically changing preview, `submissionDataElems` handles those
-        if (elem.classList.contains("preview_title")) {
-          return null;
-        }
-        const link = elem.querySelector("a");
-        if (!link) {
-          return null;
-        }
-        const faId = faIdFromViewHref(link.href);
-        if (!faId) {
-          return null;
-        }
-        const statusNode = makeLargeStatusNode();
-        elem.appendChild(statusNode);
-
-        return {
-          faId,
-          statusNode,
-        };
-      })
-      .filter(isNotNull);
-
-    let submissionDataElems = [];
-    if (unsafeWindow.submission_data) {
-      submissionDataElems = Object.entries(unsafeWindow.submission_data).map(
-        ([fa_id, _]) => {
-          const statusNode = makeLargeStatusNode();
-          return {
-            faId: fa_id,
-            statusNode,
-          };
-        }
-      );
-    }
-
-    // watch for changes to preview user and swap out status node when that happens
-    [...document.querySelectorAll("h2.preview_title")].forEach(
-      (previewTitleNode) => {
-        const swapOutNode = document.createElement("span");
-        swapOutNode.classList = "swapper";
-        previewTitleNode.appendChild(swapOutNode);
-        const previewTitleLink = previewTitleNode.querySelector("a");
-
-        const observerCb = () => {
-          const previewFaId = faIdFromViewHref(previewTitleLink.href);
-          swapOutNode.innerHTML = "";
-          const currentSubmissionElem = submissionDataElems.find(
-            ({ faId }) => faId == previewFaId
-          );
-          if (currentSubmissionElem) {
-            swapOutNode.appendChild(currentSubmissionElem.statusNode);
-          }
-        };
-        const observer = new MutationObserver(observerCb);
-        observer.observe(previewTitleLink, { childList: true, subtree: true });
-        observerCb();
-      }
-    );
-
-    // /view/<faId>/ page elements
-    let primaryViewPageElems = [];
-    let faId = faIdFromViewHref(window.location.href);
-    if (faId) {
-      primaryViewPageElems = [
-        ...document.querySelectorAll(".submission-title h2"),
-      ].map((elem) => {
-        elem.querySelector("p").style.display = "inline";
-        const statusNode = makeLargeStatusNode({ type: "h3" });
-        elem.appendChild(statusNode);
-        return {
-          faId,
-          statusNode,
-        };
-      });
-    }
-
-    return [
-      ...galleryFigures,
-      ...submissionDataElems,
-      ...featuredElem,
-      ...primaryViewPageElems,
-    ];
-  }
-
-  function gatherUserElements() {
-    // if on a gallery / browse page, the creator links from those posts
-    const userSubmissionLinks = galleryFigureElements()
-      .map((figure) => {
-        const userLinkElem = [...figure.querySelectorAll("figcaption a")]
-          .map((elem) => ({
-            elem: elem,
-            urlName: urlNameFromUserHref(elem.href),
-          }))
-          .filter(({ elem, urlName }) => urlName != null)[0];
-
-        if (userLinkElem == null) {
-          return null;
-        }
-        figure.querySelector("u").style.cssText = "display: block";
-
-        const statusNode = document.createElement("span");
-        statusNode.style.cssText = "margin-left: 2px; color: #c1c1c1";
-        statusNode.innerHTML = "(...)";
-        userLinkElem.elem.after(statusNode);
-
-        return {
-          urlName: userLinkElem.urlName,
-          shouldEnqueue: true,
-          statusNode,
-        };
-      })
-      .filter(isNotNull);
-
-    // if on a /user/ page, the primary username element
-    const userPageUrlName = urlNameFromUserHref(window.location.href);
-    let userPageMain = userPageUrlName
-      ? [...document.querySelectorAll("h1 username")].map((elem) => {
-          const statusNode = document.createElement("span");
-          statusNode.innerHTML = "(...)";
-          elem.after(statusNode);
-          return {
-            urlName: userPageUrlName,
-            shouldEnqueue: true,
-            statusNode,
-          };
-        })
-      : [];
-
-    // comments made by users on posts or user pages
-    let userComments = [...document.querySelectorAll("comment-username a")].map(
-      (elem) => {
-        const statusNode = makeLargeStatusNode();
-        elem.after(statusNode);
-        return {
-          urlName: urlNameFromUserHref(elem.href),
-          shouldEnqueue: true,
-          statusNode,
-        };
-      }
-    );
-
-    // users mentioned in post descriptions or user bios
-    let iconUsernames = [
-      ...document.querySelectorAll("a.iconusername, a.linkusername"),
-    ].map((elem) => {
-      const statusNode = makeLargeStatusNode();
-      elem.after(statusNode);
-      return {
-        urlName: urlNameFromUserHref(elem.href),
-        shouldEnqueue: true,
-        statusNode,
-      };
-    });
-
-    // watcher / watching lists on user page
-    let watchersAndWatchList = [
-      ...document.querySelectorAll("a > span.artist_name"),
-    ].map((elem) => {
-      const link = elem.parentNode;
-      const statusNode = makeLargeStatusNode();
-      link.after(statusNode);
-      return {
-        urlName: urlNameFromUserHref(link.href),
-        shouldEnqueue: true,
-        statusNode,
-      };
-    });
-
-    // users mentioned in the "hero" sections on a url page
-    let submissionDataElems = [];
-
-    if (unsafeWindow.submission_data) {
-      submissionDataElems = Object.entries(unsafeWindow.submission_data).map(
-        ([_, { lower }]) => {
-          const statusNode = makeLargeStatusNode();
-          return {
-            urlName: lower,
-            shouldEnqueue: true,
-            statusNode,
-          };
-        }
-      );
-    }
-
-    // watch for changes to preview user and swap out status node when that happens
-    const previewUser = document.querySelector("a.preview_user");
-    if (previewUser) {
-      const swapOutNode = document.createElement("span");
-      swapOutNode.classList = "swapper";
-      previewUser.after(swapOutNode);
-
-      const observerCb = () => {
-        const previewUrlName = urlNameFromUserHref(previewUser.href);
-        swapOutNode.innerHTML = "";
-        const currentSubmissionElem = submissionDataElems.find(
-          ({ urlName }) => urlName == previewUrlName
-        );
-        if (currentSubmissionElem) {
-          swapOutNode.appendChild(currentSubmissionElem.statusNode);
-        }
-      };
-      const observer = new MutationObserver(observerCb);
-      observer.observe(previewUser, { childList: true, subtree: true });
-      observerCb();
-    }
-
-    // on a /view/ page, the name of the user
-    const submissionContainerUserLinks = [
-      ...document.querySelectorAll(".submission-id-sub-container a"),
-    ].map((elem) => {
-      const statusNode = makeLargeStatusNode({ smaller: false });
-      elem.after(statusNode);
-      return {
-        urlName: urlNameFromUserHref(elem.href),
-        shouldEnqueue: true,
-        statusNode,
-      };
-    });
-
-    // on a /watchlist/by/ or /watchlist/to page
-    const watchListUserLinks = [
-      ...document.querySelectorAll(".watch-list-items.watch-row a"),
-    ].map((elem) => {
-      const statusNode = makeLargeStatusNode({
-        smaller: false,
-        style: {
-          display: "block",
-          "margin-bottom": "5px",
-          "font-size": "50%",
-        },
-      });
-      elem.parentNode.appendChild(statusNode);
-      return {
-        urlName: urlNameFromUserHref(elem.href),
-        shouldEnqueue: true,
-        statusNode,
-      };
-    });
-
-    return [
-      ...userSubmissionLinks,
-      ...userPageMain,
-      ...userComments,
-      ...iconUsernames,
-      ...watchersAndWatchList,
-      ...submissionDataElems,
-      ...submissionContainerUserLinks,
-      ...watchListUserLinks,
-    ];
-  }
-
-  function isNotNull(val) {
-    return val != null;
-  }
-
-  function urlNameFromUserHref(href) {
-    const url = new URL(href);
-    if (url.host != "www.furaffinity.net") {
-      return null;
-    }
-
-    const userPageRegex = /^\/(user|gallery|scraps|favorites|journals)\/.+/;
-    const match = url.pathname.match(userPageRegex);
-    if (!match) {
-      return null;
-    }
-    return url.pathname.split("/")[2];
-  }
-
-  function faIdFromViewHref(href) {
-    const viewPageRegex = /\/(view|full)\/(\d+)/;
-    const match = href.match(viewPageRegex);
-    const faId = (match && match[2]) || null;
-    if (faId) {
-      return parseInt(faId);
-    }
-    return null;
-  }
-
-  function renderTable(stats, styleOpts = {}) {
-    if (stats.length == 0) {
-      return null;
-    }
-
-    if (styleOpts["border-collapse"] == null) {
-      styleOpts["border-collapse"] = "collapse";
-    }
-
-    let table = document.createElement("table");
-    for (const [property, value] of Object.entries(styleOpts)) {
-      table.style.setProperty(property, value);
-    }
-
-    let tbody = document.createElement("tbody");
-    table.appendChild(tbody);
-
-    stats.each(({ name, value, sep, nameAlign, valueAlign }) => {
-      if (name == "" && value == "") {
-        tbody.innerHTML += `<tr><td>---</td><td>---</td></tr>`;
-      } else {
-        if (sep == null) {
-          sep = ":";
-        }
-        if (nameAlign == null) {
-          nameAlign = "right";
-        }
-        if (valueAlign == null) {
-          valueAlign = "right";
-        }
-
-        tbody.innerHTML += `<tr>
-        <td style="text-align:${nameAlign};padding-right:5px">${name}${sep}</td>
-        <td style="text-align:${valueAlign}">${value}</td>
-      </tr>`;
-      }
-    });
-    return table;
-  }
-
-  function optsForNumRows(numRows) {
-    switch (numRows) {
-      case 0:
-      case 1:
-      case 2:
-      case 3:
-        return { "line-height": "1.0em", "font-size": "1.0em" };
-      case 4:
-        return { "line-height": "0.9em", "font-size": "0.8em" };
-      default:
-        return { "line-height": "0.9em", "font-size": "0.6em" };
-    }
-  }
-
-  const navbarNode = setupNavbar();
-  const navbarPageStatsNode = document.createElement("div");
-  const navbarEnqueueNode = document.createElement("div");
-  const navbarLiveQueueNode = document.createElement("div");
-  const navbarLiveEntityNode = document.createElement("div");
-
-  navbarPageStatsNode.innerHTML = "querying...";
-  navbarEnqueueNode.innerHTML = "enqueueing...";
-  navbarLiveQueueNode.innerHTML = "queue loading...";
-  navbarLiveEntityNode.innerHTML = "entity loading...";
-
-  [
-    navbarPageStatsNode,
-    navbarEnqueueNode,
-    navbarLiveQueueNode,
-    navbarLiveEntityNode,
-  ].forEach((node) => {
-    node.style.display = "flex";
-    node.style.marginRight = "5px";
-  });
-
-  [navbarPageStatsNode, navbarEnqueueNode, navbarLiveQueueNode].forEach(
-    (node) => {
-      node.style.paddingRight = "5px";
-      node.style.borderRight = "1px solid #d7d7d7";
-    }
-  );
-
-  navbarNode.append(navbarPageStatsNode);
-  navbarNode.append(navbarEnqueueNode);
-  navbarNode.append(navbarLiveQueueNode);
-  navbarNode.append(navbarLiveEntityNode);
-
-  const userElements = gatherUserElements();
-  const urlNames = [...new Set(userElements.map(({ urlName }) => urlName))];
-  const urlNamesToEnqueue = [
-    ...new Set(
-      userElements
-        .filter(({ shouldEnqueue }) => shouldEnqueue)
-        .map(({ urlName }) => urlName)
-    ),
-  ];
-
-  const postElements = gatherPostElements();
-  const faIds = [...new Set(postElements.map(({ faId }) => faId))];
-
-  function renderLiveQueueStats({ livePostsStats, liveQueueStats }) {
-    let elemsCountsNode = document.createElement("div");
-    elemsCountsNode.style.width = "100%";
-    elemsCountsNode.style.height = "100%";
-    elemsCountsNode.style.display = "flex";
-    elemsCountsNode.style.flexDirection = "row";
-    elemsCountsNode.style.gap = "1em";
-
-    navbarLiveQueueNode.innerHTML = "";
-    navbarLiveQueueNode.appendChild(elemsCountsNode);
-
-    const postsStatsTable = renderTable(livePostsStats, {
-      ...optsForNumRows(livePostsStats.length),
-      width: "auto",
-    });
-
-    const queueStatsTable = renderTable(liveQueueStats, {
-      ...optsForNumRows(liveQueueStats.length),
-      width: "auto",
-    });
-
-    postsStatsTable && elemsCountsNode.appendChild(postsStatsTable);
-    queueStatsTable && elemsCountsNode.appendChild(queueStatsTable);
-  }
-
-  function renderLiveEntityStats(liveEntityStats) {
-    const liveEntityStatsTable = renderTable(liveEntityStats, {
-      ...optsForNumRows(liveEntityStats.length),
-      width: "auto",
-    });
-    navbarLiveEntityNode.innerHTML = "";
-    liveEntityStatsTable &&
-      navbarLiveEntityNode.appendChild(liveEntityStatsTable);
-  }
-
-  let completedEnqueue = false;
-
-  GM_xmlhttpRequest({
-    url: `http://${HOST}/api/fa/enqueue_objects`,
-    method: "POST",
-    headers: {
-      Accept: "application/json",
-      "Content-Type": "application/json",
-    },
-    data: JSON.stringify({
-      fa_ids: faIds,
-      url_names: urlNames,
-      url_names_to_enqueue: urlNamesToEnqueue,
-    }),
-    onload: (response) => {
-      console.log("response: ", response);
-      completedEnqueue = true;
-
-      if (response.status === 200) {
-        const jsonResponse = JSON.parse(response.response);
-        console.log("json: ", jsonResponse);
-        handleEnqueueResponse(jsonResponse);
-      } else {
-        navbarLiveQueueNode.innerHTML = `<b>${response.status} enqueing</b>`;
-      }
-    },
-  });
-
-  function handleEnqueueResponse(jsonResponse) {
-    navbarEnqueueNode.innerHTML = "";
-
-    const enqueueStats = Object.entries(jsonResponse).map(([name, value]) => ({
-      name: name.split("_").join(" "),
-      value,
-    }));
-    const enqueueStatsTable = renderTable(enqueueStats, {
-      ...optsForNumRows(enqueueStats.length),
-      width: "auto",
-    });
-
-    enqueueStatsTable && navbarEnqueueNode.append(enqueueStatsTable);
-  }
-
-  function pollLiveStats() {
-    GM_xmlhttpRequest({
-      url: `http://${HOST}/api/fa/object_statuses`,
-      method: "POST",
-      headers: {
-        Accept: "application/json",
-        "Content-Type": "application/json",
-      },
-      data: JSON.stringify({
-        fa_ids: faIds,
-        url_names: urlNames,
-        url_names_to_enqueue: urlNamesToEnqueue,
-      }),
-      onload: (response) => {
-        console.log("response: ", response);
-        if (response.status === 200) {
-          const jsonResponse = JSON.parse(response.response);
-          console.log("json: ", jsonResponse);
-          const keepPolling = handleLiveStatsResponse(jsonResponse);
-          if (!completedEnqueue || keepPolling) {
-            setTimeout(() => pollLiveStats(), 2500);
-          } else {
-            console.log("reached terminal state");
-          }
-        } else {
-          navbarStatusNode.innerHTML = `<b>${response.status} from scraper</b>`;
-        }
-      },
-    });
-  }
-
-  function handleLiveStatsResponse(jsonResponse) {
-    let allTerminalState = true;
-
-    let numNotSeenPosts = 0;
-    let numOkPosts = 0;
-    let numScannedPosts = 0;
-    let numHaveFile = 0;
-
-    for (const [gotFaId, postInfo] of Object.entries(jsonResponse.posts)) {
-      allTerminalState = allTerminalState && postInfo.terminal_state;
-      postElements
-        .filter(({ faId }) => faId == gotFaId)
-        .forEach(({ statusNode }) => {
-          statusNode.innerHTML = postInfo.state;
-
-          switch (postInfo.state) {
-            case "not_seen":
-              numNotSeenPosts += 1;
-              break;
-            case "ok":
-              numOkPosts += 1;
-              break;
-            case "scanned_post":
-              numScannedPosts += 1;
-              break;
-            case "have_file":
-              numHaveFile += 1;
-              break;
-          }
-        });
-    }
-
-    for (const [gotUrlName, userInfo] of Object.entries(jsonResponse.users)) {
-      allTerminalState = allTerminalState && userInfo.terminal_state;
-      userElements
-        .filter(({ urlName }) => urlName == gotUrlName)
-        .forEach(({ statusNode }) => {
-          statusNode.innerHTML = userInfo.state;
-        });
-    }
-
-    const livePostsStats = [
-      { name: "not seen", value: numNotSeenPosts },
-      { name: "ok", value: numOkPosts },
-      { name: "scanned", value: numScannedPosts },
-      { name: "have file", value: numHaveFile },
-    ];
-
-    let liveQueueStats = Object.entries(jsonResponse.queues.depths).map(
-      ([queue, depth]) => ({ name: queue, value: depth })
-    );
-
-    liveQueueStats = [
-      { name: "total depth", value: `${jsonResponse.queues.total_depth}` },
-      ...liveQueueStats,
-    ];
-
-    while (liveQueueStats.length < 4) {
-      liveQueueStats.push({ name: "", value: "" });
-    }
-
-    allTerminalState &&= jsonResponse.queues.total_depth == 0;
-
-    let liveEntityStats = [{ name: "no entity", value: "", sep: "" }];
-
-    const thisPageFaId = faIdFromViewHref(window.location.href);
-    const pssCommon = { sep: "", valueAlign: "left" };
-    if (thisPageFaId != null) {
-      const postData = jsonResponse.posts[thisPageFaId];
-      liveEntityStats = [
-        {
-          name: "link",
-          value: `<a target="_blank" href="${postData.info_url}" style="text-decoration: underline dotted">${thisPageFaId}</a>`,
-          ...pssCommon,
-        },
-        { name: `seen`, value: postData.seen_at, ...pssCommon },
-        { name: `scanned`, value: postData.scanned_at, ...pssCommon },
-        { name: `downloaded`, value: postData.downloaded_at, ...pssCommon },
-      ];
-    }
-
-    const thisPageUrlName = urlNameFromUserHref(window.location.href);
-    if (thisPageUrlName != null) {
-      const userData = jsonResponse.users[thisPageUrlName];
-      liveEntityStats = [
-        { name: "", value: thisPageUrlName, ...pssCommon },
-        { name: "first seen", value: userData.created_at, ...pssCommon },
-        { name: "page scan", value: userData.scanned_page_at, ...pssCommon },
-        {
-          name: "gallery scan",
-          value: userData.scanned_gallery_at,
-          ...pssCommon,
-        },
-      ];
-    }
-
-    renderLiveQueueStats({
-      livePostsStats,
-      liveQueueStats,
-    });
-    renderLiveEntityStats(liveEntityStats);
-
-    return !allTerminalState;
-  }
-
-  // right off, can render the page stats table
-  const pageStatsTable = renderTable(
-    [
-      { name: "page users", value: urlNames.length },
-      { name: "page posts", value: faIds.length },
-    ],
-    { ...optsForNumRows(2), width: "auto" }
-  );
-  navbarPageStatsNode.innerHTML = "";
-  navbarPageStatsNode.append(pageStatsTable);
-
-  renderLiveQueueStats({
-    livePostsStats: [
-      { name: "not seen", value: "---" },
-      { name: "ok", value: "---" },
-      { name: "scanned", value: "---" },
-      { name: "have file", value: "---" },
-    ],
-    liveQueueStats: [
-      { name: "queue depths", value: "---" },
-      { name: "", value: "" },
-      { name: "", value: "" },
-      { name: "", value: "" },
-    ],
-  });
-
-  pollLiveStats();
-}
-
-function twitter(mainNode) {
-  function onlyUnique(value, index, array) {
-    return array.indexOf(value) === index;
-  }
-
-  let seenTweets = new Set();
-  let ignoreUsers = new Set(["elonmusk"]);
-  let nameToStatusNodes = {};
-  let nameToStatus = { Mangoyena: "yes!" };
-
-  function observerCallback() {
-    // update status nodes for all the user links
-    function parentN(node, times) {
-      for (let i = 0; i < times; i++) {
-        node = node.parentNode;
-      }
-      return node;
-    }
-    [
-      ...document.querySelectorAll(
-        "[data-testid=primaryColumn] a[role=link][tabindex='-1']:not(aria-hidden)"
-      ),
-    ]
-      .filter(
-        (link) =>
-          link.hostname == "twitter.com" &&
-          !link.pathname.startsWith("/i/") &&
-          !link.querySelector("img")
-      )
-      .map((link) => ({
-        elem: link,
-        name: link.pathname.split("/")[1],
-        header: parentN(link, 7),
-      }))
-      .forEach(({ elem, name, header }) => {
-        if (nameToStatusNodes[name] == null) {
-          nameToStatusNodes[name] = [];
-        }
-        if (header.querySelector(".reduxStatusNode") == null) {
-          const statusNode = document.createElement("span");
-          statusNode.style.cssText = "color:white; float: right";
-          statusNode.classList = ["reduxStatusNode"];
-          header.insertBefore(statusNode, header.children[1]);
-          nameToStatusNodes[name].push(statusNode);
-        }
-
-        const status = nameToStatus[name];
-        nameToStatusNodes[name].forEach((statusNode) => {
-          if (status != null) {
-            statusNode.innerHTML = status;
-          } else {
-            statusNode.innerHTML = "(loading...)";
-          }
-        });
-      });
-
-    const tweets = [...document.querySelectorAll("a[role=link]")]
-      .filter(
-        (link) =>
-          link.hostname == "twitter.com" && link.pathname.includes("/status/")
-      )
-      .map((link) => {
-        const parts = link.pathname.split("/");
-        return {
-          name: parts[1],
-          tweetId: parts[3],
-        };
-      })
-      .filter(({ name }) => !ignoreUsers.has(name))
-      .map(JSON.stringify)
-      .filter(onlyUnique);
-
-    let newTweets = [...tweets].filter((n) => !seenTweets.has(n));
-    if (newTweets.length > 0) {
-      newTweets
-        .map(JSON.parse)
-        .forEach(({ name, tweetId }) =>
-          console.log("new tweet", `${name} / ${tweetId}`)
-        );
-
-      newTweets.forEach((tweet) => seenTweets.add(tweet));
-      enqueueNewTweets(newTweets);
-    }
-  }
-
-  const enqueuedTweets = new Set();
-  let enqueueInProgress = false;
-  function enqueueNewTweets(newTweets) {
-    newTweets.forEach((tweet) => enqueuedTweets.add(tweet));
-
-    if (!enqueueInProgress && enqueuedTweets.size > 0) {
-      enqueueInProgress = true;
-      sendEnqueue([...enqueuedTweets].map(JSON.parse));
-      enqueuedTweets.clear();
-    }
-  }
-
-  // const enqueuedNames = new Set()
-  function sendEnqueue(tweets) {
-    // const newNames = tweets.map(({ name }) => name).filter(name => !enqueuedNames.has(name));
-    console.log("enqueue tweets: ", tweets);
-
-    GM_xmlhttpRequest({
-      url: `http://${HOST}/api/twitter/enqueue_objects`,
-      method: "POST",
-      headers: {
-        Accept: "application/json",
-        "Content-Type": "application/json",
-      },
-      data: JSON.stringify({ names: tweets.map(({ name }) => name) }),
-      onload: (response) => {
-        if (response.status === 200) {
-          const jsonResponse = JSON.parse(response.response);
-          console.log("redux json: ", jsonResponse);
-
-          enqueueInProgress = false;
-          enqueueNewTweets([]);
-        } else {
-          console.error("redux error: ", response.status, response);
-        }
-      },
-    });
-  }
-
-  function startObserving() {
-    const observer = new MutationObserver(() => {
-      observer.disconnect();
-      observerCallback();
-      startObserving();
-    });
-
-    observer.observe(mainNode, {
-      childList: true,
-      subtree: true,
-    });
-  }
-  startObserving();
-}
-
-(function () {
-  if (window.location.hostname == "www.furaffinity.net") {
-    fa();
-  } else if (window.location.hostname == "twitter.com") {
-    twitter(document.querySelector("body"));
-  } else {
-    console.log("unhandled domain ", window.location.hostname);
-  }
-})();

user_scripts/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2015",
+    "module": "CommonJS",
+    "lib": ["dom", "es6", "dom.iterable"],
+    "outDir": "./dist",
+    "sourceMap": true,
+    "noImplicitAny": true,
+    "baseUrl": ".",
+    "rootDir": "."
+  },
+  "include": ["*.ts"],
+  "exclude": ["../app", "../node_modules", "dist"],
+  "compileOnSave": true
+}