dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

before redoing devcontainer files

Browse Source
This commit is contained in:
Dylan Knutson
2024-12-17 06:34:47 -08:00
parent 955a3021ae
commit 1b59b44435
36 changed files with 9857 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.devcontainer/.env Normal file
View File

@@ -0,0 +1,9 @@
PGADMIN_DEFAULT_EMAIL=admin@example.com
PGADMIN_DEFAULT_PASSWORD=password
PGADMIN_LISTEN_PORT=8080
# no password required
PGADMIN_CONFIG_SERVER_MODE=False
PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED=False
RAILS_ENV=development
NODE_ENV=development

68
.devcontainer/Dockerfile.devcontainer Normal file
View File

@@ -0,0 +1,68 @@
FROM ruby:3.2.0 AS native-gems
RUN rm -f /etc/apt/apt.conf.d/docker-clean; \
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
RUN \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update && \
apt-get install --no-install-recommends --no-install-suggests -y \
cmake
WORKDIR /usr/src/app
RUN gem install bundler -v '2.4.5'
COPY gems gems
WORKDIR /usr/src/app/gems/xdiff-rb
RUN bundle install
RUN rake compile
WORKDIR /usr/src/app/gems/rb-bsdiff
RUN bundle install
RUN rake compile
# Primary image
FROM mcr.microsoft.com/devcontainers/base:bookworm
# apt caching & install packages
RUN rm -f /etc/apt/apt.conf.d/docker-clean; \
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
RUN \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update && \
apt-get install --no-install-recommends --no-install-suggests -y \
autoconf \
build-essential \
ca-certificates \
curl \
gnupg \
libblas-dev \
libdb-dev \
libffi-dev \
libgdbm-dev \
libgdbm6 \
libgmp-dev \
liblapack-dev \
libncurses5-dev \
libpq-dev \
libreadline6-dev \
libssl-dev \
libvips42 \
libyaml-dev \
patch \
rbenv \
rustc \
uuid-dev \
zlib1g-dev
USER vscode
ENV PATH=/home/vscode/.rbenv/shims:$PATH
ENV CONFIGURE_OPTS="--disable-install-rdoc"
RUN git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build
RUN rbenv install 3.2.0
RUN rbenv global 3.2.0
COPY --from=native-gems /usr/src/app/gems/xdiff-rb /gems/xdiff-rb
COPY --from=native-gems /usr/src/app/gems/rb-bsdiff /gems/rb-bsdiff
ENV RAILS_ENV development
ENV RUBOCOP_OPTS --server

42
.devcontainer/devcontainer.json Normal file
View File

@@ -0,0 +1,42 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
{
"name": "Redux Scraper",
// "build": {
// // Sets the run context to one level up instead of the .devcontainer folder.
// "context": "..",
// // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
// "dockerfile": "Dockerfile"
// },
"dockerComposeFile": "docker-compose.yml",
"service": "app",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
// "mounts": [
// {
// "type": "volume",
// "source": "${devcontainerId}-gems",
// "target": "/home/vscode/.rbenv/versions/3.2.0/lib/ruby/gems"
// }
// ],
"features": {
"ghcr.io/meaningful-ooo/devcontainer-features/fish:1": {},
"ghcr.io/nikobockerman/devcontainer-features/fish-persistent-data:2": {},
"ghcr.io/devcontainers/features/node:1": {
"installYarnUsingApt": true,
"version": "18",
"pnpmVersion": "none",
"nvmVersion": "latest"
}
},
// "runArgs": [
// "--network=host"
// ],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],
// Uncomment the next line to run commands after the container is created.
"postCreateCommand": "${containerWorkspaceFolder}/.devcontainer/post-create.sh"
// Configure tool-specific properties.
// "customizations": {},
// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "devcontainer"
}

33
.devcontainer/docker-compose.yml Normal file
View File

@@ -0,0 +1,33 @@
version: '3.8'
volumes:
devcontainer-redux-postgres-data:
devcontainer-redux-gem-cache:
services:
app:
build:
context: ..
dockerfile: Dockerfile.devcontainer
volumes:
# - ../..:/workspaces:cached
- devcontainer-redux-gem-cache:/home/vscode/.rbenv/versions/3.2.0/lib/ruby/gems
command: sleep infinity
network_mode: service:db
env_file:
- .env
db:
image: pgvector/pgvector:pg17
restart: unless-stopped
volumes:
- devcontainer-redux-postgres-data:/var/lib/postgresql/data
env_file:
- .env
pgadmin:
image: dpage/pgadmin4:8.13.0
restart: unless-stopped
network_mode: service:db
env_file:
- .env

19
.devcontainer/post-create.sh Executable file
View File

@@ -0,0 +1,19 @@
#!/bin/bash -ex
echo "Path: $PATH"
echo "Ruby: $(which ruby)"
echo "Gem: $(which gem)"
echo "Bundler: $(which bundler)"
echo "Rake: $(which rake)"
gem install bundler -v '2.4.1'
gem install rake -v '13.0.6'
gem install ruby-lsp -v '0.22.1'
gem install rubocop -v '1.69.2'
bundle install --jobs $(getconf _NPROCESSORS_ONLN)
rbenv rehash
rubocop --start-server
npm install yarn
bin/rails yarn:install
yarn

1
.dockerignore
View File

@@ -4,7 +4,6 @@ log
tmp
ext
build
/bin
node_modules
profiler
.bundle

12
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for more information:
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
# https://containers.dev/guide/dependabot
version: 2
updates:
- package-ecosystem: "devcontainers"
directory: "/"
schedule:
interval: weekly

32
.rubocop.yml Normal file
View File

@@ -0,0 +1,32 @@
# The behavior of RuboCop can be controlled via the .rubocop.yml
# configuration file. It makes it possible to enable/disable
# certain cops (checks) and to alter their behavior if they accept
# any parameters. The file can be placed either in your home
# directory or in some project directory.
#
# RuboCop will start looking for the configuration file in the directory
# where the inspected file is and continue its way up to the root directory.
#
# See https://docs.rubocop.org/rubocop/configuration
inherit_mode:
merge:
- Exclude
AllCops:
NewCops: disable
Metrics/MethodLength:
Enabled: false
Metrics/ClassLength:
Enabled: false
Metrics/BlockLength:
Enabled: false
Style/Documentation:
Enabled: false
Metrics/AbcSize:
Enabled: false

8
.vscode/extensions.json vendored Normal file
View File

@@ -0,0 +1,8 @@
{
"recommendations": [
"Shopify.ruby-extensions-pack",
"LoranKloeze.ruby-rubocop-revived",
"CraigMaslowski.erb",
"dbaeumer.vscode-eslint"
]
}

9
.vscode/settings.json vendored
View File

@@ -1,5 +1,12 @@
{
"editor.formatOnSave": true,
"workbench.editor.titleScrollbarSizing": "large",
"window.title": "${activeEditorMedium}${separator}${rootName}${separator}${profileName}"
"window.title": "${activeEditorMedium}${separator}${rootName}${separator}${profileName}",
"workbench.preferredDarkColorTheme": "Spinel",
"workbench.preferredLightColorTheme": "Spinel Light",
"rubyLsp.formatter": "auto",
"[ruby]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "LoranKloeze.ruby-rubocop-revived"
}
}

19
Dockerfile
View File

@@ -22,6 +22,7 @@ RUN rake compile
FROM ruby:3.2.0
# set up nodejs 18.x deb repo
USER root
RUN mkdir -p /etc/apt/keyrings && \
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
@@ -36,10 +37,12 @@ RUN \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update && \
apt-get install --no-install-recommends --no-install-suggests -y \
libblas-dev liblapack-dev libvips42 ca-certificates curl gnupg nodejs
libblas-dev liblapack-dev libvips42 ca-certificates curl gnupg nodejs libpq-dev
USER vscode
ENV PATH=/home/vscode/.rbenv/shims:$PATH
WORKDIR /usr/src/app
RUN gem install bundler -v '2.4.6'
RUN gem install bundler -v '2.4.1'
RUN bundle config --global frozen 1
WORKDIR /usr/src/app
@@ -50,8 +53,9 @@ RUN bundle install
# install js dependencies
COPY package.json yarn.lock ./
RUN npm install -g yarn && \
rails yarn:install
RUN npm install yarn && \
bin/rails yarn:install && \
yarn
COPY . .
COPY --from=native-gems /usr/src/app/gems/xdiff-rb/lib/xdiff gems/xdiff-rb/lib/xdiff
@@ -59,9 +63,10 @@ COPY --from=native-gems /usr/src/app/gems/rb-bsdiff/lib gems/rb-bsdiff/lib
# precompile assets
ENV RAILS_ENV production
RUN rails assets:precompile
RUN bin/rails assets:precompile
ENV RAILS_ENV worker
ENV proxy direct
# CMD ["bundle" "exec" "good_job" "--max-threads=2"]
CMD ["/bin/bash"]
RUN mkdir -p tmp/pids
CMD /bin/bash

124
Gemfile
View File

@@ -1,36 +1,36 @@
source "https://rubygems.org"
source 'https://rubygems.org'
git_source(:github) { |repo| "https://github.com/#{repo}.git" }
ruby "3.2.0"
ruby '3.2.0'
# ruby "3.0.3"
# Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main"
gem "rails", "~> 7.0.4", ">= 7.0.4.2"
gem 'rails', '~> 7.0.4', '>= 7.0.4.2'
# The original asset pipeline for Rails [https://github.com/rails/sprockets-rails]
gem "sprockets-rails"
gem 'sprockets-rails'
# Use sqlite3 as the database for Active Record
gem "sqlite3", "~> 1.4"
gem "pg"
gem 'pg'
gem 'sqlite3', '~> 1.4'
gem "pry"
gem "pry-stack_explorer"
gem 'pry'
gem 'pry-stack_explorer'
# Use the Puma web server [https://github.com/puma/puma]
gem "puma", "~> 5.0"
gem 'puma', '~> 5.0'
# # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails]
# gem "importmap-rails"
# Hotwire's SPA-like page accelerator [https://turbo.hotwired.dev]
gem "turbo-rails"
gem 'turbo-rails'
# Hotwire's modest JavaScript framework [https://stimulus.hotwired.dev]
gem "stimulus-rails"
gem 'stimulus-rails'
# Build JSON APIs with ease [https://github.com/rails/jbuilder]
gem "jbuilder"
gem 'jbuilder'
# Use Redis adapter to run Action Cable in production
# gem "redis", "~> 4.0"
@@ -42,10 +42,10 @@ gem "jbuilder"
# gem "bcrypt", "~> 3.1.7"
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
gem "tzinfo-data", platforms: %i[ mingw mswin x64_mingw jruby ]
gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]
# Reduces boot times through caching; required in config/boot.rb
gem "bootsnap", require: false
gem 'bootsnap', require: false
# Use Sass to process CSS
# gem "sassc-rails"
@@ -55,79 +55,79 @@ gem "bootsnap", require: false
group :development, :test, :staging do
# See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
gem "debug", platforms: %i[ mri mingw x64_mingw ]
gem 'debug', platforms: %i[mri mingw x64_mingw]
end
group :development, :staging do
# Use console on exceptions pages [https://github.com/rails/web-console]
gem "web-console"
gem "htmlbeautifier"
gem "rails_live_reload"
gem "rufo", require: false
gem 'htmlbeautifier'
gem 'rails_live_reload'
gem 'rufo', require: false
gem 'web-console'
# Speed up commands on slow machines / big apps [https://github.com/rails/spring]
# gem "spring"
# Add speed badges [https://github.com/MiniProfiler/rack-mini-profiler]
gem "rack-mini-profiler", require: ["enable_rails_patches", "rack-mini-profiler"]
gem "memory_profiler"
gem "stackprof"
gem 'memory_profiler'
gem 'rack-mini-profiler', require: %w[enable_rails_patches rack-mini-profiler]
gem 'stackprof'
end
group :test do
# Use system testing [https://guides.rubyonrails.org/testing.html#system-testing]
gem "capybara"
gem "selenium-webdriver"
gem "webdrivers"
gem "rspec-rails"
gem 'capybara'
gem 'rspec-rails'
gem 'selenium-webdriver'
gem 'webdrivers'
end
gem "xdiff", path: "gems/xdiff-rb"
gem 'xdiff', path: '/gems/xdiff-rb'
# for legacy import
gem "diffy"
gem "rb-bsdiff", path: "gems/rb-bsdiff"
gem 'diffy'
gem 'rb-bsdiff', path: '/gems/rb-bsdiff'
gem "ruby-prof"
gem "ruby-prof-speedscope"
gem "table_print"
gem "addressable"
gem "nokogiri"
gem "http-cookie"
gem "curb"
gem "kaminari"
gem "colorize"
gem "daemons"
gem "ripcord"
gem "discard"
gem "influxdb-client"
gem "concurrent-ruby-ext", require: "concurrent"
gem "concurrent-ruby-edge", require: "concurrent-edge"
gem "pluck_each"
gem "good_job", "3.17.2"
gem "zstd-ruby"
gem "ruby-vips"
gem "composite_primary_keys", "~> 14.0"
gem 'addressable'
gem 'colorize'
gem 'composite_primary_keys', '~> 14.0'
gem 'concurrent-ruby-edge', require: 'concurrent-edge'
gem 'concurrent-ruby-ext', require: 'concurrent'
gem 'curb'
gem 'daemons'
gem 'discard'
gem 'good_job', '3.17.2'
gem 'http-cookie'
gem 'influxdb-client'
gem 'kaminari'
gem 'nokogiri'
gem 'pluck_each'
gem 'ripcord'
gem 'ruby-prof'
gem 'ruby-prof-speedscope'
gem 'ruby-vips'
gem 'table_print'
gem 'zstd-ruby'
gem "pg_query", ">= 2"
gem "pghero", git: "https://github.com/dymk/pghero", ref: "e314f99"
gem 'pghero', git: 'https://github.com/dymk/pghero', ref: 'e314f99'
gem 'pg_query', '>= 2'
gem "neighbor"
gem "disco"
gem "faiss"
gem "progressbar"
gem 'disco'
gem 'faiss'
gem 'neighbor'
gem 'progressbar'
group :production, :staging do
gem "rails_semantic_logger"
gem 'rails_semantic_logger'
end
group :production do
gem "sd_notify"
gem 'sd_notify'
end
gem "tailwindcss-rails", "~> 2.0"
gem 'tailwindcss-rails', '~> 2.0'
gem "shakapacker"
gem "react_on_rails"
gem "rack-cors"
gem "sanitize"
gem 'rack-cors'
gem 'react_on_rails'
gem 'sanitize'
gem 'shakapacker'

4
Gemfile.lock
View File

@@ -7,12 +7,12 @@ GIT
activerecord (>= 6)
PATH
remote: gems/rb-bsdiff
remote: /gems/rb-bsdiff
specs:
rb-bsdiff (0.1.0)
PATH
remote: gems/xdiff-rb
remote: /gems/xdiff-rb
specs:
xdiff (0.0.1)

2
app/controllers/domain/fa/api_controller.rb
View File

@@ -332,7 +332,7 @@ class Domain::Fa::ApiController < ApplicationController
end
end
def users_list_to_similar_list(user_list)
def users_list_to_similar_list(users_list)
users_list.map do |user|
profile_thumb_url = user.avatar&.file_uri&.to_s
profile_thumb_url || begin

16
app/controllers/domain/fa/export_controller.rb Normal file
View File

@@ -0,0 +1,16 @@
class Domain::Fa::ExportController < ApplicationController
skip_before_action :verify_authenticity_token,
only: %i[
enqueue_objects
object_statuses
]
skip_before_action :validate_api_token,
only: %i[
search_user_names
]
def user
@user = Domain::Fa::User.find_by!(url_name: params[:url_name])
end
end

6
app/lib/vpn_only_route_constraint.rb
View File

@@ -1,14 +1,14 @@
class VpnOnlyRouteConstraint
def matches?(request)
if request.params[:force_vpn_off] == "1"
if request.params[:force_vpn_off] == '1'
false
elsif Rails.env.test?
true
elsif Rails.env.development? || Rails.env.staging?
request.ip == "127.0.0.1" || request.ip == "::1"
request.ip == '127.0.0.1' || request.ip == '::1'
elsif Rails.env.production?
# curtus IP on vpn
request.ip == "10.200.0.3"
request.ip == '10.200.0.3'
else
false
end

124
app/models/domain/fa/user.rb
View File

@@ -1,32 +1,32 @@
class Domain::Fa::User < ReduxApplicationRecord
self.table_name = "domain_fa_users"
self.table_name = 'domain_fa_users'
has_lite_trail(schema_version: 1)
has_many :posts,
class_name: "::Domain::Fa::Post",
inverse_of: :creator,
foreign_key: :creator_id
class_name: '::Domain::Fa::Post',
inverse_of: :creator,
foreign_key: :creator_id
has_one :disco,
class_name: "::Domain::Fa::UserFactor",
class_name: '::Domain::Fa::UserFactor',
inverse_of: :user,
foreign_key: :user_id,
dependent: :destroy
has_one :avatar,
class_name: "::Domain::Fa::UserAvatar",
class_name: '::Domain::Fa::UserAvatar',
inverse_of: :user,
dependent: :destroy
enum :state, [
:ok, # so far so good, user may not yet be scanned
:scan_error, # user has been removed or otherwise, see state_detail
]
:ok, # so far so good, user may not yet be scanned
:scan_error # user has been removed or otherwise, see state_detail
]
# Who this user follows (join table)
has_many :follower_joins,
class_name: "::Domain::Fa::Follow",
class_name: '::Domain::Fa::Follow',
foreign_key: :follower_id,
inverse_of: :follower,
dependent: :destroy
@@ -38,7 +38,7 @@ class Domain::Fa::User < ReduxApplicationRecord
# Who follows this user (join table)
has_many :followed_joins,
class_name: "::Domain::Fa::Follow",
class_name: '::Domain::Fa::Follow',
foreign_key: :followed_id,
inverse_of: :followed,
dependent: :destroy
@@ -49,11 +49,11 @@ class Domain::Fa::User < ReduxApplicationRecord
source: :follower
has_many :fav_post_joins,
class_name: "::Domain::Fa::Fav",
class_name: '::Domain::Fa::Fav',
inverse_of: :user
has_many :fav_posts,
class_name: "::Domain::Fa::Post",
class_name: '::Domain::Fa::Post',
through: :fav_post_joins,
source: :post
@@ -65,12 +65,12 @@ class Domain::Fa::User < ReduxApplicationRecord
expected = self.class.name_to_url_name(name)
matches = if name.length >= 30
url_name.starts_with?(expected)
else
url_name == expected
end
url_name.starts_with?(expected)
else
url_name == expected
end
if !matches
unless matches
errors.add(
:name,
"name '#{name}' does not match url_name, expected #{expected} but was #{url_name}"
@@ -104,19 +104,19 @@ class Domain::Fa::User < ReduxApplicationRecord
end
SCAN_TYPES = {
:page => 1.month,
:gallery => 1.year,
:follows => 1.month,
:favs => 1.month,
:incremental => 1.month,
page: 1.month,
gallery: 1.year,
follows: 1.month,
favs: 1.month,
incremental: 1.month
}
SCAN_FIELD_TYPES = {
:page => :column,
:gallery => :column,
:follows => :column,
:favs => :column,
:incremental => :state_detail,
page: :column,
gallery: :column,
follows: :column,
favs: :column,
incremental: :state_detail
}
SCAN_TYPES.keys.each do |scan_type|
@@ -128,14 +128,14 @@ class Domain::Fa::User < ReduxApplicationRecord
scanned_ago_in_words(scan_type)
end
if SCAN_FIELD_TYPES[scan_type] == :state_detail
define_method(:"scanned_#{scan_type}_at") do
get_scanned_at_value(scan_type)
end
next unless SCAN_FIELD_TYPES[scan_type] == :state_detail
define_method(:"scanned_#{scan_type}_at=") do |value|
set_scanned_at_value(scan_type, value)
end
define_method(:"scanned_#{scan_type}_at") do
get_scanned_at_value(scan_type)
end
define_method(:"scanned_#{scan_type}_at=") do |value|
set_scanned_at_value(scan_type, value)
end
end
@@ -143,9 +143,9 @@ class Domain::Fa::User < ReduxApplicationRecord
def scanned_ago_in_words(scan_type)
if (timestamp = get_scanned_at_value(scan_type))
DATE_HELPER.time_ago_in_words(timestamp) + " ago"
DATE_HELPER.time_ago_in_words(timestamp) + ' ago'
else
"never"
'never'
end
end
@@ -157,10 +157,11 @@ class Domain::Fa::User < ReduxApplicationRecord
def take_posts_from(other_user)
return if other_user == self
other_posts = other_user.posts
other_posts.update_all(creator_id: self.id)
other_posts.update_all(creator_id: id)
other_user.posts.reload
self.posts.reload
posts.reload
end
def avatar_or_create
@@ -181,17 +182,18 @@ class Domain::Fa::User < ReduxApplicationRecord
end
URL_NAME_EXCEPTIONS = {
"Kammiu" => "rammiu",
'Kammiu' => 'rammiu'
}
def self.name_to_url_name(name)
name = name.strip
URL_NAME_EXCEPTIONS[name] || name.
delete("_").
gsub(/\s/, "").
downcase
URL_NAME_EXCEPTIONS[name] || name
.delete('_')
.gsub(/\s/, '')
.downcase
end
# TODO - write method for getting suggested users to follow
# TODO: - write method for getting suggested users to follow
# based on this user
# something like:
# UserFactor.nearest_neighbors(
@@ -215,9 +217,9 @@ class Domain::Fa::User < ReduxApplicationRecord
def guess_user_page_log_entry
for_path = proc { |uri_path|
HttpLogEntry.where(
uri_scheme: "https",
uri_host: "www.furaffinity.net",
uri_path: uri_path,
uri_scheme: 'https',
uri_host: 'www.furaffinity.net',
uri_path: uri_path
).order(created_at: :desc).first
}
@@ -226,16 +228,16 @@ class Domain::Fa::User < ReduxApplicationRecord
}
# older versions don't end in a trailing slash
hle_id = self.log_entry_detail && self.log_entry_detail["last_user_page_id"]
hle_id = self.log_entry_detail && self.log_entry_detail['last_user_page_id']
# first try the last scanned user page (present on most fa user models)
for_hle_id.call(hle_id) ||
# if that's missing, see if there's an existing request logged to the user page
for_path.call("/user/#{self.url_name}/") ||
for_path.call("/user/#{url_name}/") ||
# and try the non-trailing-slash version as well
for_path.call("/user/#{self.url_name}")
for_path.call("/user/#{url_name}")
# TODO - maybe can look for posts as well, those might list an avatar
# TODO: - maybe can look for posts as well, those might list an avatar
end
def to_param
@@ -245,22 +247,20 @@ class Domain::Fa::User < ReduxApplicationRecord
private
def similar_users_by(factor_col, exclude_followed_by)
query = self.
disco.
nearest_neighbors(factor_col, distance: "euclidean")
query =
disco
.nearest_neighbors(factor_col, distance: 'euclidean')
if exclude_followed_by
query = query.where.not(user_id: exclude_followed_by.follows.select(:followed_id))
end
query = query.where.not(user_id: exclude_followed_by.follows.select(:followed_id)) if exclude_followed_by
users_from_disco_query(query)
end
def users_from_disco_query(disco_query)
Domain::Fa::User.
select("domain_fa_users.*", disco_query.select_values.last).
joins(:disco).
merge(disco_query.reselect(:user_id))
Domain::Fa::User
.select('domain_fa_users.*', disco_query.select_values.last)
.joins(:disco)
.merge(disco_query.reselect(:user_id))
end
def get_scanned_at_value(scan_type)

29
app/views/domain/fa/export/user.json.jbuilder Normal file
View File

@@ -0,0 +1,29 @@
json.user do
json.name @user.name
json.full_name @user.full_name
json.url_name @user.url_name
json.state @user.state
json.log_entry_detail @user.log_entry_detail
json.user_page_log_entry do
json.partial! "log_entries/show", locals: { log_entry: HttpLogEntry.find(@user.log_entry_detail["last_user_page_id"]) }
end if @user.log_entry_detail["last_user_page_id"]
json.user_stats do
json.num_pageviews @user.num_pageviews
json.num_submissions @user.num_submissions
json.num_comments_recieved @user.num_comments_recieved
json.num_comments_given @user.num_comments_given
json.num_journals @user.num_journals
json.num_favorites @user.num_favorites
end
# json.gallery_page_log_entry do
# json.partial! "log_entries/show", locals: { log_entry: HttpLogEntry.find(@user.log_entry_detail["last_gallery_page_id"]) }
# end if @user.log_entry_detail["last_gallery_page_id"]
json.created_at @user.created_at
json.updated_at @user.updated_at
end

18
app/views/log_entries/_show.json.jbuilder Normal file
View File

@@ -0,0 +1,18 @@
json.url log_entry.uri.to_s
json.response_time_ms log_entry.response_time_ms
json.status_code log_entry.status_code
json.performed_by log_entry.performed_by
json.content_type log_entry.content_type
json.content_size log_entry.response.size
json.content_size_stored log_entry.response.bytes_stored
json.content_sha256 HexUtil.bin2hex(log_entry.response.sha256)
json.content_bytes_base64 Base64.encode64(log_entry.response.contents)
json.request_headers log_entry.request_headers.headers
json.response_headers log_entry.response_headers.headers
json.requested_at log_entry.requested_at
json.created_at log_entry.created_at
json.updated_at log_entry.updated_at

1
app/views/log_entries/show.json.jbuilder Normal file
View File

@@ -0,0 +1 @@
render "log_entries/show", locals: { log_entry: @log_entry }

3
bin/setup
View File

@@ -17,9 +17,6 @@ FileUtils.chdir APP_ROOT do
system! "gem install bundler --conservative"
system("bundle check") || system!("bundle install")
# Install JavaScript dependencies
system!("bin/yarn")
# puts "\n== Copying sample files =="
# unless File.exist?("config/database.yml")
# FileUtils.cp "config/database.yml.sample", "config/database.yml"

12
config/cookies/fa.yml
View File

@@ -2,28 +2,28 @@ ddwhatnow: &ddwhatnow
- domain: .furaffinity.net
cookies:
- name: a
value: 5a1b4c3d-7b51-41e1-a00a-88420d7d0d3e
value: ebf15f81-6978-4be0-b687-a4c950030d1b
path: /
- name: b
value: a29dec05-d47b-4f51-a9d8-5a607d12e482
value: 9daf8e3e-1867-44f4-bf38-e248dcfba3dd
path: /
- domain: rv.furaffinity.net
cookies:
- name: OAID
value: d4247fd1b8b755a4064f2e030dec919f
value: 36dcd165758a3b5ed55565030c41a3b0
path: /
vipvillageworker: &vipvillageworker
- domain: .furaffinity.net
cookies:
- name: a
value: 5a9437e0-ded0-4b8d-8255-ff0198664025
value: 64be5a6d-3f9b-4cc5-9a3d-303799d34277
- name: b
value: 2d7da6a8-959f-407e-abd1-af8f51c110be
value: c508792f-8694-485c-b077-e6626ceeb63a
- domain: rv.furaffinity.net
cookies:
- name: OAID
value: d8ebf7f9508511d8ba7307e3cb36f03c
value: 14ee7a5203d4b59fd7cb3f8513281fcc
path: /
- domain: www.furaffinity.net
cookies:

1
config/database.yml
View File

@@ -21,7 +21,6 @@ redux_prod: &redux_prod
redux_staging: &redux_staging
<<: *redux_prod
host: postgres
username: scraper_redux_staging
password: q6Jf8mXEUkAxdyHq1tUtCTPa1raX1QAT

135
config/initializers/new_framework_defaults_7_0.rb Normal file
View File

@@ -0,0 +1,135 @@
# Be sure to restart your server when you modify this file.
#
# This file eases your Rails 7.0 framework defaults upgrade.
#
# Uncomment each configuration one by one to switch to the new default.
# Once your application is ready to run with all new defaults, you can remove
# this file and set the `config.load_defaults` to `7.0`.
#
# Read the Guide for Upgrading Ruby on Rails for more info on each option.
# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html
# `button_to` view helper will render `<button>` element, regardless of whether
# or not the content is passed as the first argument or as a block.
# Rails.application.config.action_view.button_to_generates_button_tag = true
# `stylesheet_link_tag` view helper will not render the media attribute by default.
# Rails.application.config.action_view.apply_stylesheet_media_default = false
# Change the digest class for the key generators to `OpenSSL::Digest::SHA256`.
# Changing this default means invalidate all encrypted messages generated by
# your application and, all the encrypted cookies. Only change this after you
# rotated all the messages using the key rotator.
#
# See upgrading guide for more information on how to build a rotator.
# https://guides.rubyonrails.org/v7.0/upgrading_ruby_on_rails.html
# Rails.application.config.active_support.key_generator_hash_digest_class = OpenSSL::Digest::SHA256
# Change the digest class for ActiveSupport::Digest.
# Changing this default means that for example Etags change and
# various cache keys leading to cache invalidation.
# Rails.application.config.active_support.hash_digest_class = OpenSSL::Digest::SHA256
# Don't override ActiveSupport::TimeWithZone.name and use the default Ruby
# implementation.
# Rails.application.config.active_support.remove_deprecated_time_with_zone_name = true
# Calls `Rails.application.executor.wrap` around test cases.
# This makes test cases behave closer to an actual request or job.
# Several features that are normally disabled in test, such as Active Record query cache
# and asynchronous queries will then be enabled.
# Rails.application.config.active_support.executor_around_test_case = true
# Define the isolation level of most of Rails internal state.
# If you use a fiber based server or job processor, you should set it to `:fiber`.
# Otherwise the default of `:thread` if preferable.
# Rails.application.config.active_support.isolation_level = :thread
# Set both the `:open_timeout` and `:read_timeout` values for `:smtp` delivery method.
# Rails.application.config.action_mailer.smtp_timeout = 5
# The ActiveStorage video previewer will now use scene change detection to generate
# better preview images (rather than the previous default of using the first frame
# of the video).
# Rails.application.config.active_storage.video_preview_arguments =
# "-vf 'select=eq(n\\,0)+eq(key\\,1)+gt(scene\\,0.015),loop=loop=-1:size=2,trim=start_frame=1' -frames:v 1 -f image2"
# Automatically infer `inverse_of` for associations with a scope.
# Rails.application.config.active_record.automatic_scope_inversing = true
# Raise when running tests if fixtures contained foreign key violations
# Rails.application.config.active_record.verify_foreign_keys_for_fixtures = true
# Disable partial inserts.
# This default means that all columns will be referenced in INSERT queries
# regardless of whether they have a default or not.
# Rails.application.config.active_record.partial_inserts = false
# Protect from open redirect attacks in `redirect_back_or_to` and `redirect_to`.
# Rails.application.config.action_controller.raise_on_open_redirects = true
# Change the variant processor for Active Storage.
# Changing this default means updating all places in your code that
# generate variants to use image processing macros and ruby-vips
# operations. See the upgrading guide for detail on the changes required.
# The `:mini_magick` option is not deprecated; it's fine to keep using it.
# Rails.application.config.active_storage.variant_processor = :vips
# Enable parameter wrapping for JSON.
# Previously this was set in an initializer. It's fine to keep using that initializer if you've customized it.
# To disable parameter wrapping entirely, set this config to `false`.
# Rails.application.config.action_controller.wrap_parameters_by_default = true
# Specifies whether generated namespaced UUIDs follow the RFC 4122 standard for namespace IDs provided as a
# `String` to `Digest::UUID.uuid_v3` or `Digest::UUID.uuid_v5` method calls.
#
# See https://guides.rubyonrails.org/configuring.html#config-active-support-use-rfc4122-namespaced-uuids for
# more information.
# Rails.application.config.active_support.use_rfc4122_namespaced_uuids = true
# Change the default headers to disable browsers' flawed legacy XSS protection.
# Rails.application.config.action_dispatch.default_headers = {
# "X-Frame-Options" => "SAMEORIGIN",
# "X-XSS-Protection" => "0",
# "X-Content-Type-Options" => "nosniff",
# "X-Download-Options" => "noopen",
# "X-Permitted-Cross-Domain-Policies" => "none",
# "Referrer-Policy" => "strict-origin-when-cross-origin"
# }
# ** Please read carefully, this must be configured in config/application.rb **
# Change the format of the cache entry.
# Changing this default means that all new cache entries added to the cache
# will have a different format that is not supported by Rails 6.1 applications.
# Only change this value after your application is fully deployed to Rails 7.0
# and you have no plans to rollback.
# When you're ready to change format, add this to `config/application.rb` (NOT this file):
# config.active_support.cache_format_version = 7.0
# Cookie serializer: 2 options
#
# If you're upgrading and haven't set `cookies_serializer` previously, your cookie serializer
# is `:marshal`. The default for new apps is `:json`.
#
# Rails.application.config.action_dispatch.cookies_serializer = :json
#
#
# To migrate an existing application to the `:json` serializer, use the `:hybrid` option.
#
# Rails transparently deserializes existing (Marshal-serialized) cookies on read and
# re-writes them in the JSON format.
#
# It is fine to use `:hybrid` long term; you should do that until you're confident *all* your cookies
# have been converted to JSON. To keep using `:hybrid` long term, move this config to its own
# initializer or to `config/application.rb`.
#
# Rails.application.config.action_dispatch.cookies_serializer = :hybrid
#
#
# If your cookies can't yet be serialized to JSON, keep using `:marshal` for backward-compatibility.
#
# If you have configured the serializer elsewhere, you can remove this section of the file.
#
# See https://guides.rubyonrails.org/action_controller_overview.html#cookies for more information.

16
config/proxies.yml
View File

@@ -1,19 +1,11 @@
default: &default
direct:
http: http://localhost:9292
gallery_dl: http://localhost:5001
http: http://127.0.0.1:9292
gallery_dl: http://127.0.0.1:5001
proxy-1:
http: http://10.166.33.10:9292
gallery_dl: http://10.166.33.10:5001
dedipath-1:
http: http://10.200.0.6:9292
gallery_dl: http://10.200.0.6:5001
serverhost-1:
http: http://10.200.0.7:9292
gallery_dl: http://10.200.0.7:5001
http: http://127.0.0.1:9292
gallery_dl: http://127.0.0.1:5001
production:
<<: *default

30
config/routes.rb
View File

@@ -1,11 +1,11 @@
Rails.application.routes.draw do
root to: "pages#root"
root to: 'pages#root'
# Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
namespace :api do
namespace :fa do
get :similar_users, to: "/domain/fa/api#similar_users"
get :search_user_names, to: "/domain/fa/api#search_user_names"
get :similar_users, to: '/domain/fa/api#similar_users'
get :search_user_names, to: '/domain/fa/api#search_user_names'
end
end
@@ -20,28 +20,30 @@ Rails.application.routes.draw do
get :contents, on: :member
end
get "us/:script", to: "user_scripts#get", constraints: { script: /.*/ }
get 'us/:script', to: 'user_scripts#get', constraints: { script: /.*/ }
scope constraints: VpnOnlyRouteConstraint.new do
mount PgHero::Engine => "pghero"
mount GoodJob::Engine => "jobs"
mount PgHero::Engine => 'pghero'
mount GoodJob::Engine => 'jobs'
namespace :api do
get "search/user/:prefix", to: "search#user"
get 'search/user/:prefix', to: 'search#user'
namespace :fa do
post :enqueue_objects, to: "/domain/fa/api#enqueue_objects"
post :object_statuses, to: "/domain/fa/api#object_statuses"
post :enqueue_objects, to: '/domain/fa/api#enqueue_objects'
post :object_statuses, to: '/domain/fa/api#object_statuses'
get :export_user, to: '/domain/fa/export#user'
end
namespace :twitter do
post :enqueue_objects, to: "/domain/twitter/api#enqueue_objects"
post :object_statuses, to: "/domain/twitter/api#object_statuses"
post :enqueue_objects, to: '/domain/twitter/api#enqueue_objects'
post :object_statuses, to: '/domain/twitter/api#object_statuses'
end
end
namespace :domain do
namespace :fa do
resources :users, param: :url_name, only: [] do
resources :posts, controller: "/domain/fa/posts"
resources :posts, controller: '/domain/fa/posts'
end
resources :posts, param: :fa_id, only: [:inde] do
post :scan_post, on: :member
@@ -51,10 +53,10 @@ Rails.application.routes.draw do
resources :blobs, only: [:show], slug: :sha256
resources :log_entries, only: [:index, :show] do
resources :log_entries, only: %i[index show] do
get :stats, on: :collection
get "filter/*filter", on: :collection, action: :index, constraints: { filter: /.*/ }
get 'filter/*filter', on: :collection, action: :index, constraints: { filter: /.*/ }
end
end
end

2
config/webpack/serverWebpackConfig.js
View File

@@ -1,4 +1,4 @@
// The source code including full typescript support is available at:
// The source code including full typescript support is available at:
// https://github.com/shakacode/react_on_rails_demo_ssr_hmr/blob/master/config/webpack/serverWebpackConfig.js
const { merge, config } = require('shakapacker');

3
config/webpack/webpack.config.js
View File

@@ -3,7 +3,8 @@ const { existsSync } = require('fs')
const { resolve } = require('path')
const envSpecificConfig = () => {
const path = resolve(__dirname, `${env.nodeEnv}.js`)
const railsEnv = env.railsEnv;
const path = resolve(__dirname, `${railsEnv}.js`)
if (existsSync(path)) {
console.log(`Loading ENV specific webpack configuration file ${path}`)
return require(path)

2
config/webpack/webpackConfig.js
View File

@@ -1,4 +1,4 @@
// The source code including full typescript support is available at:
// The source code including full typescript support is available at:
// https://github.com/shakacode/react_on_rails_demo_ssr_hmr/blob/master/config/webpack/webpackConfig.js
const clientWebpackConfig = require('./clientWebpackConfig');

1
config/webpacker.yml
View File

@@ -47,6 +47,7 @@ development: &development
staging:
<<: *development
compile: true
test:
<<: *default

22
db/migrate/20241216180917_add_service_name_to_active_storage_blobs.active_storage.rb Normal file
View File

@@ -0,0 +1,22 @@
# This migration comes from active_storage (originally 20190112182829)
class AddServiceNameToActiveStorageBlobs < ActiveRecord::Migration[6.0]
def up
return unless table_exists?(:active_storage_blobs)
unless column_exists?(:active_storage_blobs, :service_name)
add_column :active_storage_blobs, :service_name, :string
if configured_service = ActiveStorage::Blob.service.name
ActiveStorage::Blob.unscoped.update_all(service_name: configured_service)
end
change_column :active_storage_blobs, :service_name, :string, null: false
end
end
def down
return unless table_exists?(:active_storage_blobs)
remove_column :active_storage_blobs, :service_name
end
end

28
db/migrate/20241216180918_create_active_storage_variant_records.active_storage.rb Normal file
View File

@@ -0,0 +1,28 @@
# This migration comes from active_storage (originally 20191206030411)
class CreateActiveStorageVariantRecords < ActiveRecord::Migration[6.0]
def change
return unless table_exists?(:active_storage_blobs)
# Use Active Record's configured type for primary key
create_table :active_storage_variant_records, id: primary_key_type, if_not_exists: true do |t|
t.belongs_to :blob, null: false, index: false, type: blobs_primary_key_type
t.string :variation_digest, null: false
t.index %i[blob_id variation_digest], name: 'index_active_storage_variant_records_uniqueness', unique: true
t.foreign_key :active_storage_blobs, column: :blob_id
end
end
private
def primary_key_type
config = Rails.configuration.generators
config.options[config.orm][:primary_key_type] || :primary_key
end
def blobs_primary_key_type
pkey_name = connection.primary_key(:active_storage_blobs)
pkey_column = connection.columns(:active_storage_blobs).find { |c| c.name == pkey_name }
pkey_column.bigint? ? :bigint : pkey_column.type
end
end

8
db/migrate/20241216180919_remove_not_null_on_active_storage_blobs_checksum.active_storage.rb Normal file
View File

@@ -0,0 +1,8 @@
# This migration comes from active_storage (originally 20211119233751)
class RemoveNotNullOnActiveStorageBlobsChecksum < ActiveRecord::Migration[6.0]
def change
return unless table_exists?(:active_storage_blobs)
change_column_null(:active_storage_blobs, :checksum, true)
end
end

8547
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

5
package.json
View File

@@ -19,12 +19,13 @@
"mini-css-extract-plugin": "^2.7.5",
"prop-types": "^15.8.1",
"react": "^18.2.0",
"react_ujs": "^2.6.2",
"react-dom": "^18.2.0",
"react-on-rails": "13.3.3",
"react_ujs": "^2.6.2",
"shakapacker": "6.6.0",
"style-loader": "^3.3.2",
"terser-webpack-plugin": "5"
"terser-webpack-plugin": "5",
"yarn": "^1.22.22"
},
"version": "0.1.0",
"babel": {

1292
yarn.lock
View File

File diff suppressed because it is too large Load Diff