separate dev env for curtus / regius

Gemfile

@@ -53,12 +53,12 @@ gem "bootsnap", require: false
 # Use Active Storage variants [https://guides.rubyonrails.org/active_storage_overview.html#transforming-images]
 # gem "image_processing", "~> 1.2"
 
-group :development, :test do
+group :development_regius, :development_curtus, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
   gem "debug", platforms: %i[ mri mingw x64_mingw ]
 end
 
-group :development do
+group :development_regius, :development_curtus do
   # Use console on exceptions pages [https://github.com/rails/web-console]
   gem "web-console"
 
@@ -101,3 +101,6 @@ gem "concurrent-ruby-ext", require: "concurrent"
 gem "concurrent-ruby-edge", require: "concurrent-edge"
 gem "pluck_each"
 gem "good_job"
+
+gem "neighbor"
+gem "disco"

Gemfile.lock

@@ -112,11 +112,15 @@ GEM
     diffy (3.4.2)
     discard (1.2.1)
       activerecord (>= 4.2, < 8)
+    disco (0.4.0)
+      libmf (>= 0.2)
+      numo-narray
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     erubi (1.12.0)
     et-orbi (1.2.7)
       tzinfo
+    ffi (1.15.5)
     fugit (1.8.1)
       et-orbi (~> 1, >= 1.2.7)
       raabro (~> 1.4)
@@ -156,6 +160,8 @@ GEM
       activerecord
       kaminari-core (= 1.2.2)
     kaminari-core (1.2.2)
+    libmf (0.3.0)
+      ffi
     loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -171,6 +177,8 @@ GEM
     mini_portile2 (2.8.1)
     minitest (5.17.0)
     msgpack (1.6.0)
+    neighbor (0.2.2)
+      activerecord (>= 5.2)
     net-imap (0.3.4)
       date
       net-protocol
@@ -184,6 +192,7 @@ GEM
     nokogiri (1.14.1)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
+    numo-narray (0.9.2.1)
     pg (1.4.5)
     pluck_each (0.2.0)
       activerecord (> 3.2.0)
@@ -318,12 +327,14 @@ DEPENDENCIES
   debug
   diffy
   discard
+  disco
   good_job
   http-cookie
   importmap-rails
   influxdb-client
   jbuilder
   kaminari
+  neighbor
   nokogiri
   pg
   pluck_each

app/models/domain/fa/user_factors.rb

@@ -0,0 +1,4 @@
+class Domain::Fa::UserFactor
+  belongs_to :user, class_name: "::Domain::Fa::User"
+  has_neighbors :factors
+end

config/cookies/fa.yml

@@ -82,12 +82,18 @@ testcookies: &testcookies
         value: oaidvalue
         path: /
 
-development:
+development_regius:
   direct: *ddwhatnow
   proxy-1: *vipvillageworker
   dedipath-1: *blazeandwish
   serverhost-1: *cottoniq
 
+development_curtus:
+  direct: *testcookies
+  proxy-1: *testcookies
+  dedipath-1: *testcookies
+  serverhost-1: *testcookies
+
 worker:
   direct: *ddwhatnow
   proxy-1: *vipvillageworker

config/database.yml

@@ -19,6 +19,16 @@ redux_prod: &redux_prod
   migrations_paths: db/redux_migrate
   pool: 4
 
+curtus_redux_dev: &curtus_redux_dev
+  adapter: postgresql
+  host: localhost
+  port: 5432
+  database: redux_development
+  username: scraper_redux
+  password: pdkFLqRmQwPUPaDDC4pX
+  migrations_paths: db/redux_migrate
+  pool: 4
+
 legacy_prod: &legacy_prod
   adapter: postgresql
   host: postgres.local
@@ -50,12 +60,18 @@ local_legacy_test: &local_legacy_test
   migrations_paths: db/legacy_migrate
   pool: 4
 
-development:
+development_regius:
   redux:
     <<: *redux_prod
   legacy:
     <<: *legacy_prod
 
+development_curtus:
+  redux:
+    <<: *curtus_redux_dev
+  # legacy:
+  #   <<: *legacy_prod
+
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
 # Do not set this db to the same as development or production.

config/environments/development_curtus.rb

@@ -0,0 +1,76 @@
+require "active_support/core_ext/integer/time"
+
+Rack::MiniProfiler.config.position = "top-right"
+Rack::MiniProfiler.config.skip_paths = [/\/api\/.+/]
+Rack::MiniProfiler.config.disable_caching = false
+
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+  config.web_console.development_only = false
+  config.hosts << "localhost"
+
+  # In the development environment your application's code is reloaded any time
+  # it changes. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports.
+  config.consider_all_requests_local = true
+
+  # Enable server timing
+  config.server_timing = true
+
+  # Enable/disable caching. By default caching is disabled.
+  # Run rails dev:cache to toggle caching.
+  if Rails.root.join("tmp/caching-dev.txt").exist?
+    config.action_controller.perform_caching = true
+    config.action_controller.enable_fragment_cache_logging = true
+
+    config.cache_store = :memory_store
+    config.public_file_server.headers = {
+      "Cache-Control" => "public, max-age=#{2.days.to_i}",
+    }
+  else
+    config.action_controller.perform_caching = false
+
+    config.cache_store = :null_store
+  end
+
+  # Store uploaded files on the local file system (see config/storage.yml for options).
+  config.active_storage.service = :local
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  config.action_mailer.perform_caching = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise exceptions for disallowed deprecations.
+  config.active_support.disallowed_deprecation = :raise
+
+  # Tell Active Support which deprecation messages to disallow.
+  config.active_support.disallowed_deprecation_warnings = []
+
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
+
+  # Highlight code that triggered database queries in logs.
+  config.active_record.verbose_query_logs = true
+
+  # Suppress logger output for asset requests.
+  config.assets.quiet = true
+
+  # Raises error for missing translations.
+  # config.i18n.raise_on_missing_translations = true
+
+  # Annotate rendered view with file names.
+  # config.action_view.annotate_rendered_view_with_filenames = true
+
+  # Uncomment if you wish to allow Action Cable access from any origin.
+  # config.action_cable.disable_request_forgery_protection = true
+end

config/environments/development_regius.rb

@@ -6,6 +6,7 @@ Rack::MiniProfiler.config.disable_caching = false
 
 Rails.application.configure do
   # Settings specified here will take precedence over those in config/application.rb.
+  config.web_console.development_only = false
   config.hosts << "scraper.local"
 
   # In the development environment your application's code is reloaded any time

config/influxdb.yml

@@ -1,4 +1,10 @@
-development:
+development_regius:
+  host: "http://grafana.local:8086"
+  token: "W2ikhmpQxQHZAStgSEK6s6aJxnOqeD4Zz2MI1m_lnD1JX57V9Esqm0zXb3DWbN7Gnj2GdmF_YrcvE8cy6NbIqQ=="
+  org: "primary"
+  bucket: "redux-scraper"
+
+development_curtus:
   host: "http://grafana.local:8086"
   token: "W2ikhmpQxQHZAStgSEK6s6aJxnOqeD4Zz2MI1m_lnD1JX57V9Esqm0zXb3DWbN7Gnj2GdmF_YrcvE8cy6NbIqQ=="
   org: "primary"

config/proxies.yml

@@ -18,9 +18,15 @@ default: &default
 production:
   <<: *default
 
-development:
+development_regius:
   <<: *default
 
+development_curtus:
+  direct: {}
+  proxy-1: {}
+  dedipath-1: {}
+  serverhost-1: {}
+
 worker:
   <<: *default
 

db/legacy_schema.rb

@@ -394,4 +394,5 @@ ActiveRecord::Schema[7.0].define(version: 0) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
   end
+
 end

db/redux_migrate/20230131012417_create_http_log_entries.rb

@@ -1,5 +1,8 @@
 class CreateHttpLogEntries < ActiveRecord::Migration[7.0]
   def change
+    enable_extension "pg_stat_statements"
+    enable_extension "pg_trgm"
+
     create_table :http_log_entries do |t|
       t.string :uri_scheme, null: false
       t.string :uri_host, null: false
@@ -20,6 +23,8 @@ class CreateHttpLogEntries < ActiveRecord::Migration[7.0]
 
       t.datetime :requested_at, null: false
       t.timestamps null: false
+
+      t.index ["uri_host", "uri_path", "uri_query"], name: "index_http_log_entries_on_uri_host_path_query"
     end
 
     create_table :http_log_entry_headers do |t|

db/redux_migrate/20230207210950_create_versions.rb

@@ -1,9 +1,7 @@
 # This migration creates the `versions` table, the only schema PT requires.
 # All other migrations PT provides are optional.
 class CreateVersions < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom")
-
+  def up
     create_table :versions do |t|
       t.string :item_type, null: false
       t.bigint :item_id, null: false
@@ -14,4 +12,8 @@ class CreateVersions < ActiveRecord::Migration[7.0]
     end
     add_index :versions, %i[item_type item_id]
   end
+
+  def down
+    raise("boom")
+  end
 end

db/redux_migrate/20230207211826_create_domain_fa_posts.rb

@@ -1,7 +1,5 @@
 class CreateDomainFaPosts < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom")
-
+  def up
     create_table :domain_fa_posts do |t|
       t.integer :fa_id
       t.index :fa_id, unique: true
@@ -43,4 +41,8 @@ class CreateDomainFaPosts < ActiveRecord::Migration[7.0]
     # Fa::PostHttpLogEntryJoin.post_id -> HttpLogEntry.id
     add_foreign_key :domain_fa_post_http_log_entry_joins, :http_log_entries, column: :entry_id, primary_key: :id, validate: true
   end
+
+  def down
+    raise("boom")
+  end
 end

db/redux_migrate/20230213013141_create_domain_fa_users.rb

@@ -1,7 +1,5 @@
 class CreateDomainFaUsers < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom")
-
+  def up
     create_table :domain_fa_users do |t|
       t.string :name, null: false, index: { unique: true }
       t.string :full_name
@@ -34,4 +32,8 @@ class CreateDomainFaUsers < ActiveRecord::Migration[7.0]
     add_foreign_key :domain_fa_user_http_log_entry_joins, :http_log_entries, column: :entry_id, validate: true
     add_foreign_key :domain_fa_posts, :domain_fa_users, column: :creator_id, validate: true
   end
+
+  def down
+    raise("boom")
+  end
 end

db/redux_migrate/20230220062108_create_delayed_jobs.rb

@@ -1,7 +1,5 @@
 class CreateDelayedJobs < ActiveRecord::Migration[7.0]
   def self.up
-    raise("boom")
-
     create_table :delayed_jobs do |table|
       table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue
       table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually.
@@ -20,7 +18,6 @@ class CreateDelayedJobs < ActiveRecord::Migration[7.0]
 
   def self.down
     raise("boom")
-
     drop_table :delayed_jobs
   end
 end

db/redux_migrate/20230226223215_create_domain_e621_posts.rb

@@ -1,6 +1,5 @@
 class CreateDomainE621Posts < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom!")
+  def up
     create_table :domain_e621_posts do |t|
       t.integer :e621_id, null: false
       t.integer :state, null: false
@@ -35,4 +34,8 @@ class CreateDomainE621Posts < ActiveRecord::Migration[7.0]
 
     create_versions_table :domain_e621_posts
   end
+
+  def down
+    raise("boom!")
+  end
 end

db/redux_migrate/20230226224328_create_domain_e621_tags.rb

@@ -1,7 +1,5 @@
 class CreateDomainE621Tags < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom")
-
+  def up
     create_table :domain_e621_tags do |t|
       t.string :name, null: false
       t.integer :e621_id
@@ -13,4 +11,8 @@ class CreateDomainE621Tags < ActiveRecord::Migration[7.0]
       t.index :e621_id, unique: true
     end
   end
+
+  def down
+    raise("boom")
+  end
 end

db/redux_migrate/20230226224331_create_domain_e621_taggings.rb

@@ -1,11 +1,13 @@
 class CreateDomainE621Taggings < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom")
-
+  def up
     create_table :domain_e621_taggings do |t|
       t.integer :type
       t.references :post
       t.references :tag
     end
   end
+
+  def down
+    raise("boom")
+  end
 end

db/redux_migrate/20230301013456_add_performed_by_to_http_log_entries.rb

@@ -1,8 +1,6 @@
 class AddPerformedByToHttpLogEntries < ActiveRecord::Migration[7.0]
   def up
-    add_column :http_log_entries, :performed_by, :integer
-    HttpLogEntry.update_all(performed_by: 0) # direct
-    change_column :http_log_entries, :performed_by, null: false
+    add_column :http_log_entries, :performed_by, :integer, null: false
   end
 
   def down

db/redux_migrate/20230308044852_create_twitter_tweets.rb

@@ -1,7 +1,5 @@
 class CreateTwitterTweets < ActiveRecord::Migration[7.0]
-  def change
-    raise("boom")
-
+  def up
     # for twitter tables -
     # state - enum (ok, error)
     # state_detail - json, contents depends on state, grab-bag of debugging info
@@ -83,4 +81,8 @@ class CreateTwitterTweets < ActiveRecord::Migration[7.0]
       validate: true,
     )
   end
+
+  def down
+    raise("boom")
+  end
 end

db/redux_migrate/20230330130323_install_neighbor_vector.rb

@@ -0,0 +1,5 @@
+class InstallNeighborVector < ActiveRecord::Migration[7.0]
+  def change
+    enable_extension "vector"
+  end
+end

db/redux_migrate/20230330130506_create_domain_fa_user_factors_table.rb

@@ -0,0 +1,11 @@
+class CreateDomainFaUserFactorsTable < ActiveRecord::Migration[7.0]
+  def change
+    create_table :domain_fa_user_factors do |t|
+      t.references :user
+      t.vector :factors, limit: 20
+      t.timestamps
+    end
+
+    add_foreign_key :domain_fa_user_factors, :domain_fa_users, column: :user_id, primary_key: :id, validate: true
+  end
+end

db/schema.rb

@@ -10,12 +10,13 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_03_25_124842) do
+ActiveRecord::Schema[7.0].define(version: 2023_03_30_130506) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
+  enable_extension "vector"
 
   create_table "blob_entries", id: false, force: :cascade do |t|
     t.binary "sha256", null: false
@@ -42,9 +43,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_25_124842) do
     t.datetime "updated_at"
     t.string "signature"
     t.jsonb "args"
-    t.index ["priority", "run_at"], name: "delayed_jobs_priority_run_at_idx"
-    t.index ["queue"], name: "delayed_jobs_queue_idx"
-    t.index ["signature"], name: "delayed_jobs_signature_idx", unique: true
+    t.index ["priority", "run_at"], name: "delayed_jobs_priority"
   end
 
   create_table "domain_e621_post_versions", force: :cascade do |t|
@@ -131,6 +130,14 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_25_124842) do
     t.index ["file_id"], name: "index_domain_fa_posts_on_file_id"
   end
 
+  create_table "domain_fa_user_factors", force: :cascade do |t|
+    t.bigint "user_id"
+    t.vector "factors", limit: 20
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["user_id"], name: "index_domain_fa_user_factors_on_user_id"
+  end
+
   create_table "domain_fa_users", force: :cascade do |t|
     t.string "name", null: false
     t.string "full_name"
@@ -195,10 +202,10 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_25_124842) do
   end
 
   create_table "domain_twitter_users", force: :cascade do |t|
-    t.integer "tw_id"
     t.integer "state"
     t.json "state_detail"
     t.json "raw_data"
+    t.integer "tw_id"
     t.string "name", null: false
     t.string "nick"
     t.string "description"
@@ -333,6 +340,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_03_25_124842) do
   add_foreign_key "domain_e621_post_versions", "domain_e621_posts", column: "item_id"
   add_foreign_key "domain_fa_posts", "domain_fa_users", column: "creator_id"
   add_foreign_key "domain_fa_posts", "http_log_entries", column: "file_id"
+  add_foreign_key "domain_fa_user_factors", "domain_fa_users", column: "user_id"
   add_foreign_key "domain_twitter_medias", "domain_twitter_tweets", column: "tweet_id"
   add_foreign_key "domain_twitter_medias", "http_log_entries", column: "file_id"
   add_foreign_key "domain_twitter_tweets", "domain_twitter_users", column: "author_id"