dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add procfile, remove old log watcher stuff

Browse Source
This commit is contained in:
Dylan Knutson
2023-03-27 20:01:51 +09:00
parent 3f349bb000
commit 41bcabcf4a
9 changed files with 39 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
Gemfile
View File

@@ -70,13 +70,6 @@ group :development do
gem "rufo", require: false
end
group :log_watcher do
gem "curses"
gem "listen"
gem "concurrent-ruby-ext", require: "concurrent"
gem "concurrent-ruby-edge", require: "concurrent-edge"
end
group :test do
# Use system testing [https://guides.rubyonrails.org/testing.html#system-testing]
gem "capybara"
@@ -104,10 +97,7 @@ gem "daemons"
gem "ripcord"
gem "influxdb-client"
gem "discard"
gem "concurrent-ruby-ext", require: "concurrent"
gem "concurrent-ruby-edge", require: "concurrent-edge"
gem "good_job"
# gem 'cli-ui'
# gem "paper_trail"
# gem "paper_trail-hashdiff"
# gem "hashdiff"

2
Gemfile.lock
View File

@@ -102,7 +102,6 @@ GEM
concurrent-ruby (= 1.2.0)
crass (1.0.6)
curb (1.0.5)
curses (1.4.4)
daemons (1.4.1)
date (3.3.3)
debug (1.7.1)
@@ -319,7 +318,6 @@ DEPENDENCIES
concurrent-ruby-edge
concurrent-ruby-ext
curb
curses
daemons
debug
diffy

42
Rakefile
View File

@@ -8,7 +8,6 @@ Rails.application.load_tasks
$LOAD_PATH << Rails.root.join("rake")
Rake.application.rake_require "sst"
Rake.application.rake_require "log_entry"
Rake.application.rake_require "worker"
Rake.application.rake_require "metrics"
Rake.application.rake_require "fa"
Rake.application.rake_require "e621"
@@ -26,22 +25,41 @@ task :set_logger_stdout => :environment do
ActiveRecord::Base.logger = nil
end
task :good_job_cron => :environment do
loop do
Rake::Task["fa:browse_page_job"].execute
sleep 60
end
end
task :good_job do
ENV["RAILS_ENV"] = "worker"
ENV["GOOD_JOB_MAX_THREADS"] = "1"
ENV["GOOD_JOB_QUEUES"] = [
"manual:4",
"static_file:4",
"+static_file,fa_post:4",
"+fa_user_page,fa_user_gallery:2",
"-static_file,fa_post,manual,fa_user_page,fa_user_gallery,twitter_timeline_tweets:2",
].join(";")
proxies = ["direct", "proxy-1", "dedipath-1", "serverhost-1"]
proxy = ENV["proxy"]
raise("'proxy' must be set") unless proxy
raise("'proxy' must be one of #{proxies}") unless proxies.include?(proxy)
cmd = "bundle exec good_job --queue-select-limit=256"
puts "$> #{cmd}"
env_hash = {
"RAILS_ENV" => "worker",
"GOOD_JOB_POLL_INTERVAL" => "30",
"GOOD_JOB_MAX_CACHE" => "100",
"GOOD_JOB_QUEUE_SELECT_LIMIT" => "128",
"GOOD_JOB_MAX_THREADS" => "1",
"GOOD_JOB_QUEUES" => [
"manual:4",
"static_file:4",
"+static_file,fa_post:4",
"+fa_user_page,fa_user_gallery:2",
"-static_file,fa_post,manual,fa_user_page,fa_user_gallery,twitter_timeline_tweets:2",
].join(";"),
}
env_hash.each do |key, value|
ENV[key] = value
puts "$> #{key.light_black.bold} = #{value.bold}"
end
cmd = "bundle exec good_job"
puts "$> #{cmd.bold}"
exec(cmd)
end

46
app/lib/log_watcher/ansi.rb
View File

@@ -1,46 +0,0 @@
class LogWatcher::Ansi
ANSI_COLOR_CODE = {
0 => "black",
1 => "red",
2 => "green",
3 => "yellow",
4 => "blue",
5 => "purple",
6 => "cyan",
7 => "white",
}
def sanitize_ansi_data(data)
data.gsub!(/\033\[1m/, "<b>")
data.gsub!(/\033\[0m/, "</b></span>")
data.gsub!(/\033\[[\d\;]{2,}m.*?<\/b><\/span>/) { |data|
span = "<span style='"
content = ""
/\033\[([\d\;]{2,})m(.*?)<\/b><\/span>/.match(data) { |m|
content = m[2]
m[1].split(";").each do |code|
#puts code
if match = /(\d)(\d)/.match(code)
case match[1]
when "3"
span += "color: #{ANSI_COLOR_CODE[match[2].to_i]}; "
when "4"
span += "background-color: #{ANSI_COLOR_CODE[match[2].to_i]}; "
else
#do nothing
end
else
span += "font-weight:bold; "
end
end
}
span += "'>"
"#{span}#{content}</b></span>"
}
data
end
data = "ls\r\n\e[0m\e[30;42mat-spi2\e[0m \e[01;34mpulse-PKdhtXMmr18n\e[0m test.rb \e[01;34mvmware-pocha\e[0m\r\neye.log \e[01;34mpulse-PNbbIFaT02i5\e[0m tmpFHAF0y \e[01;34mvmware-root\e[0m\r\n\e[01;34mkeyring-tpfZ5S\e[0m \e[01;34mssh-oSjkDqzK2269\e[0m tmp.txt\r\n\e[01;34mpulse-fhKjxmUqywSU\e[0m \e[01;34mtest-app\e[0m unity_support_test.1\r\n\e[32mpocha@ubuntu\e[0m \e[35m/tmp\e[0m$ "
puts sanitize_ansi_data(data)
end

315
app/lib/log_watcher/app.rb
View File

@@ -1,315 +0,0 @@
Bundler.require(:log_watcher, :development)
require "concurrent"
require "open3"
class LogWatcher::App
include Curses
WorkerCommandDef = Struct.new(:name, :env, :command)
WORKER_COMMAND_DEFS = [
WorkerCommandDef.new(
"default",
{ "QUEUE" => "default" },
"rake set_logger_stdout jobs:work",
),
WorkerCommandDef.new(
"fa_user_page",
{ "QUEUE" => "fa_user_page" },
"rake set_logger_stdout jobs:work",
),
WorkerCommandDef.new(
"fa_user_gallery",
{ "QUEUE" => "fa_user_gallery" },
"rake set_logger_stdout jobs:work",
),
WorkerCommandDef.new(
"fa_post",
{ "QUEUE" => "fa_post" },
"rake set_logger_stdout jobs:work",
),
]
def initialize
@event_channel = Concurrent::Channel.new(capacity: 32)
@workers = WORKER_COMMAND_DEFS.map do |command_def|
Worker.start(command_def, @event_channel)
end
@active_worker_idx = 0
@info_line = "waiting"
@running = true
end
Worker = Struct.new(:command_def, :pid, :thread, :lines) do
def self.start(command_def, event_channel)
instance = new(command_def, "(starting)", nil, [])
instance.thread = make_instance_thread(instance, event_channel)
instance
end
def self.make_instance_thread(worker, event_channel)
Thread.new do
# puts "starting worker '#{worker.name}' with env #{worker.env}: #{worker.command}"
Open3.popen2e(worker.env, worker.command) do |in_pipe, out_pipe, wait|
worker.pid = wait.pid
while line = out_pipe.gets
event_channel << { type: :on_line, worker: worker, line: line.chomp }
end
event_channel << { type: :on_worker_finish, worker: worker }
ensure
# puts "stopped worker '#{worker.name}'"
begin
Process.kill("INT", wait.pid)
Process.wait wait.pid
rescue
end
end
end
end
def name
command_def.name
end
def env
command_def.env
end
def command
command_def.command
end
def stop
self.thread.kill.join
end
end
def activate_color(fg, bg)
@color_pairs ||= {}
@color_pairs[[fg, bg]] ||= begin
pair_idx = @color_pairs.size + 1
init_pair(pair_idx, fg, bg)
pair_idx
end
@win.attron(color_pair(@color_pairs[[fg, bg]]))
end
def reset_color
activate_color(-1, -1)
end
def run
init_screen
start_color
use_default_colors
curs_set(0)
noecho
@win = Curses::Window.new(0, 0, 0, 0)
@getch_thread = Thread.new do
loop do
@event_channel << { type: :on_char, char: @win.getch.to_s }
end
end
loop do
take_message while @running && @event_channel.size > 0
break unless @running
clear_screen
draw_active_worker
# worker line - second from bottom
@win.setpos(@win.maxy - 2, 0)
@workers.each_with_index do |worker, idx|
if idx > 0
@win << " "
end
if idx == @active_worker_idx
activate_color(3, -1) # yellow
else
reset_color # white
end
@win << worker.name
end
clrtoeol
@win << "\n"
# status line goes - bottom
@win.setpos(@win.maxy - 1, 0)
activate_color(-1, -1)
@win << @info_line
clrtoeol
# and tell curses to do its thing
@win.refresh
rescue Interrupt
@event_channel << { type: :quit }
end
ensure
@workers.each do |worker|
worker.stop
end
@getch_thread.kill.join if @getch_thread
curs_set(1)
close_screen
end
private
def take_message
message = @event_channel.take
case message[:type]
when :on_worker_finish
@workers.reject! do |worker|
finished = worker == message[:worker]
puts "worker #{worker.name} finished" if finished
finished
end
@active_worker_idx = [[@active_worker_idx, @workers.length - 1].min, 0].max
when :on_char
on_char(message[:char])
when :on_line
on_line(message[:worker], message[:line])
when :quit
@running = false
else raise "unknown message type: #{message.inspect}"
end
end
def clear_screen
@win.erase
@win.setpos(0, 0)
# @win.maxy.times {
# @win.deleteln()
# clrtoeol
# @win << "\n"
# }
end
def active_worker
@workers[@active_worker_idx]
end
WORKER_AREA_MARGIN_BOTTOM = 2
def draw_active_worker
worker = active_worker
return unless worker
@win.setpos(0, 0)
@win << "worker "
@win.attron(A_BOLD)
activate_color(-1, -1)
@win << worker.name
@win.attroff(A_BOLD)
clrtoeol
@win << "\n"
@win << "-" * (@win.maxx - 1)
clrtoeol
@win << "\n"
# can draw down to third to the last line
max_lines = @win.maxy - @win.cury - WORKER_AREA_MARGIN_BOTTOM - 1
start_lines_idx = -[max_lines, worker.lines.length].min
lines = worker.lines[start_lines_idx..-1]
lines.each do |line|
write_parsed_line(line)
clrtoeol
@win << "\n"
end
(@win.maxy - @win.cury - WORKER_AREA_MARGIN_BOTTOM - 1).times {
clrtoeol
@win << "\n"
}
@win << "-" * (@win.maxx - 2)
clrtoeol
@win << "\n"
end
def write_parsed_line(line)
chars_max = @win.maxx - @win.curx - 2
line.each do |part|
case part[0]
when :text
@win << part[1][0..chars_max]
when :ansi
activate_color(part[1][:fg], part[1][:bg])
if part[1][:bold]
@win.attron(A_BOLD)
else
@win.attroff(A_BOLD)
end
end
end
end
def on_line(worker, line)
line_parsed = []
while line.length > 0
idx = line.index("\e[") || line.length
if idx > 0
line_parsed << [:text, line[0...idx]]
line = line[idx..-1]
else
# strip off the escape sequence '\e[' (2 bytes)
line = line[2..-1]
end_idx = line.index("m") || raise("didn't find end of escape")
control_seq = line[0..(end_idx - 1)]
ansi = {
fg: :default,
bg: :default,
bold: false,
}
control_seq.split(";").map(&:to_i).each do |code|
if code == 0
# 0 == reset
# in curses, -1 is "default color"
ansi[:fg] = -1
ansi[:bg] = -1
elsif code == 1
# 1 == bold
ansi[:bold] = true
elsif code >= 30 && code <= 37
ansi[:fg] = code - 30
elsif code >= 90 && code <= 97
# bright foreground
ansi[:fg] = code - (90 - 8)
elsif code >= 100 && code <= 107
# bright background
ansi[:fg] = code - (100 - 8)
elsif code >= 40 && code <= 47
ansi[:bg] = code - 40
elsif code == 39
# 39 == default fg color
ansi[:fg] = -1
elsif code == 49
# 49 == default bg color
ansi[:bg] = -1
else
raise("unknown code #{code}")
end
end
line_parsed << [:ansi, { fg: ansi[:fg], bg: ansi[:bg] }]
line = line[(end_idx + 1)..-1]
end
end
worker.lines << line_parsed
worker.lines.shift while worker.lines.length > 200
end
def on_char(char)
case char
when "q"
@event_channel << { type: :quit }
when "n"
@active_worker_idx = (@active_worker_idx + 1) % @workers.size
when "p"
@active_worker_idx = (@active_worker_idx - 1) % @workers.size
end
end
end

2
rake/fa.rake
View File

@@ -15,7 +15,7 @@ namespace :fa do
Domain::Fa::Job::BrowsePageJob.
set(priority: -20, queue: "manual").
perform_later({})
puts "enqueued browse page job"
puts "#{Time.now} - browse_page_job - Domain::Fa::Job::BrowsePageJob"
end
desc "run a single post scan job"

27
rake/worker.rake
View File

@@ -1,27 +0,0 @@
namespace :worker do
desc "start the log watcher"
task :watch_logs => :environment do
LogWatcher::App.new.run
end
task :print do
puts "PID: #{Process.pid}"
loop do
print "hello! "
puts ENV["msg"].send(ENV["color"].to_sym)
sleep 1
rescue Interrupt
puts "waiting..."
sleep 2
end
end
task :print_100 do
50.times do
puts "this is some text!"
end
loop do
puts "and some more..."
sleep 1
end
end
end

1
spec/jobs/domain/fa/job/browse_page_job_spec.rb
View File

@@ -5,6 +5,7 @@ describe Domain::Fa::Job::BrowsePageJob do
before do
Scraper::HttpFactory.http_client_mock = http_client_mock
end
around do |block|
ColorLogger.quiet(&block)
end

5
worker/Procfile Normal file
View File

@@ -0,0 +1,5 @@
cron: bundle exec rake good_job_cron
direct: bundle exec rake good_job proxy=direct
proxy-1: bundle exec rake good_job proxy=proxy-1
dedipath-1: bundle exec rake good_job proxy=dedipath-1
serverhost-1: bundle exec rake good_job proxy=serverhost-1