dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parse submission date in etc, conver to to utc

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-03 20:10:55 +00:00
parent fc8e74d2fb
commit acbdf72e8e
8 changed files with 42 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/jobs/domain/fa/job/scan_post_job.rb
View File

@@ -107,9 +107,6 @@ class Domain::Fa::Job::ScanPostJob < Domain::Fa::Job::Base
post.num_views = submission.num_views
post.posted_at = submission.posted_date
post.scanned_at = DateTime.current
post.save!
rescue StandardError
binding.pry
end
end

6
app/jobs/domain/inkbunny/job/api_search_page_processor.rb
View File

@@ -168,13 +168,15 @@ class Domain::Inkbunny::Job::ApiSearchPageProcessor
post.creator = creator
post.shallow_updated_at = Time.zone.now
post.title = submission_json["title"]
post.posted_at = Time.zone.parse(submission_json["create_datetime"])
post.posted_at =
Time.parse(submission_json["create_datetime"]).in_time_zone("UTC")
# rarely, a post will have a null last_file_update_datetime.
# This is a bug with Inkbunny, so handle it gracefully.
last_updated_str = submission_json["last_file_update_datetime"]
if last_updated_str.present?
post.last_file_updated_at = Time.zone.parse(last_updated_str)
post.last_file_updated_at =
Time.parse(last_updated_str).in_time_zone("UTC")
end
post.num_files = submission_json["pagecount"]&.to_i

1
app/lib/domain/fa/parser/page.rb
View File

@@ -211,6 +211,7 @@ class Domain::Fa::Parser::Page < Domain::Fa::Parser::Base
end
end
sig { returns(Domain::Fa::Parser::SubmissionParserHelper) }
def submission
@submission ||=
Domain::Fa::Parser::SubmissionParserHelper.new(

10
app/lib/domain/fa/parser/submission_parser_helper.rb
View File

@@ -107,17 +107,17 @@ class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
end
end
sig { returns(ActiveSupport::TimeWithZone) }
def posted_date
tsz = ActiveSupport::TimeZone["Eastern Time (US & Canada)"]
@posted_date ||=
case @page_version
# when VERSION_0
# info_child(6)["title"].strip
when VERSION_0, VERSION_1
idx = elem_idx_after_text_match(info_children, /Posted/)
child = info_children[idx..idx + 5].find { |ic| ic.name == "span" }
date_str = child.try(:[], "title").try(:strip)
if date_str
Time.zone.strptime(
tsz.strptime(
date_str.gsub(/(\d+)(st|nd|rd|th)/, '\1'),
"%b %d, %Y %I:%M %p",
)
@@ -125,10 +125,10 @@ class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
when VERSION_2
date_str = @elem.css(".popup_date").first["title"]
# e.g. 'Feb 18, 2023 06:47 PM'
Time.zone.strptime(date_str, "%b %d, %Y %I:%M %p") if date_str
tsz.strptime(date_str, "%b %d, %Y %I:%M %p") if date_str
else
raise("unimplemented version #{@page_version}")
end
end&.in_time_zone("UTC")
end
def rating

10
app/models/domain/fa/post.rb
View File

@@ -66,6 +66,16 @@ class Domain::Fa::Post < ReduxApplicationRecord
belongs_to :first_gallery_page, class_name: "HttpLogEntry", optional: true
belongs_to :first_seen_entry, class_name: "HttpLogEntry", optional: true
sig { params(id: Integer).returns(T.nilable(Domain::Fa::Post)) }
def self.find_by_fa_id(id)
where(fa_id: id).first
end
sig { params(id: Integer).returns(Domain::Fa::Post) }
def self.find_by_fa_id!(id)
where(fa_id: id).first!
end
sig { returns(String) }
def to_param
self.fa_id.to_s

4
spec/jobs/domain/fa/job/scan_post_job_spec.rb
View File

@@ -94,7 +94,7 @@ describe Domain::Fa::Job::ScanPostJob do
"https://d.furaffinity.net/art/-creeps/1738343855/1738343855.-creeps_slayerlr.jpg",
)
expect(post.posted_at).to be_within(1.second).of(
Time.zone.parse("Jan 31, 2025 12:17 PM"),
Time.parse("Jan 31, 2025 12:17PM EST"),
)
expect(post.scanned_at).to be_within(1.second).of(DateTime.current)
end
@@ -181,7 +181,7 @@ describe Domain::Fa::Job::ScanPostJob do
)
expect(post.creator).to eq(creator)
expect(post.posted_at).to be_within(1.second).of(
Time.zone.parse("Feb 1, 2025 07:15 AM"),
Time.parse("Feb 1, 2025 7:15 AM EST"),
)
expect(post.keywords).to match(array_including("wolfgang", "kiss"))
expect(post.scanned_at).to be_within(1.second).of(DateTime.current)

12
spec/lib/domain/fa/parser/legacy_page_spec.rb
View File

@@ -178,7 +178,9 @@ describe Domain::Fa::Parser::Page do
assert_equal "Miles-DF", sub.artist
assert_equal "//d.facdn.net/art/miles-df/1455889648/1455889648.miles-df_miles-df_stream.jpg",
sub.full_res_img
assert_equal Time.zone.parse("Feb 19, 2016 08:47 AM"), sub.posted_date
expect(Time.parse("Feb 19, 2016 8:47AM EST")).to be_within(1.second).of(
sub.posted_date,
)
assert_equal "All", sub.category
assert_equal "All", sub.theme
assert_equal "Unspecified / Any", sub.species
@@ -202,7 +204,9 @@ describe Domain::Fa::Parser::Page do
assert_equal 82, sub.num_views
assert_equal "Male", sub.gender
assert_equal "Dragon (Other)", sub.species
assert_equal Time.zone.parse("Feb 24, 2016 12:51 AM"), sub.posted_date
expect(Time.parse("Feb 24, 2016 12:51AM EST")).to be_within(1.second).of(
sub.posted_date,
)
assert_equal %w[kobold frog hunting male frog mighty hunter],
sub.keywords_array
end
@@ -220,7 +224,9 @@ describe Domain::Fa::Parser::Page do
assert sub.description_html =~ /Forgot to put this up/
assert_equal "//d.facdn.net/art/feretta/1479650817/1479650802.feretta_161004_space_vixen_hop_desktop.jpg",
sub.full_res_img
assert_equal Time.zone.parse("Nov 20, 2016 09:06 AM"), sub.posted_date
expect(Time.parse("Nov 20, 2016 9:06AM EST")).to be_within(1.second).of(
sub.posted_date,
)
assert_equal "Artwork (Digital)", sub.category
assert_equal "Miscellaneous", sub.theme
assert_equal "Canid - Vulpine", sub.species

16
spec/lib/domain/fa/parser/redux_page_spec.rb
View File

@@ -187,7 +187,8 @@ describe Domain::Fa::Parser::Page do
assert_equal :adult, sub.rating
assert_equal "//d.furaffinity.net/art/cleverderpy/1676764142/1676764142.cleverderpy_tarryn_and_rosie_shaymin_tar_cv_00001.png",
sub.full_res_img
assert_equal Time.zone.parse("Feb 18, 2023 18:49"), sub.posted_date
assert_equal Time.zone.parse("Feb 18, 2023 23:49").change(zone: "UTC"),
sub.posted_date
assert_equal "Artwork (Digital)", sub.category
assert_equal "Vore", sub.theme
assert_equal "Unspecified / Any", sub.species
@@ -215,7 +216,8 @@ describe Domain::Fa::Parser::Page do
sub.small_img
assert_equal "//d.furaffinity.net/art/blauhaher/1676764049/1676764049.blauhaher_хорнь.png",
sub.full_res_img
assert_equal Time.zone.parse("Feb 18, 2023 18:47"), sub.posted_date
assert_equal Time.zone.parse("Feb 18, 2023 23:47").change(zone: "UTC"),
sub.posted_date
assert_equal "Artwork (Digital)", sub.category
assert_equal "Doodle", sub.theme
assert_equal "Unspecified / Any", sub.species
@@ -242,7 +244,8 @@ describe Domain::Fa::Parser::Page do
assert_equal 6272, sub.num_views
assert_equal "Male", sub.gender
assert_equal "Dragon (Other)", sub.species
assert_equal Time.zone.parse("Feb 24, 2016 00:51"), sub.posted_date
assert_equal Time.zone.parse("Feb 24, 2016 5:51").change(zone: "UTC"),
sub.posted_date
assert_equal %w[kobold frog hunting male frog mighty hunter],
sub.keywords_array
end
@@ -287,7 +290,9 @@ describe Domain::Fa::Parser::Page do
],
sub.keywords_array
assert_equal "Other / Not Specified", sub.gender
assert_equal Time.zone.parse("Apr 9, 2013 00:10 AM"), sub.posted_date
expect(Time.parse("Apr 9, 2013 12:10AM EDT")).to be_within(1.second).of(
sub.posted_date,
)
end
it "submission_id_is_correct_20340491" do
@@ -481,7 +486,8 @@ describe Domain::Fa::Parser::Page do
sub.small_img
assert_equal "//d.furaffinity.net/art/felureii/1737390216/1737390216.felureii_0f3gn7ifbl8.jpg",
sub.full_res_img
assert_equal Time.zone.parse("Jan 20, 2025 11:23"), sub.posted_date
assert_equal Time.zone.parse("Jan 20, 2025 16:23").change(zone: "UTC"),
sub.posted_date
assert_equal "Artwork (Digital)", sub.category
assert_equal "All", sub.theme
assert_equal "Unspecified / Any", sub.species