dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parser fixes for fa posts

Browse Source
This commit is contained in:
Dylan Knutson
2023-03-30 19:04:57 +09:00
parent 798b2e43cb
commit 67181ce78a
4 changed files with 1155 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/lib/domain/fa/parser/submission_parser_helper.rb
View File

@@ -120,7 +120,7 @@ class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
when VERSION_0, VERSION_1
elem_after_text_match(info_children, /Category/).text.strip
when VERSION_2
category_full_str_redux.split(" / ").first.strip
category_full_str_redux&.split(" / ")&.first&.strip
else unimplemented_version!
end
end
@@ -130,7 +130,7 @@ class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
when VERSION_0, VERSION_1
elem_after_text_match(info_children, /Theme/).text.strip
when VERSION_2
category_full_str_redux.split(" / ").last.strip
category_full_str_redux&.split(" / ")&.last&.strip
else unimplemented_version!
end
end
@@ -245,8 +245,8 @@ class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
def info_text_value_redux(info_section)
info_text_elem_redux.
css(".highlight").
find { |e| e.text == info_section }.
parent.children[1..].text.strip
find { |e| e.text == info_section }&.
parent&.children&.slice(1..)&.text&.strip
end
def stats_container_redux

2
app/lib/lite_trail/active_record_class_methods.rb
View File

@@ -92,7 +92,7 @@ module LiteTrail::ActiveRecordClassMethods
if attributes[attr_name]
attributes[attr_name] = mapper.map_to(attributes[attr_name])
end
end
end if map_attribute
self.versions << lite_trail_class.create!({
event: "destroy",

1133
test/fixtures/files/domain/fa/parser/redux/submission_16750893.html vendored Normal file
View File

File diff suppressed because it is too large Load Diff

17
test/lib/domain/fa/parser/redux_page_test.rb
View File

@@ -233,6 +233,23 @@ class Domain::Fa::Parser::ReduxPageTest < ActiveSupport::TestCase
assert_equal "//d.furaffinity.net/download/art/tasuric/1481384060/1481384060.tasuric_tasflash_camnom.swf", sub.full_res_img
end
def test_submission_id_16750893_category
parser = get_parser("submission_16750893.html")
assert_page_type parser, :probably_submission?
sub = parser.submission
assert_equal 16750893, sub.id
assert_equal "Day at the range with kagura", sub.title
assert_equal :mature, sub.rating
assert_equal "Soulhunt3r", sub.artist
# this post in particular is special - it has no category nor theme (??)
assert_nil sub.category
assert_nil sub.theme
assert_equal "Western Dragon", sub.species
assert (/Today was a good day kagura/) =~ sub.description_html
assert_equal "//d.furaffinity.net/art/soulhunt3r/1433541729/1433541729.soulhunt3r_image.jpg", sub.full_res_img
end
def get_parser(file, require_logged_in: true)
path = File.join("domain/fa/parser/redux", file)
contents = read_fixture_file(path) || raise("Couldn't open #{path}")