Overhaul extractors.cr to use modules

This commit is contained in:
syeopite 2021-08-03 00:22:31 -07:00
parent 3dea670091
commit 142317c2be
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82

View File

@ -3,47 +3,34 @@
# Tuple of Parsers/Extractors so we can easily cycle through them.
private ITEM_CONTAINER_EXTRACTOR = {
YoutubeTabsExtractor.new,
SearchResultsExtractor.new,
ContinuationExtractor.new,
Extractors::YouTubeTabs,
Extractors::SearchResults,
Extractors::Continuation,
}
private ITEM_PARSERS = {
VideoParser.new,
ChannelParser.new,
GridPlaylistParser.new,
PlaylistParser.new,
CategoryParser.new,
Parsers::VideoRendererParser,
Parsers::ChannelRendererParser,
Parsers::GridPlaylistRendererParser,
Parsers::PlaylistRendererParser,
Parsers::CategoryRendererParser,
}
private struct AuthorFallback
property name, id
def initialize(@name : String? = nil, @id : String? = nil)
end
end
record AuthorFallback, name : String? = nil, id : String? = nil
# The following are the parsers for parsing raw item data into neatly packaged structs.
# They're accessed through the process() method which validates the given data as applicable
# to their specific struct and then use the internal parse() method to assemble the struct
# specific to their category.
private abstract struct ItemParser
# Base type for all item parsers.
def process(item : JSON::Any, author_fallback : AuthorFallback)
end
private def parse(item_contents : JSON::Any, author_fallback : AuthorFallback)
end
end
private struct VideoParser < ItemParser
def process(item, author_fallback)
private module Parsers
module VideoRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
return self.parse(item_contents, author_fallback)
end
end
private def parse(item_contents, author_fallback)
private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s
title = item_contents["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
@ -92,16 +79,16 @@ private struct VideoParser < ItemParser
premiere_timestamp: premiere_timestamp,
})
end
end
end
private struct ChannelParser < ItemParser
def process(item, author_fallback)
module ChannelRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?)
return self.parse(item_contents, author_fallback)
end
end
private def parse(item_contents, author_fallback)
private def self.parse(item_contents, author_fallback)
author = item_contents["title"]["simpleText"]?.try &.as_s || author_fallback.name || ""
author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id || ""
@ -123,16 +110,16 @@ private struct ChannelParser < ItemParser
auto_generated: auto_generated,
})
end
end
end
private struct GridPlaylistParser < ItemParser
def process(item, author_fallback)
module GridPlaylistRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["gridPlaylistRenderer"]?
return self.parse(item_contents, author_fallback)
end
end
private def parse(item_contents, author_fallback)
private def self.parse(item_contents, author_fallback)
title = item_contents["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
plid = item_contents["playlistId"]?.try &.as_s || ""
@ -149,16 +136,16 @@ private struct GridPlaylistParser < ItemParser
thumbnail: playlist_thumbnail,
})
end
end
end
private struct PlaylistParser < ItemParser
def process(item, author_fallback)
module PlaylistRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["playlistRenderer"]?
return self.parse(item_contents, author_fallback)
end
end
def parse(item_contents, author_fallback)
private def self.parse(item_contents, author_fallback)
title = item_contents["title"]["simpleText"]?.try &.as_s || ""
plid = item_contents["playlistId"]?.try &.as_s || ""
@ -193,16 +180,16 @@ private struct PlaylistParser < ItemParser
thumbnail: playlist_thumbnail,
})
end
end
end
private struct CategoryParser < ItemParser
def process(item, author_fallback)
module CategoryRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["shelfRenderer"]?
return self.parse(item_contents, author_fallback)
end
end
def parse(item_contents, author_fallback)
private def self.parse(item_contents, author_fallback)
# Title extraction is a bit complicated. There are two possible routes for it
# as well as times when the title attribute just isn't sent by YT.
title_container = item_contents["title"]? || ""
@ -255,6 +242,7 @@ private struct CategoryParser < ItemParser
badges: badges,
})
end
end
end
# The following are the extractors for extracting an array of items from
@ -262,22 +250,15 @@ end
# a structure we can more easily use via the parsers above. Their internals are
# identical to the item parsers.
private abstract struct ItemsContainerExtractor
def process(item : Hash(String, JSON::Any))
end
private def extract(target : JSON::Any)
end
end
private struct YoutubeTabsExtractor < ItemsContainerExtractor
def process(initial_data)
private module Extractors
module YouTubeTabs
def self.process(initial_data : Hash(String, JSON::Any))
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end
end
private def extract(target)
private def self.extract(target)
raw_items = [] of JSON::Any
selected_tab = extract_selected_tab(target["tabs"])
content = selected_tab["content"]
@ -302,16 +283,16 @@ private struct YoutubeTabsExtractor < ItemsContainerExtractor
return raw_items
end
end
end
private struct SearchResultsExtractor < ItemsContainerExtractor
def process(initial_data)
module SearchResults
def self.process(initial_data : Hash(String, JSON::Any))
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end
end
private def extract(target)
private def self.extract(target)
raw_items = [] of Array(JSON::Any)
content = target["primaryContents"]
renderer = content["sectionListRenderer"]["contents"].as_a.each do |node|
@ -324,10 +305,10 @@ private struct SearchResultsExtractor < ItemsContainerExtractor
return raw_items
end
end
end
private struct ContinuationExtractor < ItemsContainerExtractor
def process(initial_data)
module Continuation
def self.process(initial_data : Hash(String, JSON::Any))
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
@ -335,7 +316,7 @@ private struct ContinuationExtractor < ItemsContainerExtractor
end
end
private def extract(target)
private def self.extract(target)
raw_items = [] of JSON::Any
if content = target["gridContinuation"]?
raw_items = content["items"].as_a
@ -345,6 +326,7 @@ private struct ContinuationExtractor < ItemsContainerExtractor
return raw_items
end
end
end
# Parses an item from Youtube's JSON response into a more usable structure.