Merge pull request #1985 from SamantazFox/improve-youtube-api-helper
Improve youtube api calls
This commit is contained in:
syeopite
GitHub
commit
d82734641b
4 changed files with 118 additions and 72 deletions
|
|
@ -229,22 +229,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
|
|||
page = 1
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
|
||||
response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
|
||||
|
||||
videos = [] of SearchVideo
|
||||
begin
|
||||
initial_data = JSON.parse(response_body)
|
||||
raise InfoException.new("Could not extract channel JSON") if !initial_data
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data")
|
||||
videos = extract_videos(initial_data.as_h, author, ucid)
|
||||
rescue ex
|
||||
if response_body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
|
||||
response_body.includes?("https://www.google.com/sorry/index")
|
||||
raise InfoException.new("Could not extract channel info. Instance is likely blocked.")
|
||||
end
|
||||
raise ex
|
||||
end
|
||||
initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
|
||||
videos = extract_videos(initial_data, author, ucid)
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
|
||||
rss.xpath_nodes("//feed/entry").each do |entry|
|
||||
|
|
@ -304,10 +290,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
|
|||
ids = [] of String
|
||||
|
||||
loop do
|
||||
response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
|
||||
initial_data = JSON.parse(response_body)
|
||||
raise InfoException.new("Could not extract channel JSON") if !initial_data
|
||||
videos = extract_videos(initial_data.as_h, author, ucid)
|
||||
initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
|
||||
videos = extract_videos(initial_data, author, ucid)
|
||||
|
||||
count = videos.size
|
||||
videos = videos.map { |video| ChannelVideo.new({
|
||||
|
|
@ -358,8 +342,7 @@ end
|
|||
def fetch_channel_playlists(ucid, author, continuation, sort_by)
|
||||
if continuation
|
||||
response_json = request_youtube_api_browse(continuation)
|
||||
result = JSON.parse(response_json)
|
||||
continuationItems = result["onResponseReceivedActions"]?
|
||||
continuationItems = response_json["onResponseReceivedActions"]?
|
||||
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
|
||||
|
||||
return [] of SearchItem, nil if !continuationItems
|
||||
|
|
@ -964,21 +947,16 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
|
|||
videos = [] of SearchVideo
|
||||
|
||||
2.times do |i|
|
||||
response_json = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
|
||||
initial_data = JSON.parse(response_json)
|
||||
break if !initial_data
|
||||
videos.concat extract_videos(initial_data.as_h, author, ucid)
|
||||
initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
|
||||
videos.concat extract_videos(initial_data, author, ucid)
|
||||
end
|
||||
|
||||
return videos.size, videos
|
||||
end
|
||||
|
||||
def get_latest_videos(ucid)
|
||||
response_json = get_channel_videos_response(ucid)
|
||||
initial_data = JSON.parse(response_json)
|
||||
return [] of SearchVideo if !initial_data
|
||||
initial_data = get_channel_videos_response(ucid)
|
||||
author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
|
||||
items = extract_videos(initial_data.as_h, author, ucid)
|
||||
|
||||
return items
|
||||
return extract_videos(initial_data, author, ucid)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,28 +4,116 @@
|
|||
|
||||
# Hard-coded constants required by the API
|
||||
HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
||||
HARDCODED_CLIENT_VERS = "2.20210318.08.00"
|
||||
HARDCODED_CLIENT_VERS = "2.20210330.08.00"
|
||||
|
||||
def request_youtube_api_browse(continuation)
|
||||
####################################################################
|
||||
# make_youtube_api_context(region)
|
||||
#
|
||||
# Return, as a Hash, the "context" data required to request the
|
||||
# youtube API endpoints.
|
||||
#
|
||||
def make_youtube_api_context(region : String | Nil) : Hash
|
||||
return {
|
||||
"client" => {
|
||||
"hl" => "en",
|
||||
"gl" => region || "US", # Can't be empty!
|
||||
"clientName" => "WEB",
|
||||
"clientVersion" => HARDCODED_CLIENT_VERS,
|
||||
},
|
||||
}
|
||||
end
|
||||
|
||||
####################################################################
|
||||
# request_youtube_api_browse(continuation)
|
||||
# request_youtube_api_browse(browse_id, params)
|
||||
#
|
||||
# Requests the youtubei/v1/browse endpoint with the required headers
|
||||
# and POST data in order to get a JSON reply in english US that can
|
||||
# be easily parsed.
|
||||
#
|
||||
# The requested data can either be:
|
||||
#
|
||||
# - A continuation token (ctoken). Depending on this token's
|
||||
# contents, the returned data can be comments, playlist videos,
|
||||
# search results, channel community tab, ...
|
||||
#
|
||||
# - A playlist ID (parameters MUST be an empty string)
|
||||
#
|
||||
def request_youtube_api_browse(continuation : String)
|
||||
# JSON Request data, required by the API
|
||||
data = {
|
||||
"context": {
|
||||
"client": {
|
||||
"hl": "en",
|
||||
"gl": "US",
|
||||
"clientName": "WEB",
|
||||
"clientVersion": HARDCODED_CLIENT_VERS,
|
||||
},
|
||||
},
|
||||
"continuation": continuation,
|
||||
"context" => make_youtube_api_context("US"),
|
||||
"continuation" => continuation,
|
||||
}
|
||||
|
||||
# Send the POST request and return result
|
||||
return _youtube_api_post_json("/youtubei/v1/browse", data)
|
||||
end
|
||||
|
||||
def request_youtube_api_browse(browse_id : String, params : String)
|
||||
# JSON Request data, required by the API
|
||||
data = {
|
||||
"browseId" => browse_id,
|
||||
"context" => make_youtube_api_context("US"),
|
||||
}
|
||||
|
||||
# Append the additionnal parameters if those were provided
|
||||
# (this is required for channel info, playlist and community, e.g)
|
||||
if params != ""
|
||||
data["params"] = params
|
||||
end
|
||||
|
||||
return _youtube_api_post_json("/youtubei/v1/browse", data)
|
||||
end
|
||||
|
||||
####################################################################
|
||||
# request_youtube_api_search(search_query, params, region)
|
||||
#
|
||||
# Requests the youtubei/v1/search endpoint with the required headers
|
||||
# and POST data in order to get a JSON reply. As the search results
|
||||
# vary depending on the region, a region code can be specified in
|
||||
# order to get non-US results.
|
||||
#
|
||||
# The requested data is a search string, with some additional
|
||||
# paramters, formatted as a base64 string.
|
||||
#
|
||||
def request_youtube_api_search(search_query : String, params : String, region = nil)
|
||||
# JSON Request data, required by the API
|
||||
data = {
|
||||
"query" => URI.encode_www_form(search_query),
|
||||
"context" => make_youtube_api_context(region),
|
||||
"params" => params,
|
||||
}
|
||||
|
||||
return _youtube_api_post_json("/youtubei/v1/search", data)
|
||||
end
|
||||
|
||||
####################################################################
|
||||
# _youtube_api_post_json(endpoint, data)
|
||||
#
|
||||
# Internal function that does the actual request to youtube servers
|
||||
# and handles errors.
|
||||
#
|
||||
# The requested data is an endpoint (URL without the domain part)
|
||||
# and the data as a Hash object.
|
||||
#
|
||||
def _youtube_api_post_json(endpoint, data)
|
||||
# Send the POST request and parse result
|
||||
response = YT_POOL.client &.post(
|
||||
"/youtubei/v1/browse?key=#{HARDCODED_API_KEY}",
|
||||
headers: HTTP::Headers{"content-type" => "application/json"},
|
||||
"#{endpoint}?key=#{HARDCODED_API_KEY}",
|
||||
headers: HTTP::Headers{"content-type" => "application/json; charset=UTF-8"},
|
||||
body: data.to_json
|
||||
)
|
||||
|
||||
return response.body
|
||||
initial_data = JSON.parse(response.body).as_h
|
||||
|
||||
# Error handling
|
||||
if initial_data.has_key?("error")
|
||||
code = initial_data["error"]["code"]
|
||||
message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "")
|
||||
|
||||
raise InfoException.new("Could not extract JSON. Youtube API returned \
|
||||
error #{code} with message:<br>\"#{message}\"")
|
||||
end
|
||||
|
||||
return initial_data
|
||||
end
|
||||
|
|
|
|||
|
|
@ -361,16 +361,7 @@ def fetch_playlist(plid, locale)
|
|||
plid = "UU#{plid.lchop("UC")}"
|
||||
end
|
||||
|
||||
response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en")
|
||||
if response.status_code != 200
|
||||
if response.headers["location"]?.try &.includes? "/sorry/index"
|
||||
raise InfoException.new("Could not extract playlist info. Instance is likely blocked.")
|
||||
else
|
||||
raise InfoException.new("Not a playlist.")
|
||||
end
|
||||
end
|
||||
|
||||
initial_data = extract_initial_data(response.body)
|
||||
initial_data = request_youtube_api_browse("VL" + plid, params: "")
|
||||
|
||||
playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?
|
||||
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
|
||||
|
|
@ -451,17 +442,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
|
|||
offset = (offset / 100).to_i64 * 100_i64
|
||||
|
||||
ctoken = produce_playlist_continuation(playlist.id, offset)
|
||||
initial_data = JSON.parse(request_youtube_api_browse(ctoken)).as_h
|
||||
initial_data = request_youtube_api_browse(ctoken)
|
||||
else
|
||||
response = YT_POOL.client &.get("/playlist?list=#{playlist.id}&gl=US&hl=en")
|
||||
initial_data = extract_initial_data(response.body)
|
||||
initial_data = request_youtube_api_browse("VL" + playlist.id, params: "")
|
||||
end
|
||||
|
||||
if initial_data
|
||||
return extract_playlist_videos(initial_data)
|
||||
else
|
||||
return [] of PlaylistVideo
|
||||
end
|
||||
return extract_playlist_videos(initial_data)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -246,8 +246,7 @@ def channel_search(query, page, channel)
|
|||
continuation = produce_channel_search_continuation(ucid, query, page)
|
||||
response_json = request_youtube_api_browse(continuation)
|
||||
|
||||
result = JSON.parse(response_json)
|
||||
continuationItems = result["onResponseReceivedActions"]?
|
||||
continuationItems = response_json["onResponseReceivedActions"]?
|
||||
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
|
||||
|
||||
return 0, [] of SearchItem if !continuationItems
|
||||
|
|
@ -264,14 +263,9 @@ end
|
|||
def search(query, search_params = produce_search_params(content_type: "all"), region = nil)
|
||||
return 0, [] of SearchItem if query.empty?
|
||||
|
||||
body = YT_POOL.client(region, &.get("/results?search_query=#{URI.encode_www_form(query)}&sp=#{search_params}&hl=en").body)
|
||||
return 0, [] of SearchItem if body.empty?
|
||||
|
||||
initial_data = extract_initial_data(body)
|
||||
initial_data = request_youtube_api_search(query, search_params, region)
|
||||
items = extract_items(initial_data)
|
||||
|
||||
# initial_data["estimatedResults"]?.try &.as_s.to_i64
|
||||
|
||||
return items.size, items
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue