videos: improve fetching of streaming data

This commit is contained in:
Samantaz Fox 2022-11-06 21:23:58 +01:00
parent 2acff70811
commit cc5c83333f
No known key found for this signature in database
GPG key ID: F42821059186176E
2 changed files with 48 additions and 36 deletions

View file

@ -380,12 +380,6 @@ def fetch_video(id, region)
end end
end end
# Try to fetch video info using an embedded client
if info["reason"]?
embed_info = extract_video_info(video_id: id, context_screen: "embed")
info = embed_info if !embed_info["reason"]?
end
if reason = info["reason"]? if reason = info["reason"]?
if reason == "Video unavailable" if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "") raise NotFoundException.new(reason.as_s || "")

View file

@ -50,12 +50,9 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
} }
end end
def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) def extract_video_info(video_id : String, proxy_region : String? = nil)
# Init client config for the API # Init client config for the API
client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
if context_screen == "embed"
client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
end
# Fetch data from the player endpoint # Fetch data from the player endpoint
player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
@ -69,7 +66,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
reason ||= player_response.dig("playabilityStatus", "reason").as_s reason ||= player_response.dig("playabilityStatus", "reason").as_s
# Stop here if video is not a scheduled livestream # Stop here if video is not a scheduled livestream
if playability_status != "LIVE_STREAM_OFFLINE" if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
return { return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new(reason), "reason" => JSON::Any.new(reason),
@ -84,7 +81,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
end end
# Don't fetch the next endpoint if the video is unavailable. # Don't fetch the next endpoint if the video is unavailable.
if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status) if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
player_response = player_response.merge(next_response) player_response = player_response.merge(next_response)
end end
@ -92,33 +89,34 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
params = parse_video_info(video_id, player_response) params = parse_video_info(video_id, player_response)
params["reason"] = JSON::Any.new(reason) if reason params["reason"] = JSON::Any.new(reason) if reason
# Fetch the video streams using an Android client in order to get the decrypted URLs and new_player_response = nil
# maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
# https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
if reason.nil?
if context_screen == "embed"
client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
else
client_config.client_type = YoutubeAPI::ClientType::Android
end
android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
# Sometimes, the video is available from the web client, but not on Android, so check if reason.nil?
# that here, and fallback to the streaming data from the web client if needed. # Fetch the video streams using an Android client in order to get the
# See: https://github.com/iv-org/invidious/issues/2549 # decrypted URLs and maybe fix throttling issues (#2194). See the
if video_id != android_player.dig("videoDetails", "videoId") # following issue for an explanation about decrypted URLs:
# YouTube may return a different video player response than expected. # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713 client_config.client_type = YoutubeAPI::ClientType::Android
raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)") new_player_response = try_fetch_streaming_data(video_id, client_config)
elsif android_player["playabilityStatus"]["status"] == "OK" elsif !reason.includes?("your country") # Handled separately
params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") # The Android embedded client could help here
else client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") new_player_response = try_fetch_streaming_data(video_id, client_config)
end
end end
# TODO: clean that up # Last hope
{"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| if new_player_response.nil?
client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Replace player response and reset reason
if !new_player_response.nil?
player_response = new_player_response
params.delete("reason")
end
{"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
params[f] = player_response[f] if player_response[f]? params[f] = player_response[f] if player_response[f]?
end end
@ -128,6 +126,26 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
return params return params
end end
def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
response = YoutubeAPI.player(video_id: id, params: "", client_config: client_config)
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
if id != response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise VideoNotAvailableException.new(
"The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
)
elsif playability_status == "OK"
return response
else
return nil
end
end
def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
# Top level elements # Top level elements