diff --git a/config/config.example.yml b/config/config.example.yml index 6e48db7f..6ae213d0 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -54,6 +54,23 @@ db: ## #signature_server: +## +## Path to the Invidious companion. +## An external program for loading the video streams from YouTube servers. +## +## When this setting is commented out, Invidious companion is not used. +## +## When this setting is configured and "external_port" is used then +## you need to configure Invidious companion routes into your reverse proxy. +## If "external_port" is not configured then Invidious will proxy the requests +## to Invidious companion. +## +## Accepted values: "http(s)://:" +## Default: +## +# invidious_companion: +# - http://127.0.0.1:8282 + ######################################### # diff --git a/src/invidious/config.cr b/src/invidious/config.cr index d9a42893..2d836420 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -138,6 +138,9 @@ class Config # poToken for passing bot attestation property po_token : String? = nil + # Invidious companion + property invidious_companion : Array(String)? = nil + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index d89e752c..43d0eb2f 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -20,6 +20,10 @@ module Invidious::Routes::API::Manifest haltf env, status_code: 403 end + if local && CONFIG.invidious_companion + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + end + if dashmpd = video.dash_manifest_url response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 24693662..498ede4b 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -294,6 +294,9 @@ module Invidious::Routes::VideoPlayback end if local + if (CONFIG.invidious_companion) + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + end url = URI.parse(url).request_target.not_nil! url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index aabe8dfc..2cf8a725 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -190,6 +190,13 @@ module Invidious::Routes::Watch captions: video.captions ) + if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash")) + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s) + end + templated "watch" end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ae09e736..131549d0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -192,6 +192,10 @@ struct Video } end + def invidious_companion : Hash(String, JSON::Any) + info["invidiousCompanion"].try &.as_h + end + # Macros defining getters/setters for various types of data private macro getset_string(name) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index fb8935d9..7a744bdf 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -104,42 +104,44 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - new_player_response = nil + if CONFIG.invidious_companion.nil? + new_player_response = nil - # Second try in case WEB_CREATOR doesn't work with po_token. - # Only trigger if reason found and po_token configured. - if reason && CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Second try in case WEB_CREATOR doesn't work with po_token. + # Only trigger if reason found and po_token configured. + if reason && CONFIG.po_token + client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Don't use Android client if po_token is passed because po_token doesn't - # work for Android client. - if reason.nil? && CONFIG.po_token.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Don't use Android client if po_token is passed because po_token doesn't + # work for Android client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Last hope - # Only trigger if reason found or didn't work wth Android client. - # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token. - if reason && CONFIG.po_token.nil? - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Last hope + # Only trigger if reason found or didn't work wth Android client. + # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token. + if reason && CONFIG.po_token.nil? + client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Replace player response and reset reason - if !new_player_response.nil? - # Preserve captions & storyboard data before replacement - new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? - new_player_response["captions"] = player_response["captions"] if player_response["captions"]? + # Replace player response and reset reason + if !new_player_response.nil? + # Preserve captions & storyboard data before replacement + new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? - player_response = new_player_response - params.delete("reason") + player_response = new_player_response + params.delete("reason") + end end {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| @@ -157,6 +159,10 @@ def extract_video_info(video_id : String) params["streamingData"] = streaming_data end + if CONFIG.invidious_companion + params["invidiousCompanion"] = player_response["invidiousCompanion"] + end + # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -456,11 +462,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Music section "music" => JSON.parse(music_list.to_json), # Author infos - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), + "invidiousCompanion" => JSON::Any.new(subs_text), } return params diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 5c28358b..017b3462 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,6 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -34,8 +35,11 @@ <% end %> <% end %> <% else %> - <% if params.quality == "dash" %> - + <% if params.quality == "dash" + src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion) + %> + <% end %> <% @@ -44,6 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index baa3cd92..571ca30b 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -616,12 +616,19 @@ module YoutubeAPI headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", "x-youtube-client-name" => client_config.name_proto, "x-youtube-client-version" => client_config.version, } + if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player" + headers["Authorization"] = "Bearer " + CONFIG.hmac_key + end + + if !CONFIG.invidious_companion + headers["Accept-Encoding"] = "gzip, deflate" + end + if user_agent = client_config.user_agent headers["User-Agent"] = user_agent end @@ -635,11 +642,32 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") + invidious_companion_urls = CONFIG.invidious_companion + # Send the POST request - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - self._decompress(response.body_io, response.headers["Content-Encoding"]?) + if invidious_companion_urls && endpoint == "/youtubei/v1/player" + puts "invidious companion section" + puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] + begin + invidious_companion_response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), + &.post(endpoint, headers: headers, body: data.to_json)) + body = invidious_companion_response.body + if (invidious_companion_response.status_code != 200) + raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body) + end + rescue ex + raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end + else + body = YT_POOL.client() do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + self._decompress(response.body_io, response.headers["Content-Encoding"]?) + end + end + end + + if body.nil? && CONFIG.invidious_companion + raise InfoException.new("Error while communicating with Invidious companion: no response data.") end # Convert result to Hash