\n)
+
+ if tab == selected_tab
+ str << "\t"
+ str << translate(locale, "channel_tab_#{tab_name}_label")
+ str << "\n"
+ else
+ # Video tab doesn't have the last path component
+ url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
+
+ str << %(\t)
+ str << translate(locale, "channel_tab_#{tab_name}_label")
+ str << "\n"
+ end
+
+ str << "
"
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr
index afe31a36..bc329205 100644
--- a/src/invidious/hashtag.cr
+++ b/src/invidious/hashtag.cr
@@ -8,7 +8,8 @@ module Invidious::Hashtag
client_config = YoutubeAPI::ClientConfig.new(region: region)
response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
- return extract_items(response)
+ items, _ = extract_items(response)
+ return items
end
def generate_continuation(hashtag : String, cursor : Int)
diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr
index c52e2a0d..635f0984 100644
--- a/src/invidious/helpers/serialized_yt_data.cr
+++ b/src/invidious/helpers/serialized_yt_data.cr
@@ -265,4 +265,11 @@ class Category
end
end
+struct Continuation
+ getter token
+
+ def initialize(@token : String)
+ end
+end
+
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
index 2f525e08..b445107b 100644
--- a/src/invidious/jobs/notification_job.cr
+++ b/src/invidious/jobs/notification_job.cr
@@ -1,12 +1,12 @@
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
- private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+ private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
def initialize(@connection_channel, @pg_url)
end
def begin
- connections = [] of Channel(PQ::Notification)
+ connections = [] of ::Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
index 92681408..80812a63 100644
--- a/src/invidious/jobs/refresh_channels_job.cr
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
backoff = 2.minutes
loop do
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
index 4b52c959..4f8130df 100644
--- a/src/invidious/jobs/refresh_feeds_job.cr
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
index a431a48a..8584fb9c 100644
--- a/src/invidious/jobs/subscribe_to_feeds_job.cr
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index 6b81c546..ca2b2734 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,13 +1,7 @@
module Invidious::Routes::API::V1::Channels
- def self.home(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
-
+ # Macro to avoid duplicating some code below
+ # This sets the `channel` variable, or handles Exceptions.
+ private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
@@ -18,17 +12,25 @@ module Invidious::Routes::API::V1::Channels
rescue ex
return error_json(500, ex)
end
+ end
- page = 1
- if channel.auto_generated
- videos = [] of SearchVideo
- count = 0
- else
- begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
- rescue ex
- return error_json(500, ex)
- end
+ def self.home(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve "sort by" setting from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+
+ begin
+ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+ rescue ex
+ return error_json(500, ex)
end
JSON.build do |json|
@@ -100,31 +102,13 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
- related_channels = fetch_related_channels(channel)
+ related_channels, _ = fetch_related_channels(channel)
rescue ex
- related_channels = [] of AboutRelatedChannel
+ related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
- json.object do
- json.field "author", related_channel.author
- json.field "authorId", related_channel.ucid
- json.field "authorUrl", related_channel.author_url
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
+ related_channel.to_json(locale, json)
end
end
end # relatedChannels
@@ -134,61 +118,112 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
- locale = env.get("preferences").as(Preferences).locale
+ # Remove parameters that could affect this endpoint's behavior
+ env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
+ env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
-
- begin
- videos = get_latest_videos(ucid)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
- end
- end
- end
+ return self.videos(env)
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
- ucid = env.params.url["ucid"]
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
- sort_by = env.params.query["sort"]?.try &.downcase
- sort_by ||= env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve some URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex : NotFoundException
- return error_json(404, ex)
+ videos, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
rescue ex
return error_json(500, ex)
end
- begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.shorts(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ continuation = env.params.query["continuation"]?
+
+ begin
+ videos, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.streams(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ continuation = env.params.query["continuation"]?
+
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -204,16 +239,9 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
@@ -255,6 +283,37 @@ module Invidious::Routes::API::V1::Channels
end
end
+ def self.channels(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ continuation = env.params.query["continuation"]?
+
+ begin
+ items, next_continuation = fetch_related_channels(channel, continuation)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.object do
+ json.field "relatedChannels" do
+ json.array do
+ items.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index c6e02cbd..d3969d29 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -7,21 +7,19 @@ module Invidious::Routes::Channels
def self.videos(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
- locale, user, subscriptions, continuation, ucid, channel = data
+ return data if !data.is_a?(Tuple)
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
+ locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_options = {"last", "oldest", "newest"}
- sort_by ||= "last"
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, (sort_by || "last")
+ )
+
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -33,34 +31,85 @@ module Invidious::Routes::Channels
items.each(&.author = "")
else
sort_options = {"newest", "oldest", "popular"}
- sort_by ||= "newest"
- count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_videos(
+ channel, continuation: continuation, sort_by: (sort_by || "newest")
+ )
end
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
+ templated "channel"
+ end
+
+ def self.shorts(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "shorts"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ # TODO: support sort option for shorts
+ sort_by = ""
+ sort_options = [] of String
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation
+ )
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
+ templated "channel"
+ end
+
+ def self.streams(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "streams"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ # TODO: support sort option for livestreams
+ sort_by = ""
+ sort_options = [] of String
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation
+ )
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
+ return data if !data.is_a?(Tuple)
+
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "last"
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, (sort_by || "last")
+ )
+
items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
items.each(&.author = "")
- templated "playlists"
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
+ templated "channel"
end
def self.community(env)
@@ -74,12 +123,15 @@ module Invidious::Routes::Channels
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
- # sort_by = env.params.query["sort_by"]?.try &.downcase
if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
+ # TODO: support sort options for community posts
+ sort_by = ""
+ sort_options = [] of String
+
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
@@ -95,6 +147,26 @@ module Invidious::Routes::Channels
templated "community"
end
+ def self.channels(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if channel.auto_generated
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ items, next_continuation = fetch_related_channels(channel, continuation)
+
+ # Featured/related channels can't be sorted
+ sort_options = [] of String
+ sort_by = nil
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
+ templated "channel"
+ end
+
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@@ -125,7 +197,7 @@ module Invidious::Routes::Channels
end
selected_tab = env.request.path.split("/")[-1]
- if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
+ if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 1995677c..491022a5 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -117,14 +117,17 @@ module Invidious::Routing
get "/channel/:ucid", Routes::Channels, :home
get "/channel/:ucid/home", Routes::Channels, :home
get "/channel/:ucid/videos", Routes::Channels, :videos
+ get "/channel/:ucid/shorts", Routes::Channels, :shorts
+ get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
+ get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
get "/channel/:ucid/live", Routes::Channels, :live
get "/user/:user/live", Routes::Channels, :live
get "/c/:user/live", Routes::Channels, :live
- ["", "/videos", "/playlists", "/community", "/about"].each do |path|
+ {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path|
# /c/LinusTechTips
get "/c/:user#{path}", Routes::Channels, :brand_redirect
# /user/linustechtips | Not always the same as /c/
@@ -222,6 +225,10 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+ get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
+ get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+ get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
+
{% for route in {"videos", "latest", "playlists", "community", "search"} %}
get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index d1409c06..7e909590 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -9,7 +9,8 @@ module Invidious::Search
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
- return extract_items(initial_data)
+ items, _ = extract_items(initial_data)
+ return items
end
# Search a youtube channel
@@ -30,16 +31,7 @@ module Invidious::Search
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
- continuation_items = response_json["onResponseReceivedActions"]?
- .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
- return [] of SearchItem if !continuation_items
-
- items = [] of SearchItem
- continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
- extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
- end
-
+ items, _ = extract_items(response_json, "", ucid)
return items
end
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index dea86abe..a29315ef 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,8 +1,24 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url =
+ case selected_tab
+ when .shorts? then "/channel/#{ucid}/shorts"
+ when .streams? then "/channel/#{ucid}/streams"
+ when .playlists? then "/channel/#{ucid}/playlists"
+ when .channels? then "/channel/#{ucid}/channels"
+ else
+ "/channel/#{ucid}"
+ end
+
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+-%>
<% content_for "header" do %>
+<%- if selected_tab.videos? -%>
@@ -14,91 +30,14 @@
-
-
- 
">
-
-
- ">
-
-
-
-<% end %>
-
-
-
--
-
-
-
-
-
-<%= channel.description_html %>
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
+<%= rendered "components/channel_info" %>
- <%= translate(locale, "View channel on YouTube") %>
-
-
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <%= translate(locale, "Videos") %>
-
- <% end %>
-
- <% if channel.auto_generated %>
- <%= translate(locale, "Playlists") %>
- <% else %>
- <%= translate(locale, "Playlists") %>
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
- <% sort_options.each do |sort| %>
-
-
- <% if sort_by == sort %>
- <%= translate(locale, sort) %>
- <% else %>
-
- <%= translate(locale, sort) %>
-
- <% end %>
-
- <% end %>
- @@ -111,17 +50,10 @@
-
- <% if page > 1 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
+
- <% if count == 60 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <% if next_continuation %>
+
<%= translate(locale, "Next page") %>
<% end %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 3bc29e55..9e11d562 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,71 +1,21 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url = "/channel/#{ucid}/community"
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
+-%>
<% content_for "header" do %>
+
<%= author %> - Invidious
<% end %>
-<% if channel.banner %>
-
diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr new file mode 100644 index 00000000..f216359f --- /dev/null +++ b/src/invidious/views/components/channel_info.ecr @@ -0,0 +1,60 @@ +<% if channel.banner %> +<%= author %> - Invidious
-<% end %>
-
-<% if channel.banner %>
- \d+ minutes? )?(?\d+ seconds?)+ -/.match(a11y_data)
- minutes = regex_match.try &.["min"].to_i(strict: false) || 0
- seconds = regex_match.try &.["sec"].to_i(strict: false) || 0
+ minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0
+ seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0
duration = (minutes*60 + seconds)
@@ -475,6 +485,35 @@ private module Parsers
return {{@type.name}}
end
end
+
+ # Parses an InnerTube continuationItemRenderer into a Continuation.
+ # Returns nil when the given object isn't a continuationItemRenderer.
+ #
+ # continuationItemRenderer contains various metadata ued to load more
+ # content (i.e when the user scrolls down). The interesting bit is the
+ # protobuf object known as the "continutation token". Previously, those
+ # were generated from sratch, but recent (as of 11/2022) Youtube changes
+ # are forcing us to extract them from replies.
+ #
+ module ContinuationItemRendererParser
+ def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+ if item_contents = item["continuationItemRenderer"]?
+ return self.parse(item_contents)
+ end
+ end
+
+ private def self.parse(item_contents)
+ token = item_contents
+ .dig?("continuationEndpoint", "continuationCommand", "token")
+ .try &.as_s
+
+ return Continuation.new(token) if token
+ end
+
+ def self.parser_name
+ return {{@type.name}}
+ end
+ end
end
# The following are the extractors for extracting an array of items from
@@ -510,7 +549,7 @@ private module Extractors
# }]
#
module YouTubeTabs
- def self.process(initial_data : Hash(String, JSON::Any))
+ def self.process(initial_data : InitialData)
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end
@@ -575,7 +614,7 @@ private module Extractors
# }
#
module SearchResults
- def self.process(initial_data : Hash(String, JSON::Any))
+ def self.process(initial_data : InitialData)
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end
@@ -608,8 +647,8 @@ private module Extractors
# The way they are structured is too varied to be accurately written down here.
# However, they all eventually lead to an array of parsable items after traversing
# through the JSON structure.
- module Continuation
- def self.process(initial_data : Hash(String, JSON::Any))
+ module ContinuationContent
+ def self.process(initial_data : InitialData)
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
@@ -691,8 +730,7 @@ end
# Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
-def extract_item(item : JSON::Any, author_fallback : String? = "",
- author_id_fallback : String? = "")
+def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
# We "allow" nil values but secretly use empty strings instead. This is to save us the
# hassle of modifying every author_fallback and author_id_fallback arg usage
# which is more often than not nil.
@@ -702,24 +740,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
# Each parser automatically validates the data given to see if the data is
# applicable to itself. If not nil is returned and the next parser is attempted.
ITEM_PARSERS.each do |parser|
- LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
+ LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
if result = parser.process(item, author_fallback)
- LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
-
+ LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
return result
else
- LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
+ LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
end
end
end
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem.
-def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
- author_id_fallback : String? = nil) : Array(SearchItem)
- items = [] of SearchItem
-
+#
+# This function yields the container so that items can be parsed separately.
+#
+def extract_items(initial_data : InitialData, &block)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@@ -727,24 +764,37 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
unpackaged_data = initial_data
end
- # This is identical to the parser cycling of extract_item().
+ # This is identical to the parser cycling of parse_item().
ITEM_CONTAINER_EXTRACTOR.each do |extractor|
LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
if container = extractor.process(unpackaged_data)
LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
# Extract items in container
- container.each do |item|
- if parsed_result = extract_item(item, author_fallback, author_id_fallback)
- items << parsed_result
- end
- end
-
- break
+ container.each { |item| yield item }
else
LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
end
end
-
- return items
+end
+
+# Wrapper using the block function above
+def extract_items(
+ initial_data : InitialData,
+ author_fallback : String? = nil,
+ author_id_fallback : String? = nil
+) : {Array(SearchItem), String?}
+ items = [] of SearchItem
+ continuation = nil
+
+ extract_items(initial_data) do |item|
+ parsed = parse_item(item, author_fallback, author_id_fallback)
+
+ case parsed
+ when .is_a?(Continuation) then continuation = parsed.token
+ when .is_a?(SearchItem) then items << parsed
+ end
+ end
+
+ return items, continuation
end
diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr
index f8245160..0cb3c079 100644
--- a/src/invidious/yt_backend/extractors_utils.cr
+++ b/src/invidious/yt_backend/extractors_utils.cr
@@ -68,10 +68,10 @@ rescue ex
return false
end
-def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
- extracted = extract_items(initial_data, author_fallback, author_id_fallback)
+def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo)
+ extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback)
- target = [] of SearchItem
+ target = [] of (SearchItem | Continuation)
extracted.each do |i|
if i.is_a?(Category)
i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str
target << i
end
end
- return target.select(SearchVideo).map(&.as(SearchVideo))
+
+ return target.select(SearchVideo)
end
def extract_selected_tab(tabs)
# Extract the selected tab from the array of tabs Youtube returns
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"]
end
-
-def fetch_continuation_token(items : Array(JSON::Any))
- # Fetches the continuation token from an array of items
- return items.last["continuationItemRenderer"]?
- .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
-end
-
-def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
- # Fetches the continuation token from initial data
- if initial_data["onResponseReceivedActions"]?
- continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
- else
- tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
- continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
- end
-
- return fetch_continuation_token(continuation_items.as_a)
-end
- ">
-
-
- ">
-
-
-
-<% end %>
-
-
-
--
.request_target %>)
-
-
-
-
-
-<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
+<%= rendered "components/channel_info" %>
- <%= translate(locale, "View channel on YouTube") %>
-
-
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
- diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr new file mode 100644 index 00000000..f216359f --- /dev/null +++ b/src/invidious/views/components/channel_info.ecr @@ -0,0 +1,60 @@ +<% if channel.banner %> +
+ ">
+
+
+ ">
+
+
+
+<% end %>
+
+
+
++
+
+
+
+
+<%= channel.description_html %>
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr
deleted file mode 100644
index c8718e7b..00000000
--- a/src/invidious/views/playlists.ecr
+++ /dev/null
@@ -1,108 +0,0 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-
-<% content_for "header" do %>
-
+
+
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+
+
+
+
+ <% sort_options.each do |sort| %>
+
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+ <%= translate(locale, sort) %>
+ <% end %>
+
+ <% end %>
+
- ">
-
-
- ">
-
-
-
-<% end %>
-
-
-
--
-
-
-
-
-<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %>
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
-
-
-
-
-
-
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
-
-
-
- <% if !channel.auto_generated %>
- <%= translate(locale, "Playlists") %>
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
- <% {"last", "oldest", "newest"}.each do |sort| %>
-
-
- <% if sort_by == sort %>
- <%= translate(locale, sort) %>
- <% else %>
-
- <%= translate(locale, sort) %>
-
- <% end %>
-
- <% end %>
-
-
-
-
--
-<% items.each do |item| %>
- <%= rendered "components/item" %>
-<% end %>
-
-
-
-
-
diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr
index edc722cf..65d107b2 100644
--- a/src/invidious/yt_backend/extractors.cr
+++ b/src/invidious/yt_backend/extractors.cr
@@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data"
private ITEM_CONTAINER_EXTRACTOR = {
Extractors::YouTubeTabs,
Extractors::SearchResults,
- Extractors::Continuation,
+ Extractors::ContinuationContent,
}
private ITEM_PARSERS = {
@@ -18,8 +18,11 @@ private ITEM_PARSERS = {
Parsers::CategoryRendererParser,
Parsers::RichItemRendererParser,
Parsers::ReelItemRendererParser,
+ Parsers::ContinuationItemRendererParser,
}
+private alias InitialData = Hash(String, JSON::Any)
+
record AuthorFallback, name : String, id : String
# Namespace for logic relating to parsing InnerTube data into various datastructs.
@@ -345,14 +348,9 @@ private module Parsers
content_container = item_contents["contents"]
end
- raw_contents = content_container["items"]?.try &.as_a
- if !raw_contents.nil?
- raw_contents.each do |item|
- result = extract_item(item)
- if !result.nil?
- contents << result
- end
- end
+ content_container["items"]?.try &.as_a.each do |item|
+ result = parse_item(item, author_fallback.name, author_fallback.id)
+ contents << result if result.is_a?(SearchItem)
end
Category.new({
@@ -384,7 +382,9 @@ private module Parsers
end
private def self.parse(item_contents, author_fallback)
- return VideoRendererParser.process(item_contents, author_fallback)
+ child = VideoRendererParser.process(item_contents, author_fallback)
+ child ||= ReelItemRendererParser.process(item_contents, author_fallback)
+ return child
end
def self.parser_name
@@ -408,9 +408,19 @@ private module Parsers
private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s
- video_details_container = item_contents.dig(
+ reel_player_overlay = item_contents.dig(
"navigationEndpoint", "reelWatchEndpoint",
- "overlay", "reelPlayerOverlayRenderer",
+ "overlay", "reelPlayerOverlayRenderer"
+ )
+
+ # Sometimes, the "reelPlayerOverlayRenderer" object is missing the
+ # important part of the response. We use this exception to tell
+ # the calling function to fetch the content again.
+ if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
+ raise RetryOnceException.new
+ end
+
+ video_details_container = reel_player_overlay.dig(
"reelPlayerHeaderSupportedRenderers",
"reelPlayerHeaderRenderer"
)
@@ -436,9 +446,9 @@ private module Parsers
# View count
- view_count_text = video_details_container.dig?("viewCountText", "simpleText")
- view_count_text ||= video_details_container
- .dig?("viewCountText", "accessibility", "accessibilityData", "label")
+ # View count used to be in the reelWatchEndpoint, but that changed?
+ view_count_text = item_contents.dig?("viewCountText", "simpleText")
+ view_count_text ||= video_details_container.dig?("viewCountText", "simpleText")
view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
@@ -450,8 +460,8 @@ private module Parsers
regex_match = /- (?
- <% if continuation %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-