Use the new youtube api for comments (#2217)

* use the new youtube api for comments
* remove PG_DB & action parameter + allow force region
* support new comments data with onResponseReceivedEndpoints

src/invidious.cr

@@ -1873,9 +1873,6 @@ get "/api/v1/comments/:id" do |env|
   format = env.params.query["format"]?
   format ||= "json"
 
-  action = env.params.query["action"]?
-  action ||= "action_get_comments"
-
   continuation = env.params.query["continuation"]?
   sort_by = env.params.query["sort_by"]?.try &.downcase
 
@@ -1883,7 +1880,7 @@ get "/api/v1/comments/:id" do |env|
     sort_by ||= "top"
 
     begin
-      comments = fetch_youtube_comments(id, PG_DB, continuation, format, locale, thin_mode, region, sort_by: sort_by, action: action)
+      comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
     rescue ex
       next error_json(500, ex)
     end

src/invidious/comments.cr

@@ -56,10 +56,7 @@ class RedditListing
   property modhash : String
 end
 
-def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top", action = "action_get_comments")
+def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_by = "top")
-  video = get_video(id, db, region: region)
-  session_token = video.session_token
-
   case cursor
   when nil, ""
     ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by)
@@ -71,43 +68,41 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
     ctoken = cursor
   end
 
-  if !session_token
+  client_config = YoutubeAPI::ClientConfig.new(region: region)
-    if format == "json"
+  response = YoutubeAPI.next(continuation: ctoken, client_config: client_config)
-      return {"comments" => [] of String}.to_json
-    else
-      return {"contentHtml" => "", "commentCount" => 0}.to_json
-    end
-  end
 
-  post_req = {
+  if response["continuationContents"]?
-    page_token:    ctoken,
+    response = response["continuationContents"]
-    session_token: session_token,
-  }
-
-  headers = HTTP::Headers{
-    "cookie" => video.cookie,
-  }
-
-  response = YT_POOL.client(region, &.post("/comment_service_ajax?#{action}=1&hl=en&gl=US&pbj=1", headers, form: post_req))
-  response = JSON.parse(response.body)
-
-  # For some reason youtube puts it in an array for comment_replies but otherwise it's the same
-  if action == "action_get_comment_replies"
-    response = response[1]
-  end
-
-  if !response["response"]["continuationContents"]?
-    raise InfoException.new("Could not fetch comments")
-  end
-
-  response = response["response"]["continuationContents"]
     if response["commentRepliesContinuation"]?
       body = response["commentRepliesContinuation"]
     else
       body = response["itemSectionContinuation"]
     end
-
     contents = body["contents"]?
+    header = body["header"]?
+    if body["continuations"]?
+      moreRepliesContinuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s
+    end
+  elsif response["onResponseReceivedEndpoints"]?
+    onResponseReceivedEndpoints = response["onResponseReceivedEndpoints"]
+    onResponseReceivedEndpoints.as_a.each do |item|
+      case item["reloadContinuationItemsCommand"]["slot"]
+      when "RELOAD_CONTINUATION_SLOT_HEADER"
+        header = item["reloadContinuationItemsCommand"]["continuationItems"][0]
+      when "RELOAD_CONTINUATION_SLOT_BODY"
+        contents = item["reloadContinuationItemsCommand"]["continuationItems"]
+        contents.as_a.reject! do |item|
+          if item["continuationItemRenderer"]?
+            moreRepliesContinuation = item["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
+            true
+          end
+        end
+      end
+    end
+  else
+    raise InfoException.new("Could not fetch comments")
+  end
+
   if !contents
     if format == "json"
       return {"comments" => [] of String}.to_json
@@ -118,11 +113,10 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
 
   response = JSON.build do |json|
     json.object do
-      if body["header"]?
+      if header
-        count_text = body["header"]["commentsHeaderRenderer"]["countText"]
+        count_text = header["commentsHeaderRenderer"]["countText"]
         comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?)
           .try &.as_s.gsub(/\D/, "").to_i? || 0
-
         json.field "commentCount", comment_count
       end
 
@@ -211,7 +205,11 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
                   reply_count = 1
                 end
 
+                if node_replies["continuations"]?
                   continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
+                elsif node_replies["contents"]?
+                  continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
+                end
                 continuation ||= ""
 
                 json.field "replies" do
@@ -226,16 +224,15 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
         end
       end
 
-      if body["continuations"]?
+      if moreRepliesContinuation
-        continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s
+        json.field "continuation", moreRepliesContinuation
-        json.field "continuation", continuation
       end
     end
   end
 
   if format == "html"
     response = JSON.parse(response)
-    content_html = template_youtube_comments(response, locale, thin_mode, action == "action_get_comment_replies")
+    content_html = template_youtube_comments(response, locale, thin_mode)
 
     response = JSON.build do |json|
       json.object do

src/invidious/routes/watch.cr

@@ -92,7 +92,7 @@ module Invidious::Routes::Watch
 
         if source == "youtube"
           begin
-            comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+            comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
           rescue ex
             if preferences.comments[1] == "reddit"
               comments, reddit_thread = fetch_reddit_comments(id)
@@ -111,12 +111,12 @@ module Invidious::Routes::Watch
             comment_html = replace_links(comment_html)
           rescue ex
             if preferences.comments[1] == "youtube"
-              comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+              comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
             end
           end
         end
       else
-        comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+        comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
       end
 
       comment_html ||= ""