graphql twitter api & soundcloud fix

closes #127

src/modules/pageRender/page.js

@@ -502,7 +502,7 @@ export default function(obj) {
             ${urgentNotice({
                 emoji: "🐱",
                 text: "report any issues!",
-                visible: true,
+                visible: false,
                 action: "popup('about', 1, 'changelog')"
             })}
             <div id="cobalt-main-box" class="center">

src/modules/processing/match.js

@@ -8,7 +8,7 @@ import matchActionDecider from "./matchActionDecider.js";
 
 import bilibili from "./services/bilibili.js";
 import reddit from "./services/reddit.js";
-import twitter from "./services/twitter_lite.js";
+import twitter from "./services/twitter.js";
 import youtube from "./services/youtube.js";
 import vk from "./services/vk.js";
 import tiktok from "./services/tiktok.js";

src/modules/processing/services/soundcloud.js

@@ -36,26 +36,31 @@ async function findClientID() {
 export default async function(obj) {
     let html;
     if (!obj.author && !obj.song && obj.shortLink) {
-        html = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`).then((r) => { return r.status === 404 ? false : r.text() }).catch(() => { return false });
+        html = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`).then((r) => {
+            return r.status === 404 ? false : r.text()
+        }).catch(() => { return false });
     }
     if (obj.author && obj.song) {
-        html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`).then((r) => { return r.text() }).catch(() => { return false });
+        html = await fetch(
+            `https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`
+        ).then((r) => {
+            return r.text()
+        }).catch(() => { return false });
     }
     if (!html) return { error: 'ErrorCouldntFetch' };
-    if (!(html.includes('<script>window.__sc_hydration = ')
+    if (!(html.includes('<script>window.__sc_hydration = ') && html.includes('{"hydratable":"sound","data":'))) {
-    && html.includes('"format":{"protocol":"progressive","mime_type":"audio/mpeg"},')
-    && html.includes('{"hydratable":"sound","data":'))) {
         return { error: ['ErrorBrokenLink', 'soundcloud'] }
     }
 
-    let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0])
+    let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0]);
     if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' };
 
     let clientId = await findClientID();
     if (!clientId) return { error: 'ErrorSoundCloudNoClientId' };
 
-    let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive"),
+    let fileUrlBase = json.media.transcodings.filter((v) => { if (v["format"]["mime_type"].startsWith("audio/ogg")) return true })[0]["url"],
         fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
+
     if (fileUrl.substring(0, 54) !== "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
 
     if (json.duration > maxVideoDuration) return { error: ['ErrorLengthAudioConvert', maxVideoDuration / 60000] };

src/modules/processing/services/twitter.js

@@ -1,9 +1,8 @@
 import { genericUserAgent } from "../../config.js";
 
 function bestQuality(arr) {
-    return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
+    return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"]
 }
-const apiURL = "https://api.twitter.com"
 
 export default async function(obj) {
     let _headers = {
@@ -12,10 +11,12 @@ export default async function(obj) {
         "host": "api.twitter.com",
         "x-twitter-client-language": "en",
         "x-twitter-active-user": "yes",
-        "Accept-Language": "en"
+        "accept-language": "en"
     };
-    let conversationURL = `${apiURL}/2/timeline/conversation/${obj.id}.json?cards_platform=Web-12&tweet_mode=extended&include_cards=1&include_ext_media_availability=true&include_ext_sensitive_media_warning=true&simple_quoted_tweet=true&trim_user=1`;
+
-    let activateURL = `${apiURL}/1.1/guest/activate.json`;
+    let activateURL = `https://api.twitter.com/1.1/guest/activate.json`;
+    let graphqlTweetURL = `https://twitter.com/i/api/graphql/0hWvDhmW8YQ-S_ib3azIrw/TweetResultByRestId`;
+    let graphqlSpaceURL = `https://twitter.com/i/api/graphql/Gdz2uCtmIGMmhjhHG3V7nA/AudioSpaceById`;
 
     let req_act = await fetch(activateURL, {
         method: "POST",
@@ -23,23 +24,39 @@ export default async function(obj) {
     }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
     if (!req_act) return { error: 'ErrorCouldntFetch' };
 
+    _headers["host"] = "twitter.com";
+    _headers["content-type"] = "application/json";
+
     _headers["x-guest-token"] = req_act["guest_token"];
-    _headers["cookie"] = `guest_id=v1%3A${req_act["guest_token"]};`;
+    _headers["cookie"] = `guest_id=v1%3A${req_act["guest_token"]}`;
 
-    if (!obj.spaceId) {
+    if (obj.id) {
-        let conversation = await fetch(conversationURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
+        let query = {
-        if (!conversation || !conversation.globalObjects.tweets[obj.id]) return { error: 'ErrorTweetUnavailable' };
+            variables: {"tweetId": obj.id, "withCommunity": false, "includePromotedContent": false, "withVoice": false},
+            features: {"creator_subscriptions_tweet_preview_api_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_media_download_video_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}
+        }
+        query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1);
+        query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
+        query = `${graphqlTweetURL}?variables=${query.variables}&features=${query.features}`;
 
-        let baseMedia, baseTweet = conversation.globalObjects.tweets[obj.id];
+        let TweetResultByRestId = await fetch(query, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
-        if (baseTweet.retweeted_status_id_str && conversation.globalObjects.tweets[baseTweet.retweeted_status_id_str].extended_entities) {
+
-            baseMedia = conversation.globalObjects.tweets[baseTweet.retweeted_status_id_str].extended_entities
+        // {"data":{"tweetResult":{"result":{"__typename":"TweetUnavailable","reason":"Protected"}}}}
+        if (!TweetResultByRestId || TweetResultByRestId.data.tweetResult.result.__typename !== "Tweet") return { error: 'ErrorTweetUnavailable' };
+
+        let baseMedia,
+            baseTweet = TweetResultByRestId.data.tweetResult.result.legacy;
+
+        if (baseTweet.retweeted_status_result && baseTweet.retweeted_status_result.result.legacy.extended_entities.media) {
+            baseMedia = baseTweet.retweeted_status_result.result.legacy.extended_entities
         } else if (baseTweet.extended_entities && baseTweet.extended_entities.media) {
             baseMedia = baseTweet.extended_entities
         }
         if (!baseMedia) return { error: 'ErrorNoVideosInTweet' };
 
         let single, multiple = [], media = baseMedia["media"];
-        media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
+            media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true });
+
         if (media.length > 1) {
             for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
         } else if (media.length === 1) {
@@ -55,7 +72,9 @@ export default async function(obj) {
         } else {
             return { error: 'ErrorNoVideosInTweet' }
         }
-    } else {
+    }
+    // spaces no longer work with guest authorization
+    if (obj.spaceId) {
         _headers["host"] = "twitter.com";
         _headers["content-type"] = "application/json";
 
@@ -65,7 +84,7 @@ export default async function(obj) {
         }
         query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1);
         query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
-        query = `https://twitter.com/i/api/graphql/Gdz2uCtmIGMmhjhHG3V7nA/AudioSpaceById?variables=${query.variables}&features=${query.features}`;
+        query = `${graphqlSpaceURL}?variables=${query.variables}&features=${query.features}`;
 
         let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false });
         if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' };

src/modules/processing/services/twitter_lite.js

@@ -1,39 +0,0 @@
-function bestQuality(arr) {
-    return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"]
-}
-
-export default async function(obj) {
-    if (!obj.spaceId) {
-        let synd = await fetch(`https://cdn.syndication.twimg.com/tweet-result?id=${obj.id}`, {
-            headers: {
-                "User-Agent": "Googlebot/2.1 (+http://www.google.com/bot.html)"
-            }
-        }).then((r) => { return r.status === 200 ? r.text() : false }).catch((e) => { return false });
-        if (!synd) {
-            return { error: 'ErrorTweetUnavailable' }
-        } else {
-            synd = JSON.parse(synd);
-        }
-        if (!synd.mediaDetails) return { error: 'ErrorNoVideosInTweet' };
-
-        let single, multiple = [], media = synd.mediaDetails;
-        media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
-        if (media.length > 1) {
-            for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
-        } else if (media.length === 1) {
-            single = bestQuality(media[0]["video_info"]["variants"])
-        } else {
-            return { error: 'ErrorNoVideosInTweet' }
-        }
-
-        if (single) {
-            return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
-        } else if (multiple) {
-            return { picker: multiple }
-        } else {
-            return { error: 'ErrorNoVideosInTweet' }
-        }
-    } else {
-        return { error: 'ErrorTwitterRIP' }
-    }
-}

src/test/tests.json

@@ -97,15 +97,7 @@
         }
     }, {
         "name": "retweeted video",
-        "url": "https://twitter.com/winload_exe/status/1639005390854602758",
+        "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg",
-        "params": {},
-        "expected": {
-            "code": 200,
-            "status": "redirect"
-        }
-    }, {
-        "name": "retweeted video",
-        "url": "https://twitter.com/winload_exe/status/1639005390854602758",
         "params": {},
         "expected": {
             "code": 200,
@@ -121,7 +113,7 @@
         }
     }, {
         "name": "retweeted video, isAudioOnly",
-        "url": "https://twitter.com/winload_exe/status/1633091769482063874",
+        "url": "https://twitter.com/hugekiwinuts/status/1618671150829309953?s=46&t=gItGzgwGQQJJaJrO6qc1Pg",
         "params": {
             "aFormat": "mp3",
             "isAudioOnly": false,