services: add snapchat support (#429)

* feat: snapchat support

* chore: remove redundancy

* chore: a bit of better matching

* chore: update readme

* refactor(snapchat): refactor story matching to use pickers

* fix: small fix to directly linked stories

* fix(snapchat): fix filenames

* chore: update readme

* ref(snapchat): rewrite service, new test, split redirects into a util

* fix(snapchat): small fixes

* chore: deepscan error fixed

* fix: remove debug logging

* fix(snapchat): fix merge, clean up code with new utils

* fix(snapchat): update with suggested changes

---------

Signed-off-by: Snazzah <7025343+Snazzah@users.noreply.github.com>
Co-authored-by: jj <log@riseup.net>

README.md

@@ -25,6 +25,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | pinterest                      | ✅            | ✅         | ✅         | ➖         | ➖              |
 | reddit                         | ✅            | ✅         | ✅         | ❌         | ❌              |
 | rutube                         | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat stories & spotlights  | ✅            | ✅         | ✅         | ➖         | ➖              |
 | soundcloud                     | ➖            | ✅         | ➖         | ✅         | ✅              |
 | streamable                     | ✅            | ✅         | ✅         | ➖         | ➖              |
 | tiktok                         | ✅            | ✅         | ✅         | ❌         | ❌              |
@@ -49,6 +50,7 @@ this list is not final and keeps expanding over time. if support for a service y
 | facebook   | supports public accessible videos content only.                                                                      |
 | pinterest  | supports photos, gifs, videos and stories.                                                                           |
 | reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
 | rutube     | supports yappy & private links.                                                                                      |
 | soundcloud | supports private links.                                                                                              |
 | tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |

src/modules/processing/match.js

@@ -24,6 +24,7 @@ import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
 import rutube from "./services/rutube.js";
 import dailymotion from "./services/dailymotion.js";
+import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 
@@ -189,6 +190,14 @@ export default async function(host, patternMatch, lang, obj) {
             case "dailymotion":
                 r = await dailymotion(patternMatch);
                 break;
+            case "snapchat":
+                r = await snapchat({
+                    url,
+                    username: patternMatch.username,
+                    storyId: patternMatch.storyId,
+                    spotlightId: patternMatch.spotlightId,
+                    shortLink: patternMatch.shortLink || false
+                });
             case "loom":
                 r = await loom({
                     id: patternMatch.id

src/modules/processing/matchActionDecider.js

@@ -73,6 +73,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
             switch (host) {
                 case "instagram":
                 case "twitter":
+                case "snapchat":
                     params = { picker: r.picker };
                     break;
                 case "tiktok":
@@ -136,6 +137,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
                 case "tumblr":
                 case "pinterest":
                 case "streamable":
+                case "snapchat":
                 case "loom":
                     responseType = "redirect";
                     break;

src/modules/processing/services/snapchat.js

@@ -0,0 +1,96 @@
+import { genericUserAgent } from "../../config.js";
+import { getRedirectingURL } from "../../sub/utils.js";
+import { extract, normalizeURL } from "../url.js";
+
+const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&amp;uc=\d+)" as="video"\/>/;
+const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
+
+async function getSpotlight(id) {
+    const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {  
+        return { error: 'ErrorCouldntFetch' };  
+    }
+
+    const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
+    if (videoURL) {
+        return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    }
+}
+
+async function getStory(username, storyId) {
+    const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, {
+        headers: { 'User-Agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+    if (!html) {  
+        return { error: 'ErrorCouldntFetch' };  
+    }  
+
+    const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
+    if (nextDataString) {
+        const data = JSON.parse(nextDataString);
+        const storyIdParam = data.query.profileParams[1];
+
+        if (storyIdParam && data.props.pageProps.story) {
+            const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
+            if (story) {
+                if (story.snapMediaType === 0) {
+                    return {
+                        urls: story.snapUrls.mediaUrl,
+                        isPhoto: true
+                    }
+                }
+
+                return {
+                    urls: story.snapUrls.mediaUrl,
+                    filename: `snapchat_${storyId}.mp4`,
+                    audioFilename: `snapchat_${storyId}_audio`
+                }
+            }
+        }
+
+        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        if (defaultStory) {
+            return {
+                picker: defaultStory.snapList.map((snap) => ({
+                    type: snap.snapMediaType === 0 ? 'photo' : 'video',
+                    url: snap.snapUrls.mediaUrl,
+                    thumb: snap.snapUrls.mediaPreviewUrl.value
+                }))
+            }
+        }
+    }
+}
+
+export default async function(obj) {
+    let params = obj;
+    if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) {
+        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
+    
+        if (!link?.startsWith('https://www.snapchat.com/')) {  
+            return { error: 'ErrorCouldntFetch' };  
+        }  
+
+        const extractResult = extract(normalizeURL(link));
+        if (extractResult?.host !== 'snapchat') {  
+            return { error: 'ErrorCouldntFetch' };  
+        }
+
+        params = extractResult.patternMatch;
+    }
+
+    if (params.spotlightId) {
+        const result = await getSpotlight(params.spotlightId);
+        if (result) return result;
+    } else if (params.username) {
+        const result = await getStory(params.username, params.storyId);
+        if (result) return result;
+    }
+
+    return { error: 'ErrorCouldntFetch' };
+}

src/modules/processing/servicesConfig.json

@@ -114,6 +114,12 @@
             "patterns": ["video/:id"],
             "enabled": true
         },
+        "snapchat": {
+            "alias": "snapchat stories & spotlights",
+            "subdomains": ["t", "story"],
+            "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"],
+            "enabled": true
+        },
         "loom": {
             "alias": "loom videos",
             "patterns": ["share/:id"],

src/modules/processing/servicesPatternTesters.js

@@ -30,6 +30,11 @@ export const testers = {
         (patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255)
         || patternMatch.shortLink?.length <= 32,
 
+    "snapchat": (patternMatch) =>
+        (patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255)) 
+        || patternMatch.spotlightId?.length <= 255 
+        || patternMatch.shortLink?.length <= 16,
+
     "streamable": (patternMatch) =>
         patternMatch.id?.length === 6,
 

src/modules/sub/utils.js

@@ -45,6 +45,13 @@ export function cleanHTML(html) {
     return clean
 }
 
+export function getRedirectingURL(url) {
+    return fetch(url, { redirect: 'manual' }).then((r) => {
+        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
+            return r.headers.get('location');
+    }).catch(() => null);
+}
+
 export function merge(a, b) {
     for (const k of Object.keys(b)) {
         if (Array.isArray(b[k])) {

src/util/tests.json

@@ -1132,6 +1132,31 @@
             "status": "stream"
         }
     }],
+    "snapchat": [{
+        "name": "spotlight",
+        "url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "shortlinked spotlight",
+        "url": "https://t.snapchat.com/4ZsiBLDi",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "redirect"
+        }
+    }, {
+        "name": "story",
+        "url": "https://www.snapchat.com/add/bazerkmakane",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "picker"
+        }
+    }],
     "loom": [{
         "name": "1080p video",
         "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
@@ -1218,4 +1243,4 @@
             "status": "redirect"
         }
     }]
-}
+}