services: add snapchat support (#429)
* feat: snapchat support * chore: remove redundancy * chore: a bit of better matching * chore: update readme * refactor(snapchat): refactor story matching to use pickers * fix: small fix to directly linked stories * fix(snapchat): fix filenames * chore: update readme * ref(snapchat): rewrite service, new test, split redirects into a util * fix(snapchat): small fixes * chore: deepscan error fixed * fix: remove debug logging * fix(snapchat): fix merge, clean up code with new utils * fix(snapchat): update with suggested changes --------- Signed-off-by: Snazzah <7025343+Snazzah@users.noreply.github.com> Co-authored-by: jj <log@riseup.net>
This commit is contained in:
Snazzah
GitHub
parent
c77ee2eb44
commit
4080cd4581
8 changed files with 153 additions and 1 deletions
|
|
@ -25,6 +25,7 @@ this list is not final and keeps expanding over time. if support for a service y
|
|||
| pinterest | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| reddit | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| rutube | ✅ | ✅ | ✅ | ✅ | ✅ |
|
||||
| snapchat stories & spotlights | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| soundcloud | ➖ | ✅ | ➖ | ✅ | ✅ |
|
||||
| streamable | ✅ | ✅ | ✅ | ➖ | ➖ |
|
||||
| tiktok | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
|
|
@ -49,6 +50,7 @@ this list is not final and keeps expanding over time. if support for a service y
|
|||
| facebook | supports public accessible videos content only. |
|
||||
| pinterest | supports photos, gifs, videos and stories. |
|
||||
| reddit | supports gifs and videos. |
|
||||
| snapchat | supports spotlights and stories. lets you pick what to save from stories. |
|
||||
| rutube | supports yappy & private links. |
|
||||
| soundcloud | supports private links. |
|
||||
| tiktok | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios. |
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import streamable from "./services/streamable.js";
|
|||
import twitch from "./services/twitch.js";
|
||||
import rutube from "./services/rutube.js";
|
||||
import dailymotion from "./services/dailymotion.js";
|
||||
import snapchat from "./services/snapchat.js";
|
||||
import loom from "./services/loom.js";
|
||||
import facebook from "./services/facebook.js";
|
||||
|
||||
|
|
@ -189,6 +190,14 @@ export default async function(host, patternMatch, lang, obj) {
|
|||
case "dailymotion":
|
||||
r = await dailymotion(patternMatch);
|
||||
break;
|
||||
case "snapchat":
|
||||
r = await snapchat({
|
||||
url,
|
||||
username: patternMatch.username,
|
||||
storyId: patternMatch.storyId,
|
||||
spotlightId: patternMatch.spotlightId,
|
||||
shortLink: patternMatch.shortLink || false
|
||||
});
|
||||
case "loom":
|
||||
r = await loom({
|
||||
id: patternMatch.id
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
|
|||
switch (host) {
|
||||
case "instagram":
|
||||
case "twitter":
|
||||
case "snapchat":
|
||||
params = { picker: r.picker };
|
||||
break;
|
||||
case "tiktok":
|
||||
|
|
@ -136,6 +137,7 @@ export default function(r, host, userFormat, isAudioOnly, lang, isAudioMuted, di
|
|||
case "tumblr":
|
||||
case "pinterest":
|
||||
case "streamable":
|
||||
case "snapchat":
|
||||
case "loom":
|
||||
responseType = "redirect";
|
||||
break;
|
||||
|
|
|
|||
96
src/modules/processing/services/snapchat.js
Normal file
96
src/modules/processing/services/snapchat.js
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import { genericUserAgent } from "../../config.js";
|
||||
import { getRedirectingURL } from "../../sub/utils.js";
|
||||
import { extract, normalizeURL } from "../url.js";
|
||||
|
||||
const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="(https:\/\/cf-st\.sc-cdn\.net\/d\/[\w.?=]+&uc=\d+)" as="video"\/>/;
|
||||
const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
|
||||
|
||||
async function getSpotlight(id) {
|
||||
const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
|
||||
headers: { 'User-Agent': genericUserAgent }
|
||||
}).then((r) => r.text()).catch(() => null);
|
||||
if (!html) {
|
||||
return { error: 'ErrorCouldntFetch' };
|
||||
}
|
||||
|
||||
const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
|
||||
if (videoURL) {
|
||||
return {
|
||||
urls: videoURL,
|
||||
filename: `snapchat_${id}.mp4`,
|
||||
audioFilename: `snapchat_${id}_audio`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getStory(username, storyId) {
|
||||
const html = await fetch(`https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`, {
|
||||
headers: { 'User-Agent': genericUserAgent }
|
||||
}).then((r) => r.text()).catch(() => null);
|
||||
if (!html) {
|
||||
return { error: 'ErrorCouldntFetch' };
|
||||
}
|
||||
|
||||
const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
|
||||
if (nextDataString) {
|
||||
const data = JSON.parse(nextDataString);
|
||||
const storyIdParam = data.query.profileParams[1];
|
||||
|
||||
if (storyIdParam && data.props.pageProps.story) {
|
||||
const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
|
||||
if (story) {
|
||||
if (story.snapMediaType === 0) {
|
||||
return {
|
||||
urls: story.snapUrls.mediaUrl,
|
||||
isPhoto: true
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
urls: story.snapUrls.mediaUrl,
|
||||
filename: `snapchat_${storyId}.mp4`,
|
||||
audioFilename: `snapchat_${storyId}_audio`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const defaultStory = data.props.pageProps.curatedHighlights[0];
|
||||
if (defaultStory) {
|
||||
return {
|
||||
picker: defaultStory.snapList.map((snap) => ({
|
||||
type: snap.snapMediaType === 0 ? 'photo' : 'video',
|
||||
url: snap.snapUrls.mediaUrl,
|
||||
thumb: snap.snapUrls.mediaPreviewUrl.value
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default async function(obj) {
|
||||
let params = obj;
|
||||
if (obj.url.hostname === 't.snapchat.com' && obj.shortLink) {
|
||||
const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
|
||||
|
||||
if (!link?.startsWith('https://www.snapchat.com/')) {
|
||||
return { error: 'ErrorCouldntFetch' };
|
||||
}
|
||||
|
||||
const extractResult = extract(normalizeURL(link));
|
||||
if (extractResult?.host !== 'snapchat') {
|
||||
return { error: 'ErrorCouldntFetch' };
|
||||
}
|
||||
|
||||
params = extractResult.patternMatch;
|
||||
}
|
||||
|
||||
if (params.spotlightId) {
|
||||
const result = await getSpotlight(params.spotlightId);
|
||||
if (result) return result;
|
||||
} else if (params.username) {
|
||||
const result = await getStory(params.username, params.storyId);
|
||||
if (result) return result;
|
||||
}
|
||||
|
||||
return { error: 'ErrorCouldntFetch' };
|
||||
}
|
||||
|
|
@ -114,6 +114,12 @@
|
|||
"patterns": ["video/:id"],
|
||||
"enabled": true
|
||||
},
|
||||
"snapchat": {
|
||||
"alias": "snapchat stories & spotlights",
|
||||
"subdomains": ["t", "story"],
|
||||
"patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"],
|
||||
"enabled": true
|
||||
},
|
||||
"loom": {
|
||||
"alias": "loom videos",
|
||||
"patterns": ["share/:id"],
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@ export const testers = {
|
|||
(patternMatch.author?.length <= 255 && patternMatch.song?.length <= 255)
|
||||
|| patternMatch.shortLink?.length <= 32,
|
||||
|
||||
"snapchat": (patternMatch) =>
|
||||
(patternMatch.username?.length <= 32 && (!patternMatch.storyId || patternMatch.storyId?.length <= 255))
|
||||
|| patternMatch.spotlightId?.length <= 255
|
||||
|| patternMatch.shortLink?.length <= 16,
|
||||
|
||||
"streamable": (patternMatch) =>
|
||||
patternMatch.id?.length === 6,
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,13 @@ export function cleanHTML(html) {
|
|||
return clean
|
||||
}
|
||||
|
||||
export function getRedirectingURL(url) {
|
||||
return fetch(url, { redirect: 'manual' }).then((r) => {
|
||||
if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
|
||||
return r.headers.get('location');
|
||||
}).catch(() => null);
|
||||
}
|
||||
|
||||
export function merge(a, b) {
|
||||
for (const k of Object.keys(b)) {
|
||||
if (Array.isArray(b[k])) {
|
||||
|
|
|
|||
|
|
@ -1132,6 +1132,31 @@
|
|||
"status": "stream"
|
||||
}
|
||||
}],
|
||||
"snapchat": [{
|
||||
"name": "spotlight",
|
||||
"url": "https://www.snapchat.com/spotlight/W7_EDlXWTBiXAEEniNoMPwAAYdWxucG9pZmNqAY46j0a5AY46j0YbAAAAAQ",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
}, {
|
||||
"name": "shortlinked spotlight",
|
||||
"url": "https://t.snapchat.com/4ZsiBLDi",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "redirect"
|
||||
}
|
||||
}, {
|
||||
"name": "story",
|
||||
"url": "https://www.snapchat.com/add/bazerkmakane",
|
||||
"params": {},
|
||||
"expected": {
|
||||
"code": 200,
|
||||
"status": "picker"
|
||||
}
|
||||
}],
|
||||
"loom": [{
|
||||
"name": "1080p video",
|
||||
"url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761",
|
||||
|
|
@ -1218,4 +1243,4 @@
|
|||
"status": "redirect"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue