From 02c7ae81045d35401301cf15346fcb41dfee61bf Mon Sep 17 00:00:00 2001
From: u-spec-png <54671367+u-spec-png@users.noreply.github.com>
Date: Sun, 12 Sep 2021 05:37:44 +0000
Subject: [PATCH] [Newgrounds] Add `NewgroundsUserIE` and improve extractor
(#942)
Authored by: u-spec-png
---
yt_dlp/extractor/extractors.py | 1 +
yt_dlp/extractor/newgrounds.py | 86 +++++++++++++++++++++++++++++-----
2 files changed, 74 insertions(+), 13 deletions(-)
diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py
index 16bc78ffc..e456475e5 100644
--- a/yt_dlp/extractor/extractors.py
+++ b/yt_dlp/extractor/extractors.py
@@ -867,6 +867,7 @@ from .neteasemusic import (
from .newgrounds import (
NewgroundsIE,
NewgroundsPlaylistIE,
+ NewgroundsUserIE,
)
from .newstube import NewstubeIE
from .nextmedia import (
diff --git a/yt_dlp/extractor/newgrounds.py b/yt_dlp/extractor/newgrounds.py
index 41549a2f1..25b468b7d 100644
--- a/yt_dlp/extractor/newgrounds.py
+++ b/yt_dlp/extractor/newgrounds.py
@@ -1,5 +1,7 @@
+# coding: utf-8
from __future__ import unicode_literals
+import functools
import re
from .common import InfoExtractor
@@ -8,8 +10,9 @@ from ..utils import (
int_or_none,
parse_count,
parse_duration,
- parse_filesize,
unified_timestamp,
+ OnDemandPagedList,
+ try_get,
)
@@ -88,10 +91,10 @@ class NewgroundsIE(InfoExtractor):
webpage = self._download_webpage(url, media_id)
title = self._html_search_regex(
- r'([^>]+)', webpage, 'title')
+ r'(.+?)', webpage, 'title')
media_url_string = self._search_regex(
- r'"url"\s*:\s*("[^"]+"),', webpage, 'media url', default=None, fatal=False)
+ r'"url"\s*:\s*("[^"]+"),', webpage, 'media url', default=None)
if media_url_string:
media_url = self._parse_json(media_url_string, media_id)
@@ -128,20 +131,26 @@ class NewgroundsIE(InfoExtractor):
(r'\s*Uploaded\s*\s*([^<]+\s*[^<]+)',
r'\s*Uploaded\s*\s*([^<]+)'), webpage, 'timestamp',
default=None))
- duration = parse_duration(self._search_regex(
- r'(?s)\s*Song\s*\s*.+?\s*([^<]+)', webpage,
+ duration = parse_duration(self._html_search_regex(
+ r'"duration"\s*:\s*["\']?([\d]+)["\']?,', webpage,
'duration', default=None))
- view_count = parse_count(self._html_search_regex(r'(?s)\s*Views\s*\s*([\d\.,]+)', webpage,
- 'view_count', fatal=False, default=None))
+ view_count = parse_count(self._html_search_regex(
+ r'(?s)\s*Views\s*\s*([\d\.,]+)', webpage,
+ 'view count', default=None))
- filesize_approx = parse_filesize(self._html_search_regex(
- r'(?s)\s*Song\s*\s*(.+?)', webpage, 'filesize',
+ filesize = int_or_none(self._html_search_regex(
+ r'"filesize"\s*:\s*["\']?([\d]+)["\']?,', webpage, 'filesize',
default=None))
- if len(formats) == 1:
- formats[0]['filesize_approx'] = filesize_approx
- if 'Song' in webpage:
+ video_type_description = self._html_search_regex(
+ r'"description"\s*:\s*["\']?([^"\']+)["\']?,', webpage, 'filesize',
+ default=None)
+
+ if len(formats) == 1:
+ formats[0]['filesize'] = filesize
+
+ if video_type_description == 'Audio File':
formats[0]['vcodec'] = 'none'
self._check_formats(formats, media_id)
self._sort_formats(formats)
@@ -160,6 +169,7 @@ class NewgroundsIE(InfoExtractor):
class NewgroundsPlaylistIE(InfoExtractor):
+ IE_NAME = 'Newgrounds:playlist'
_VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:collection|[^/]+/search/[^/]+)/(?P[^/?#&]+)'
_TESTS = [{
'url': 'https://www.newgrounds.com/collection/cats',
@@ -202,7 +212,57 @@ class NewgroundsPlaylistIE(InfoExtractor):
continue
entries.append(
self.url_result(
- 'https://www.newgrounds.com/%s' % path,
+ f'https://www.newgrounds.com/{path}',
ie=NewgroundsIE.ie_key(), video_id=media_id))
return self.playlist_result(entries, playlist_id, title)
+
+
+class NewgroundsUserIE(InfoExtractor):
+ IE_NAME = 'Newgrounds:user'
+ _VALID_URL = r'https?://(?P[^\.]+)\.newgrounds\.com/(?:movies|audio)/?(?:[#?]|$)'
+ _TESTS = [{
+ 'url': 'https://burn7.newgrounds.com/audio',
+ 'info_dict': {
+ 'id': 'burn7',
+ },
+ 'playlist_mincount': 150,
+ }, {
+ 'url': 'https://burn7.newgrounds.com/movies',
+ 'info_dict': {
+ 'id': 'burn7',
+ },
+ 'playlist_mincount': 2,
+ }, {
+ 'url': 'https://brian-beaton.newgrounds.com/movies',
+ 'info_dict': {
+ 'id': 'brian-beaton',
+ },
+ 'playlist_mincount': 10,
+ }]
+ _PAGE_SIZE = 30
+
+ def _fetch_page(self, channel_id, url, page):
+ page += 1
+ posts_info = self._download_json(
+ f'{url}/page/{page}', channel_id,
+ note=f'Downloading page {page}', headers={
+ 'Accept': 'application/json, text/javascript, */*; q = 0.01',
+ 'X-Requested-With': 'XMLHttpRequest',
+ })
+ sequence = posts_info.get('sequence', [])
+ for year in sequence:
+ posts = try_get(posts_info, lambda x: x['years'][str(year)]['items'])
+ for post in posts:
+ path, media_id = self._search_regex(
+ r']+\bhref=["\'][^"\']+((?:portal/view|audio/listen)/(\d+))[^>]+>',
+ post, 'url', group=(1, 2))
+ yield self.url_result(f'https://www.newgrounds.com/{path}', NewgroundsIE.ie_key(), media_id)
+
+ def _real_extract(self, url):
+ channel_id = self._match_id(url)
+
+ entries = OnDemandPagedList(functools.partial(
+ self._fetch_page, channel_id, url), self._PAGE_SIZE)
+
+ return self.playlist_result(entries, channel_id)