[Douyin] Rewrite extractor (#1157)

Closes #1121
Authored by: MinePlayersPE

yt_dlp/extractor/douyin.py

@@ -1,145 +0,0 @@
-# coding: utf-8
-
-from ..utils import (
-    int_or_none,
-    traverse_obj,
-    url_or_none,
-)
-from .common import (
-    InfoExtractor,
-    compat_urllib_parse_unquote,
-)
-
-
-class DouyinIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'https://www.douyin.com/video/6961737553342991651',
-        'md5': '10523312c8b8100f353620ac9dc8f067',
-        'info_dict': {
-            'id': '6961737553342991651',
-            'ext': 'mp4',
-            'title': '#杨超越  小小水手带你去远航❤️',
-            'uploader': '杨超越',
-            'upload_date': '20210513',
-            'timestamp': 1620905839,
-            'uploader_id': '110403406559',
-            'view_count': int,
-            'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
-        }
-    }, {
-        'url': 'https://www.douyin.com/video/6982497745948921092',
-        'md5': 'd78408c984b9b5102904cf6b6bc2d712',
-        'info_dict': {
-            'id': '6982497745948921092',
-            'ext': 'mp4',
-            'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
-            'uploader': '杨超越工作室',
-            'upload_date': '20210708',
-            'timestamp': 1625739481,
-            'uploader_id': '408654318141572',
-            'view_count': int,
-            'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
-        }
-    }, {
-        'url': 'https://www.douyin.com/video/6953975910773099811',
-        'md5': '72e882e24f75064c218b76c8b713c185',
-        'info_dict': {
-            'id': '6953975910773099811',
-            'ext': 'mp4',
-            'title': '#一起看海  出现在你的夏日里',
-            'uploader': '杨超越',
-            'upload_date': '20210422',
-            'timestamp': 1619098692,
-            'uploader_id': '110403406559',
-            'view_count': int,
-            'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
-        }
-    }, {
-        'url': 'https://www.douyin.com/video/6950251282489675042',
-        'md5': 'b4db86aec367ef810ddd38b1737d2fed',
-        'info_dict': {
-            'id': '6950251282489675042',
-            'ext': 'mp4',
-            'title': '哈哈哈，成功了哈哈哈哈哈哈',
-            'uploader': '杨超越',
-            'upload_date': '20210412',
-            'timestamp': 1618231483,
-            'uploader_id': '110403406559',
-            'view_count': int,
-            'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
-        }
-    }, {
-        'url': 'https://www.douyin.com/video/6963263655114722595',
-        'md5': '1abe1c477d05ee62efb40bf2329957cf',
-        'info_dict': {
-            'id': '6963263655114722595',
-            'ext': 'mp4',
-            'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
-            'uploader': '杨超越',
-            'upload_date': '20210517',
-            'timestamp': 1621261163,
-            'uploader_id': '110403406559',
-            'view_count': int,
-            'like_count': int,
-            'repost_count': int,
-            'comment_count': int,
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        render_data = self._parse_json(
-            self._search_regex(
-                r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>(%7B.+%7D)</script>',
-                webpage, 'render data'),
-            video_id, transform_source=compat_urllib_parse_unquote)
-        details = traverse_obj(render_data, (..., 'aweme', 'detail'), get_all=False)
-
-        thumbnails = [{'url': self._proto_relative_url(url)} for url in traverse_obj(
-            details, ('video', ('cover', 'dynamicCover', 'originCover')), expected_type=url_or_none, default=[])]
-
-        common = {
-            'width': traverse_obj(details, ('video', 'width'), expected_type=int),
-            'height': traverse_obj(details, ('video', 'height'), expected_type=int),
-            'ext': 'mp4',
-        }
-        formats = [{**common, 'url': self._proto_relative_url(url)} for url in traverse_obj(
-            details, ('video', 'playAddr', ..., 'src'), expected_type=url_or_none, default=[]) if url]
-        self._remove_duplicate_formats(formats)
-
-        download_url = traverse_obj(details, ('download', 'url'), expected_type=url_or_none)
-        if download_url:
-            formats.append({
-                **common,
-                'format_id': 'download',
-                'url': self._proto_relative_url(download_url),
-                'quality': 1,
-            })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': details.get('desc') or self._html_search_meta('title', webpage),
-            'formats': formats,
-            'thumbnails': thumbnails,
-            'uploader': traverse_obj(details, ('authorInfo', 'nickname'), expected_type=str),
-            'uploader_id': traverse_obj(details, ('authorInfo', 'uid'), expected_type=str),
-            'uploader_url': 'https://www.douyin.com/user/%s' % traverse_obj(
-                details, ('authorInfo', 'secUid'), expected_type=str),
-            'timestamp': int_or_none(details.get('createTime')),
-            'duration': traverse_obj(details, ('video', 'duration'), expected_type=int),
-            'view_count': traverse_obj(details, ('stats', 'playCount'), expected_type=int),
-            'like_count': traverse_obj(details, ('stats', 'diggCount'), expected_type=int),
-            'repost_count': traverse_obj(details, ('stats', 'shareCount'), expected_type=int),
-            'comment_count': traverse_obj(details, ('stats', 'commentCount'), expected_type=int),
-        }

yt_dlp/extractor/extractors.py

@@ -340,7 +340,6 @@ from .discoveryplusindia import (
     DiscoveryPlusIndiaShowIE,
 )
 from .dotsub import DotsubIE
-from .douyin import DouyinIE
 from .douyutv import (
     DouyuShowIE,
     DouyuTVIE,
@@ -1445,6 +1444,7 @@ from .threeqsdn import ThreeQSDNIE
 from .tiktok import (
     TikTokIE,
     TikTokUserIE,
+    DouyinIE,
 )
 from .tinypic import TinyPicIE
 from .tmz import TMZIE

yt_dlp/extractor/tiktok.py

@@ -8,12 +8,14 @@ import time
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     int_or_none,
     str_or_none,
     traverse_obj,
     try_get,
+    url_or_none,
     qualities,
 )
 
@@ -21,6 +23,10 @@ from ..utils import (
 class TikTokBaseIE(InfoExtractor):
     _APP_VERSION = '20.9.3'
     _MANIFEST_APP_VERSION = '291'
+    _APP_NAME = 'trill'
+    _AID = 1180
+    _API_HOSTNAME = 'api-t2.tiktokv.com'
+    _UPLOADER_URL_FORMAT = 'https://www.tiktok.com/@%s'
     QUALITIES = ('360p', '540p', '720p')
 
     def _call_api(self, ep, query, video_id, fatal=True,
@@ -46,7 +52,7 @@ class TikTokBaseIE(InfoExtractor):
             'carrier_region': 'US',
             'sys_region': 'US',
             'region': 'US',
-            'app_name': 'trill',
+            'app_name': self._APP_NAME,
             'app_language': 'en',
             'language': 'en',
             'timezone_name': 'America/New_York',
@@ -55,20 +61,20 @@ class TikTokBaseIE(InfoExtractor):
             'ac': 'wifi',
             'mcc_mnc': '310260',
             'is_my_cn': 0,
-            'aid': 1180,
+            'aid': self._AID,
             'ssmix': 'a',
             'as': 'a1qwert123',
             'cp': 'cbfhckdckkde1',
         }
-        self._set_cookie('.tiktokv.com', 'odin_tt', ''.join(random.choice('0123456789abcdef') for i in range(160)))
+        self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for i in range(160)))
         return self._download_json(
-            'https://api-t2.tiktokv.com/aweme/v1/%s/' % ep, video_id=video_id,
+            'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id,
             fatal=fatal, note=note, errnote=errnote, headers={
                 'User-Agent': f'com.ss.android.ugc.trill/{self._MANIFEST_APP_VERSION} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
                 'Accept': 'application/json',
             }, query=real_query)
 
-    def _parse_aweme_video(self, aweme_detail):
+    def _parse_aweme_video_app(self, aweme_detail):
         aweme_id = aweme_detail['aweme_id']
         video_info = aweme_detail['video']
 
@@ -146,6 +152,7 @@ class TikTokBaseIE(InfoExtractor):
                     'tbr': try_get(bitrate, lambda x: x['bit_rate'] / 1000),
                     'vcodec': 'h265' if traverse_obj(
                         bitrate, 'is_bytevc1', 'is_h265') else 'h264',
+                    'fps': bitrate.get('FPS'),
                 }))
 
         self._remove_duplicate_formats(formats)
@@ -165,7 +172,9 @@ class TikTokBaseIE(InfoExtractor):
         stats_info = aweme_detail.get('statistics', {})
         author_info = aweme_detail.get('author', {})
         music_info = aweme_detail.get('music', {})
-        user_id = str_or_none(author_info.get('nickname'))
+        user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
+                                                             'sec_uid', 'id', 'uid', 'unique_id',
+                                                             expected_type=str_or_none, get_all=False))
 
         contained_music_track = traverse_obj(
             music_info, ('matched_song', 'title'), ('matched_pgc_sound', 'title'), expected_type=str)
@@ -187,9 +196,9 @@ class TikTokBaseIE(InfoExtractor):
             'repost_count': int_or_none(stats_info.get('share_count')),
             'comment_count': int_or_none(stats_info.get('comment_count')),
             'uploader': str_or_none(author_info.get('unique_id')),
-            'creator': user_id,
+            'creator': str_or_none(author_info.get('nickname')),
             'uploader_id': str_or_none(author_info.get('uid')),
-            'uploader_url': f'https://www.tiktok.com/@{user_id}' if user_id else None,
+            'uploader_url': user_url,
             'track': music_track,
             'album': str_or_none(music_info.get('album')) or None,
             'artist': music_author,
@@ -199,6 +208,79 @@ class TikTokBaseIE(InfoExtractor):
             'duration': int_or_none(traverse_obj(video_info, 'duration', ('download_addr', 'duration')), scale=1000)
         }
 
+    def _parse_aweme_video_web(self, aweme_detail, webpage, url):
+        video_info = aweme_detail['video']
+        author_info = traverse_obj(aweme_detail, 'author', 'authorInfo', default={})
+        music_info = aweme_detail.get('music') or {}
+        stats_info = aweme_detail.get('stats') or {}
+        user_url = self._UPLOADER_URL_FORMAT % (traverse_obj(author_info,
+                                                             'secUid', 'id', 'uid', 'uniqueId',
+                                                             expected_type=str_or_none, get_all=False))
+
+        formats = []
+        play_url = video_info.get('playAddr')
+        width = video_info.get('width')
+        height = video_info.get('height')
+        if isinstance(play_url, str):
+            formats = [{
+                'url': self._proto_relative_url(play_url),
+                'ext': 'mp4',
+                'width': width,
+                'height': height,
+            }]
+        elif isinstance(play_url, list):
+            formats = [{
+                'url': self._proto_relative_url(url),
+                'ext': 'mp4',
+                'width': width,
+                'height': height,
+            } for url in traverse_obj(play_url, (..., 'src'), expected_type=url_or_none, default=[]) if url]
+
+        download_url = url_or_none(video_info.get('downloadAddr')) or traverse_obj(video_info, ('download', 'url'), expected_type=url_or_none)
+        if download_url:
+            formats.append({
+                'format_id': 'download',
+                'url': self._proto_relative_url(download_url),
+                'ext': 'mp4',
+                'width': width,
+                'height': height,
+            })
+        self._remove_duplicate_formats(formats)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for thumbnail_name in ('thumbnail', 'cover', 'dynamicCover', 'originCover'):
+            if aweme_detail.get(thumbnail_name):
+                thumbnails = [{
+                    'url': self._proto_relative_url(aweme_detail[thumbnail_name]),
+                    'width': width,
+                    'height': height
+                }]
+
+        return {
+            'id': traverse_obj(aweme_detail, 'id', 'awemeId', expected_type=str_or_none),
+            'title': aweme_detail.get('desc'),
+            'duration': try_get(aweme_detail, lambda x: x['video']['duration'], int),
+            'view_count': int_or_none(stats_info.get('playCount')),
+            'like_count': int_or_none(stats_info.get('diggCount')),
+            'repost_count': int_or_none(stats_info.get('shareCount')),
+            'comment_count': int_or_none(stats_info.get('commentCount')),
+            'timestamp': int_or_none(aweme_detail.get('createTime')),
+            'creator': str_or_none(author_info.get('nickname')),
+            'uploader': str_or_none(author_info.get('uniqueId')),
+            'uploader_id': str_or_none(author_info.get('id')),
+            'uploader_url': user_url,
+            'track': str_or_none(music_info.get('title')),
+            'album': str_or_none(music_info.get('album')) or None,
+            'artist': str_or_none(music_info.get('authorName')),
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': str_or_none(aweme_detail.get('desc')),
+            'http_headers': {
+                'Referer': url
+            }
+        }
+
 
 class TikTokIE(TikTokBaseIE):
     _VALID_URL = r'https?://www\.tiktok\.com/@[\w\.-]+/video/(?P<id>\d+)'
@@ -255,60 +337,10 @@ class TikTokIE(TikTokBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_aweme(self, props_data, webpage, url):
-        video_info = try_get(
-            props_data, lambda x: x['pageProps']['itemInfo']['itemStruct'], dict)
-        author_info = try_get(
-            props_data, lambda x: x['pageProps']['itemInfo']['itemStruct']['author'], dict) or {}
-        music_info = try_get(
-            props_data, lambda x: x['pageProps']['itemInfo']['itemStruct']['music'], dict) or {}
-        stats_info = try_get(props_data, lambda x: x['pageProps']['itemInfo']['itemStruct']['stats'], dict) or {}
-
-        user_id = str_or_none(author_info.get('uniqueId'))
-        download_url = try_get(video_info, (lambda x: x['video']['playAddr'],
-                                            lambda x: x['video']['downloadAddr']))
-        height = try_get(video_info, lambda x: x['video']['height'], int)
-        width = try_get(video_info, lambda x: x['video']['width'], int)
-        thumbnails = [{
-            'url': video_info.get('thumbnail') or self._og_search_thumbnail(webpage),
-            'width': width,
-            'height': height
-        }]
-        tracker = try_get(props_data, lambda x: x['initialProps']['$wid'])
-
-        return {
-            'id': str_or_none(video_info.get('id')),
-            'url': download_url,
-            'ext': 'mp4',
-            'height': height,
-            'width': width,
-            'title': video_info.get('desc') or self._og_search_title(webpage),
-            'duration': try_get(video_info, lambda x: x['video']['duration'], int),
-            'view_count': int_or_none(stats_info.get('playCount')),
-            'like_count': int_or_none(stats_info.get('diggCount')),
-            'repost_count': int_or_none(stats_info.get('shareCount')),
-            'comment_count': int_or_none(stats_info.get('commentCount')),
-            'timestamp': try_get(video_info, lambda x: int(x['createTime']), int),
-            'creator': str_or_none(author_info.get('nickname')),
-            'uploader': user_id,
-            'uploader_id': str_or_none(author_info.get('id')),
-            'uploader_url': f'https://www.tiktok.com/@{user_id}',
-            'track': str_or_none(music_info.get('title')),
-            'album': str_or_none(music_info.get('album')) or None,
-            'artist': str_or_none(music_info.get('authorName')),
-            'thumbnails': thumbnails,
-            'description': str_or_none(video_info.get('desc')),
-            'webpage_url': self._og_search_url(webpage),
-            'http_headers': {
-                'Referer': url,
-                'Cookie': 'tt_webid=%s; tt_webid_v2=%s' % (tracker, tracker),
-            }
-        }
-
     def _extract_aweme_app(self, aweme_id):
         aweme_detail = self._call_api('aweme/detail', {'aweme_id': aweme_id}, aweme_id,
                                       note='Downloading video details', errnote='Unable to download video details')['aweme_detail']
-        return self._parse_aweme_video(aweme_detail)
+        return self._parse_aweme_video_app(aweme_detail)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -330,7 +362,7 @@ class TikTokIE(TikTokBaseIE):
         # Chech statusCode for success
         status = props_data.get('pageProps').get('statusCode')
         if status == 0:
-            return self._extract_aweme(props_data, webpage, url)
+            return self._parse_aweme_video_web(props_data['pageProps']['itemInfo']['itemStruct'], webpage, url)
         elif status == 10216:
             raise ExtractorError('This video is private', expected=True)
 
@@ -413,3 +445,115 @@ class TikTokUserIE(TikTokBaseIE):
         })
         own_id = self._html_search_regex(r'snssdk\d*://user/profile/(\d+)', webpage, 'user ID')
         return self.playlist_result(self._entries_api(webpage, own_id, user_id), user_id)
+
+
+class DouyinIE(TikTokIE):
+    _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://www.douyin.com/video/6961737553342991651',
+        'md5': '10523312c8b8100f353620ac9dc8f067',
+        'info_dict': {
+            'id': '6961737553342991651',
+            'ext': 'mp4',
+            'title': '#杨超越  小小水手带你去远航❤️',
+            'uploader': '杨超越',
+            'upload_date': '20210513',
+            'timestamp': 1620905839,
+            'uploader_id': '110403406559',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        }
+    }, {
+        'url': 'https://www.douyin.com/video/6982497745948921092',
+        'md5': 'd78408c984b9b5102904cf6b6bc2d712',
+        'info_dict': {
+            'id': '6982497745948921092',
+            'ext': 'mp4',
+            'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
+            'uploader': '杨超越工作室',
+            'upload_date': '20210708',
+            'timestamp': 1625739481,
+            'uploader_id': '408654318141572',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        }
+    }, {
+        'url': 'https://www.douyin.com/video/6953975910773099811',
+        'md5': '72e882e24f75064c218b76c8b713c185',
+        'info_dict': {
+            'id': '6953975910773099811',
+            'ext': 'mp4',
+            'title': '#一起看海  出现在你的夏日里',
+            'uploader': '杨超越',
+            'upload_date': '20210422',
+            'timestamp': 1619098692,
+            'uploader_id': '110403406559',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        }
+    }, {
+        'url': 'https://www.douyin.com/video/6950251282489675042',
+        'md5': 'b4db86aec367ef810ddd38b1737d2fed',
+        'info_dict': {
+            'id': '6950251282489675042',
+            'ext': 'mp4',
+            'title': '哈哈哈，成功了哈哈哈哈哈哈',
+            'uploader': '杨超越',
+            'upload_date': '20210412',
+            'timestamp': 1618231483,
+            'uploader_id': '110403406559',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        }
+    }, {
+        'url': 'https://www.douyin.com/video/6963263655114722595',
+        'md5': '1abe1c477d05ee62efb40bf2329957cf',
+        'info_dict': {
+            'id': '6963263655114722595',
+            'ext': 'mp4',
+            'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
+            'uploader': '杨超越',
+            'upload_date': '20210517',
+            'timestamp': 1621261163,
+            'uploader_id': '110403406559',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+        }
+    }]
+    _APP_VERSION = '9.6.0'
+    _MANIFEST_APP_VERSION = '960'
+    _APP_NAME = 'aweme'
+    _AID = 1128
+    _API_HOSTNAME = 'aweme.snssdk.com'
+    _UPLOADER_URL_FORMAT = 'https://www.douyin.com/user/%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        try:
+            return self._extract_aweme_app(video_id)
+        except ExtractorError as e:
+            self.report_warning(f'{e}; Retrying with webpage')
+
+        webpage = self._download_webpage(url, video_id)
+        render_data_json = self._search_regex(
+            r'<script [^>]*\bid=[\'"]RENDER_DATA[\'"][^>]*>(%7B.+%7D)</script>',
+            webpage, 'render data', default=None)
+        if not render_data_json:
+            # TODO: Run verification challenge code to generate signature cookies
+            raise ExtractorError('Fresh cookies (not necessarily logged in) are needed')
+
+        render_data = self._parse_json(
+            render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
+        return self._parse_aweme_video_web(
+            traverse_obj(render_data, (..., 'aweme', 'detail'), get_all=False), webpage, url)