kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[extractor/tiktok] Fix subs, DouyinIE, improve _VALID_URL (#5676)

Browse source 
Closes #5665, Closes #2267
Authored by: bashonly
This commit is contained in:
bashonly 2022-12-01 04:00:32 +00:00 committed by GitHub
parent 9bcfe33be7
commit ba72399723
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 99 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

152
yt_dlp/extractor/tiktok.py
View file

@ -16,6 +16,7 @@ from ..utils import (
int_or_none, int_or_none,
join_nonempty, join_nonempty,
qualities, qualities,
remove_start,
srt_subtitles_timecode, srt_subtitles_timecode,
str_or_none, str_or_none,
traverse_obj, traverse_obj,
@ -51,7 +52,7 @@ class TikTokBaseIE(InfoExtractor):
return self._download_json( return self._download_json(
'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id, 'https://%s/aweme/v1/%s/' % (self._API_HOSTNAME, ep), video_id=video_id,
fatal=fatal, note=note, errnote=errnote, headers={ fatal=fatal, note=note, errnote=errnote, headers={
'User-Agent': f'com.ss.android.ugc.trill/{manifest_app_version} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)', 'User-Agent': f'com.ss.android.ugc.{self._APP_NAME}/{manifest_app_version} (Linux; U; Android 10; en_US; Pixel 4; Build/QQ3A.200805.001; Cronet/58.0.2991.0)',
'Accept': 'application/json', 'Accept': 'application/json',
}, query=query) }, query=query)
@ -126,11 +127,21 @@ class TikTokBaseIE(InfoExtractor):
continue continue
raise e raise e
def _extract_aweme_app(self, aweme_id):
feed_list = self._call_api(
'feed', {'aweme_id': aweme_id}, aweme_id, note='Downloading video feed',
errnote='Unable to download video feed').get('aweme_list') or []
aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None)
if not aweme_detail:
raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
return self._parse_aweme_video_app(aweme_detail)
def _get_subtitles(self, aweme_detail, aweme_id): def _get_subtitles(self, aweme_detail, aweme_id):
# TODO: Extract text positioning info # TODO: Extract text positioning info
subtitles = {} subtitles = {}
# aweme/detail endpoint subs
captions_info = traverse_obj( captions_info = traverse_obj(
aweme_detail, ('interaction_stickers', ..., 'auto_video_caption_info', 'auto_captions', ...), expected_type=dict, default=[]) aweme_detail, ('interaction_stickers', ..., 'auto_video_caption_info', 'auto_captions', ...), expected_type=dict)
for caption in captions_info: for caption in captions_info:
caption_url = traverse_obj(caption, ('url', 'url_list', ...), expected_type=url_or_none, get_all=False) caption_url = traverse_obj(caption, ('url', 'url_list', ...), expected_type=url_or_none, get_all=False)
if not caption_url: if not caption_url:
@ -145,6 +156,24 @@ class TikTokBaseIE(InfoExtractor):
f'{i + 1}\n{srt_subtitles_timecode(line["start_time"] / 1000)} --> {srt_subtitles_timecode(line["end_time"] / 1000)}\n{line["text"]}' f'{i + 1}\n{srt_subtitles_timecode(line["start_time"] / 1000)} --> {srt_subtitles_timecode(line["end_time"] / 1000)}\n{line["text"]}'
for i, line in enumerate(caption_json['utterances']) if line.get('text')) for i, line in enumerate(caption_json['utterances']) if line.get('text'))
}) })
# feed endpoint subs
if not subtitles:
for caption in traverse_obj(aweme_detail, ('video', 'cla_info', 'caption_infos', ...), expected_type=dict):
if not caption.get('url'):
continue
subtitles.setdefault(caption.get('lang') or 'en', []).append({
'ext': remove_start(caption.get('caption_format'), 'web'),
'url': caption['url'],
})
# webpage subs
if not subtitles:
for caption in traverse_obj(aweme_detail, ('video', 'subtitleInfos', ...), expected_type=dict):
if not caption.get('Url'):
continue
subtitles.setdefault(caption.get('LanguageCodeName') or 'en', []).append({
'ext': remove_start(caption.get('Format'), 'web'),
'url': caption['Url'],
})
return subtitles return subtitles
def _parse_aweme_video_app(self, aweme_detail): def _parse_aweme_video_app(self, aweme_detail):
@ -354,7 +383,7 @@ class TikTokBaseIE(InfoExtractor):
'timestamp': int_or_none(aweme_detail.get('createTime')), 'timestamp': int_or_none(aweme_detail.get('createTime')),
'creator': str_or_none(author_info.get('nickname')), 'creator': str_or_none(author_info.get('nickname')),
'uploader': str_or_none(author_info.get('uniqueId') or aweme_detail.get('author')), 'uploader': str_or_none(author_info.get('uniqueId') or aweme_detail.get('author')),
'uploader_id': str_or_none(author_info.get('id') or aweme_detail.get('authorId')), 'uploader_id': str_or_none(traverse_obj(author_info, 'id', 'uid', 'authorId')),
'uploader_url': user_url, 'uploader_url': user_url,
'track': str_or_none(music_info.get('title')), 'track': str_or_none(music_info.get('title')),
'album': str_or_none(music_info.get('album')) or None, 'album': str_or_none(music_info.get('album')) or None,
@ -521,14 +550,6 @@ class TikTokIE(TikTokBaseIE):
'only_matching': True 'only_matching': True
}] }]
def _extract_aweme_app(self, aweme_id):
feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id,
note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or []
aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None)
if not aweme_detail:
raise ExtractorError('Unable to find video in feed', video_id=aweme_id)
return self._parse_aweme_video_app(aweme_detail)
def _real_extract(self, url): def _real_extract(self, url):
video_id, user_id = self._match_valid_url(url).group('id', 'user_id') video_id, user_id = self._match_valid_url(url).group('id', 'user_id')
try: try:
@ -763,56 +784,68 @@ class TikTokTagIE(TikTokBaseListIE):
return self.playlist_result(self._entries(tag_id, display_id), tag_id, display_id) return self.playlist_result(self._entries(tag_id, display_id), tag_id, display_id)
class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE class DouyinIE(TikTokBaseIE):
_VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)' _VALID_URL = r'https?://(?:www\.)?douyin\.com/video/(?P<id>[0-9]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.douyin.com/video/6961737553342991651', 'url': 'https://www.douyin.com/video/6961737553342991651',
'md5': '10523312c8b8100f353620ac9dc8f067', 'md5': 'a97db7e3e67eb57bf40735c022ffa228',
'info_dict': { 'info_dict': {
'id': '6961737553342991651', 'id': '6961737553342991651',
'ext': 'mp4', 'ext': 'mp4',
'title': '#杨超越 小小水手带你去远航❤️', 'title': '#杨超越 小小水手带你去远航❤️',
'uploader': '杨超越', 'description': '#杨超越 小小水手带你去远航❤️',
'upload_date': '20210513',
'timestamp': 1620905839,
'uploader_id': '110403406559', 'uploader_id': '110403406559',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
'creator': '杨超越',
'duration': 19782,
'timestamp': 1620905839,
'upload_date': '20210513',
'track': '@杨超越创作的原声',
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'repost_count': int, 'repost_count': int,
'comment_count': int, 'comment_count': int,
} },
}, { }, {
'url': 'https://www.douyin.com/video/6982497745948921092', 'url': 'https://www.douyin.com/video/6982497745948921092',
'md5': 'd78408c984b9b5102904cf6b6bc2d712', 'md5': '34a87ebff3833357733da3fe17e37c0e',
'info_dict': { 'info_dict': {
'id': '6982497745948921092', 'id': '6982497745948921092',
'ext': 'mp4', 'ext': 'mp4',
'title': '这个夏日和小羊@杨超越 一起遇见白色幻想', 'title': '这个夏日和小羊@杨超越 一起遇见白色幻想',
'uploader': '杨超越工作室', 'description': '这个夏日和小羊@杨超越 一起遇见白色幻想',
'upload_date': '20210708',
'timestamp': 1625739481,
'uploader_id': '408654318141572', 'uploader_id': '408654318141572',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAZJpnglcjW2f_CMVcnqA_6oVBXKWMpH0F8LIHuUu8-lA',
'creator': '杨超越工作室',
'duration': 42608,
'timestamp': 1625739481,
'upload_date': '20210708',
'track': '@杨超越工作室创作的原声',
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'repost_count': int, 'repost_count': int,
'comment_count': int, 'comment_count': int,
} },
}, { }, {
'url': 'https://www.douyin.com/video/6953975910773099811', 'url': 'https://www.douyin.com/video/6953975910773099811',
'md5': '72e882e24f75064c218b76c8b713c185', 'md5': 'dde3302460f19db59c47060ff013b902',
'info_dict': { 'info_dict': {
'id': '6953975910773099811', 'id': '6953975910773099811',
'ext': 'mp4', 'ext': 'mp4',
'title': '#一起看海 出现在你的夏日里', 'title': '#一起看海 出现在你的夏日里',
'uploader': '杨超越', 'description': '#一起看海 出现在你的夏日里',
'upload_date': '20210422',
'timestamp': 1619098692,
'uploader_id': '110403406559', 'uploader_id': '110403406559',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
'creator': '杨超越',
'duration': 17228,
'timestamp': 1619098692,
'upload_date': '20210422',
'track': '@杨超越创作的原声',
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'repost_count': int, 'repost_count': int,
'comment_count': int, 'comment_count': int,
} },
}, { }, {
'url': 'https://www.douyin.com/video/6950251282489675042', 'url': 'https://www.douyin.com/video/6950251282489675042',
'md5': 'b4db86aec367ef810ddd38b1737d2fed', 'md5': 'b4db86aec367ef810ddd38b1737d2fed',
@ -828,25 +861,30 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
'like_count': int, 'like_count': int,
'repost_count': int, 'repost_count': int,
'comment_count': int, 'comment_count': int,
} },
'skip': 'No longer available',
}, { }, {
'url': 'https://www.douyin.com/video/6963263655114722595', 'url': 'https://www.douyin.com/video/6963263655114722595',
'md5': '1abe1c477d05ee62efb40bf2329957cf', 'md5': 'cf9f11f0ec45d131445ec2f06766e122',
'info_dict': { 'info_dict': {
'id': '6963263655114722595', 'id': '6963263655114722595',
'ext': 'mp4', 'ext': 'mp4',
'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈', 'title': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
'uploader': '杨超越', 'description': '#哪个爱豆的105度最甜 换个角度看看我哈哈',
'upload_date': '20210517',
'timestamp': 1621261163,
'uploader_id': '110403406559', 'uploader_id': '110403406559',
'uploader_url': 'https://www.douyin.com/user/MS4wLjABAAAAEKnfa654JAJ_N5lgZDQluwsxmY0lhfmEYNQBBkwGG98',
'creator': '杨超越',
'duration': 15115,
'timestamp': 1621261163,
'upload_date': '20210517',
'track': '@杨超越创作的原声',
'view_count': int, 'view_count': int,
'like_count': int, 'like_count': int,
'repost_count': int, 'repost_count': int,
'comment_count': int, 'comment_count': int,
} },
}] }]
_APP_VERSIONS = [('9.6.0', '960')] _APP_VERSIONS = [('23.3.0', '230300')]
_APP_NAME = 'aweme' _APP_NAME = 'aweme'
_AID = 1128 _AID = 1128
_API_HOSTNAME = 'aweme.snssdk.com' _API_HOSTNAME = 'aweme.snssdk.com'
@ -859,7 +897,8 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
try: try:
return self._extract_aweme_app(video_id) return self._extract_aweme_app(video_id)
except ExtractorError as e: except ExtractorError as e:
self.report_warning(f'{e}; trying with webpage') e.expected = True
self.to_screen(f'{e}; trying with webpage')
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
render_data_json = self._search_regex( render_data_json = self._search_regex(
@ -867,7 +906,10 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
webpage, 'render data', default=None) webpage, 'render data', default=None)
if not render_data_json: if not render_data_json:
# TODO: Run verification challenge code to generate signature cookies # TODO: Run verification challenge code to generate signature cookies
raise ExtractorError('Fresh cookies (not necessarily logged in) are needed') cookies = self._get_cookies(self._WEBPAGE_HOST)
expected = not cookies.get('s_v_web_id') or not cookies.get('ttwid')
raise ExtractorError(
'Fresh cookies (not necessarily logged in) are needed', expected=expected)
render_data = self._parse_json( render_data = self._parse_json(
render_data_json, video_id, transform_source=compat_urllib_parse_unquote) render_data_json, video_id, transform_source=compat_urllib_parse_unquote)
@ -875,31 +917,35 @@ class DouyinIE(TikTokIE): # XXX: Do not subclass from concrete IE
class TikTokVMIE(InfoExtractor): class TikTokVMIE(InfoExtractor):
_VALID_URL = r'https?://(?:vm|vt)\.tiktok\.com/(?P<id>\w+)' _VALID_URL = r'https?://(?:(?:vm|vt)\.tiktok\.com|(?:www\.)tiktok\.com/t)/(?P<id>\w+)'
IE_NAME = 'vm.tiktok' IE_NAME = 'vm.tiktok'
_TESTS = [{ _TESTS = [{
'url': 'https://vm.tiktok.com/ZSe4FqkKd', 'url': 'https://www.tiktok.com/t/ZTRC5xgJp',
'info_dict': { 'info_dict': {
'id': '7023491746608712966', 'id': '7170520270497680683',
'ext': 'mp4', 'ext': 'mp4',
'title': 'md5:5607564db90271abbbf8294cca77eddd', 'title': 'md5:c64f6152330c2efe98093ccc8597871c',
'description': 'md5:5607564db90271abbbf8294cca77eddd', 'uploader_id': '6687535061741700102',
'duration': 11, 'upload_date': '20221127',
'upload_date': '20211026',
'uploader_id': '7007385080558846981',
'creator': 'Memes',
'artist': 'Memes',
'track': 'original sound',
'uploader': 'susmandem',
'timestamp': 1635284105,
'thumbnail': r're:https://.+\.webp.*',
'like_count': int,
'view_count': int, 'view_count': int,
'like_count': int,
'comment_count': int, 'comment_count': int,
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAObqu3WCTXxmw2xwZ3iLEHnEecEIw7ks6rxWqOqOhaPja9BI7gqUQnjw8_5FSoDXX',
'album': 'Wave of Mutilation: Best of Pixies',
'thumbnail': r're:https://.+\.webp.*',
'duration': 5,
'timestamp': 1669516858,
'repost_count': int, 'repost_count': int,
'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAXcNoOEOxVyBzuII_E--T0MeCrLP0ay1Sm6x_n3dluiWEoWZD0VlQOytwad4W0i0n', 'artist': 'Pixies',
} 'track': 'Where Is My Mind?',
'description': 'md5:c64f6152330c2efe98093ccc8597871c',
'uploader': 'sigmachaddeus',
'creator': 'SigmaChad',
},
}, {
'url': 'https://vm.tiktok.com/ZSe4FqkKd',
'only_matching': True,
}, { }, {
'url': 'https://vt.tiktok.com/ZSe4FqkKd', 'url': 'https://vt.tiktok.com/ZSe4FqkKd',
'only_matching': True, 'only_matching': True,