kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[extractor/instagram] Extract more metadata (#4708)

Browse source 
Authored by: pritam20ps05
This commit is contained in:
Pritam Das 2022-09-22 16:27:20 +05:30 committed by GitHub
parent 80eb0bd9b9
commit 2e7675489f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 89 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

148
yt_dlp/extractor/instagram.py
View file

@ -173,18 +173,9 @@ class InstagramBaseIE(InfoExtractor):
if isinstance(product_info, list): if isinstance(product_info, list):
product_info = product_info[0] product_info = product_info[0]
comment_data = traverse_obj(product_info, ('edge_media_to_parent_comment', 'edges'))
comments = [{
'author': traverse_obj(comment_dict, ('node', 'owner', 'username')),
'author_id': traverse_obj(comment_dict, ('node', 'owner', 'id')),
'id': traverse_obj(comment_dict, ('node', 'id')),
'text': traverse_obj(comment_dict, ('node', 'text')),
'timestamp': traverse_obj(comment_dict, ('node', 'created_at'), expected_type=int_or_none),
} for comment_dict in comment_data] if comment_data else None
user_info = product_info.get('user') or {} user_info = product_info.get('user') or {}
info_dict = { info_dict = {
'id': product_info.get('code') or _pk_to_id(product_info.get('pk')), 'id': _pk_to_id(traverse_obj(product_info, 'pk', 'id', expected_type=str_or_none)[:19]),
'title': product_info.get('title') or f'Video by {user_info.get("username")}', 'title': product_info.get('title') or f'Video by {user_info.get("username")}',
'description': traverse_obj(product_info, ('caption', 'text'), expected_type=str_or_none), 'description': traverse_obj(product_info, ('caption', 'text'), expected_type=str_or_none),
'timestamp': int_or_none(product_info.get('taken_at')), 'timestamp': int_or_none(product_info.get('taken_at')),
@ -194,7 +185,7 @@ class InstagramBaseIE(InfoExtractor):
'view_count': int_or_none(product_info.get('view_count')), 'view_count': int_or_none(product_info.get('view_count')),
'like_count': int_or_none(product_info.get('like_count')), 'like_count': int_or_none(product_info.get('like_count')),
'comment_count': int_or_none(product_info.get('comment_count')), 'comment_count': int_or_none(product_info.get('comment_count')),
'comments': comments, '__post_extractor': self.extract_comments(_pk_to_id(product_info.get('pk'))),
'http_headers': { 'http_headers': {
'Referer': 'https://www.instagram.com/', 'Referer': 'https://www.instagram.com/',
} }
@ -216,6 +207,23 @@ class InstagramBaseIE(InfoExtractor):
**self._extract_product_media(product_info) **self._extract_product_media(product_info)
} }
def _get_comments(self, video_id):
comments_info = self._download_json(
f'{self._API_BASE_URL}/media/{_id_to_pk(video_id)}/comments/?can_support_threading=true&permalink_enabled=false', video_id,
fatal=False, errnote='Comments extraction failed', note='Downloading comments info', headers=self._API_HEADERS) or {}
comment_data = traverse_obj(comments_info, ('edge_media_to_parent_comment', 'edges'), 'comments')
for comment_dict in comment_data or []:
yield {
'author': traverse_obj(comment_dict, ('node', 'owner', 'username'), ('user', 'username')),
'author_id': traverse_obj(comment_dict, ('node', 'owner', 'id'), ('user', 'pk')),
'author_thumbnail': traverse_obj(comment_dict, ('node', 'owner', 'profile_pic_url'), ('user', 'profile_pic_url'), expected_type=url_or_none),
'id': traverse_obj(comment_dict, ('node', 'id'), 'pk'),
'text': traverse_obj(comment_dict, ('node', 'text'), 'text'),
'like_count': traverse_obj(comment_dict, ('node', 'edge_liked_by', 'count'), 'comment_like_count', expected_type=int_or_none),
'timestamp': traverse_obj(comment_dict, ('node', 'created_at'), 'created_at', expected_type=int_or_none),
}
class InstagramIOSIE(InfoExtractor): class InstagramIOSIE(InfoExtractor):
IE_DESC = 'IOS instagram:// URL' IE_DESC = 'IOS instagram:// URL'
@ -258,7 +266,7 @@ class InstagramIE(InstagramBaseIE):
'title': 'Video by naomipq', 'title': 'Video by naomipq',
'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8', 'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
'thumbnail': r're:^https?://.*\.jpg', 'thumbnail': r're:^https?://.*\.jpg',
'duration': 0, 'duration': 8.747,
'timestamp': 1371748545, 'timestamp': 1371748545,
'upload_date': '20130620', 'upload_date': '20130620',
'uploader_id': '2815873', 'uploader_id': '2815873',
@ -268,27 +276,34 @@ class InstagramIE(InstagramBaseIE):
'comment_count': int, 'comment_count': int,
'comments': list, 'comments': list,
}, },
'expected_warnings': [
'General metadata extraction failed',
'Main webpage is locked behind the login page',
],
}, { }, {
# missing description # reel
'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears', 'url': 'https://www.instagram.com/reel/Chunk8-jurw/',
'md5': 'f6d8277f74515fa3ff9f5791426e42b1',
'info_dict': { 'info_dict': {
'id': 'BA-pQFBG8HZ', 'id': 'Chunk8-jurw',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Video by britneyspears', 'title': 'Video by instagram',
'description': 'md5:c9cde483606ed6f80fbe9283a6a2b290',
'thumbnail': r're:^https?://.*\.jpg', 'thumbnail': r're:^https?://.*\.jpg',
'duration': 0, 'duration': 5.016,
'timestamp': 1453760977, 'timestamp': 1661529231,
'upload_date': '20160125', 'upload_date': '20220826',
'uploader_id': '12246775', 'uploader_id': '25025320',
'uploader': 'Britney Spears', 'uploader': 'Instagram',
'channel': 'britneyspears', 'channel': 'instagram',
'like_count': int, 'like_count': int,
'comment_count': int, 'comment_count': int,
'comments': list, 'comments': list,
}, },
'params': { 'expected_warnings': [
'skip_download': True, 'General metadata extraction failed',
}, 'Main webpage is locked behind the login page',
],
}, { }, {
# multi video post # multi video post
'url': 'https://www.instagram.com/p/BQ0eAlwhDrw/', 'url': 'https://www.instagram.com/p/BQ0eAlwhDrw/',
@ -297,18 +312,24 @@ class InstagramIE(InstagramBaseIE):
'id': 'BQ0dSaohpPW', 'id': 'BQ0dSaohpPW',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Video 1', 'title': 'Video 1',
'thumbnail': r're:^https?://.*\.jpg',
'view_count': int,
}, },
}, { }, {
'info_dict': { 'info_dict': {
'id': 'BQ0dTpOhuHT', 'id': 'BQ0dTpOhuHT',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Video 2', 'title': 'Video 2',
'thumbnail': r're:^https?://.*\.jpg',
'view_count': int,
}, },
}, { }, {
'info_dict': { 'info_dict': {
'id': 'BQ0dT7RBFeF', 'id': 'BQ0dT7RBFeF',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Video 3', 'title': 'Video 3',
'thumbnail': r're:^https?://.*\.jpg',
'view_count': int,
}, },
}], }],
'info_dict': { 'info_dict': {
@ -316,6 +337,10 @@ class InstagramIE(InstagramBaseIE):
'title': 'Post by instagram', 'title': 'Post by instagram',
'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957', 'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957',
}, },
'expected_warnings': [
'General metadata extraction failed',
'Main webpage is locked behind the login page',
],
}, { }, {
# IGTV # IGTV
'url': 'https://www.instagram.com/tv/BkfuX9UB-eK/', 'url': 'https://www.instagram.com/tv/BkfuX9UB-eK/',
@ -334,7 +359,11 @@ class InstagramIE(InstagramBaseIE):
'comment_count': int, 'comment_count': int,
'comments': list, 'comments': list,
'description': 'Meet Cass Hirst (@cass.fb), a fingerboarding pro who can perform tiny ollies and kickflips while blindfolded.', 'description': 'Meet Cass Hirst (@cass.fb), a fingerboarding pro who can perform tiny ollies and kickflips while blindfolded.',
} },
'expected_warnings': [
'General metadata extraction failed',
'Main webpage is locked behind the login page',
],
}, { }, {
'url': 'https://instagram.com/p/-Cmh1cukG2/', 'url': 'https://instagram.com/p/-Cmh1cukG2/',
'only_matching': True, 'only_matching': True,
@ -367,6 +396,15 @@ class InstagramIE(InstagramBaseIE):
video_id, url = self._match_valid_url(url).group('id', 'url') video_id, url = self._match_valid_url(url).group('id', 'url')
media, webpage = {}, '' media, webpage = {}, ''
if self._get_cookies(url).get('sessionid'):
info = traverse_obj(self._download_json(
f'{self._API_BASE_URL}/media/{_id_to_pk(video_id)}/info/', video_id,
fatal=False, errnote='Video info extraction failed',
note='Downloading video info', headers=self._API_HEADERS), ('items', 0))
if info:
media.update(info)
return self._extract_product(media)
api_check = self._download_json( api_check = self._download_json(
f'{self._API_BASE_URL}/web/get_ruling_for_content/?content_type=MEDIA&target_id={_id_to_pk(video_id)}', f'{self._API_BASE_URL}/web/get_ruling_for_content/?content_type=MEDIA&target_id={_id_to_pk(video_id)}',
video_id, headers=self._API_HEADERS, fatal=False, note='Setting up session', errnote=False) or {} video_id, headers=self._API_HEADERS, fatal=False, note='Setting up session', errnote=False) or {}
@ -374,40 +412,32 @@ class InstagramIE(InstagramBaseIE):
if not csrf_token: if not csrf_token:
self.report_warning('No csrf token set by Instagram API', video_id) self.report_warning('No csrf token set by Instagram API', video_id)
elif api_check.get('status') != 'ok':
self.report_warning('Instagram API is not granting access', video_id)
else: else:
if self._get_cookies(url).get('sessionid'): csrf_token = csrf_token.value if api_check.get('status') == 'ok' else None
media.update(traverse_obj(self._download_json( if not csrf_token:
f'{self._API_BASE_URL}/media/{_id_to_pk(video_id)}/info/', video_id, self.report_warning('Instagram API is not granting access', video_id)
fatal=False, note='Downloading video info', headers={
**self._API_HEADERS,
'X-CSRFToken': csrf_token.value,
}), ('items', 0)) or {})
if media:
return self._extract_product(media)
variables = { variables = {
'shortcode': video_id, 'shortcode': video_id,
'child_comment_count': 3, 'child_comment_count': 3,
'fetch_comment_count': 40, 'fetch_comment_count': 40,
'parent_comment_count': 24, 'parent_comment_count': 24,
'has_threaded_comments': True, 'has_threaded_comments': True,
} }
general_info = self._download_json( general_info = self._download_json(
'https://www.instagram.com/graphql/query/', video_id, fatal=False, 'https://www.instagram.com/graphql/query/', video_id, fatal=False, errnote=False,
headers={ headers={
**self._API_HEADERS, **self._API_HEADERS,
'X-CSRFToken': csrf_token.value, 'X-CSRFToken': csrf_token or '',
'X-Requested-With': 'XMLHttpRequest', 'X-Requested-With': 'XMLHttpRequest',
'Referer': url, 'Referer': url,
}, query={ }, query={
'query_hash': '9f8827793ef34641b2fb195d4d41151c', 'query_hash': '9f8827793ef34641b2fb195d4d41151c',
'variables': json.dumps(variables, separators=(',', ':')), 'variables': json.dumps(variables, separators=(',', ':')),
}) })
media.update(traverse_obj(general_info, ('data', 'shortcode_media')) or {}) media.update(traverse_obj(general_info, ('data', 'shortcode_media')) or {})
if not media: if not general_info:
self.report_warning('General metadata extraction failed (some metadata might be missing).', video_id) self.report_warning('General metadata extraction failed (some metadata might be missing).', video_id)
webpage, urlh = self._download_webpage_handle(url, video_id) webpage, urlh = self._download_webpage_handle(url, video_id)
shared_data = self._search_json( shared_data = self._search_json(
@ -418,12 +448,12 @@ class InstagramIE(InstagramBaseIE):
shared_data, ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'), shared_data, ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
('entry_data', 'PostPage', 0, 'media'), expected_type=dict) or {}) ('entry_data', 'PostPage', 0, 'media'), expected_type=dict) or {})
else: else:
self.report_warning('Main webpage is locked behind the login page. Retrying with embed webpage') self.report_warning('Main webpage is locked behind the login page. Retrying with embed webpage (some metadata might be missing).')
webpage = self._download_webpage( webpage = self._download_webpage(
f'{url}/embed/', video_id, note='Downloading embed webpage', fatal=False) f'{url}/embed/', video_id, note='Downloading embed webpage', fatal=False)
additional_data = self._search_json( additional_data = self._search_json(
r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*', webpage, 'additional data', video_id, fatal=False) r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*', webpage, 'additional data', video_id, fatal=False)
if not additional_data: if not additional_data and not media:
self.raise_login_required('Requested content is not available, rate-limit reached or login required') self.raise_login_required('Requested content is not available, rate-limit reached or login required')
product_item = traverse_obj(additional_data, ('items', 0), expected_type=dict) product_item = traverse_obj(additional_data, ('items', 0), expected_type=dict)