kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[YouTube] Use new browse API for continuation page extraction. (#131)

Browse source 
Known issues (these issues existed in previous API as well)
* Mix playlists only give 1 page (25 vids)
* Trending only gives 1 video
* History gives 5 pages (200 vids)

Co-authored-by: colethedj, pukkandan
This commit is contained in:
Matthew 2021-03-03 10:32:40 +00:00 committed by GitHub
parent c705177da2
commit a5c5623470
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 38 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
yt_dlp/extractor/youtube.py
View file

@ -2,6 +2,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import hashlib
import itertools import itertools
import json import json
import os.path import os.path
@ -274,7 +275,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
'context': { 'context': {
'client': { 'client': {
'clientName': 'WEB', 'clientName': 'WEB',
'clientVersion': '2.20201021.03.00', 'clientVersion': '2.20210301.08.00',
} }
}, },
} }
@ -283,15 +284,28 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
_YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;' _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
_YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)' _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
def _call_api(self, ep, query, video_id, fatal=True): def _generate_sapisidhash_header(self):
sapisid_cookie = self._get_cookies('https://www.youtube.com').get('SAPISID')
if sapisid_cookie is None:
return
time_now = round(time.time())
sapisidhash = hashlib.sha1((str(time_now) + " " + sapisid_cookie.value + " " + "https://www.youtube.com").encode("utf-8")).hexdigest()
return "SAPISIDHASH %s_%s" % (time_now, sapisidhash)
def _call_api(self, ep, query, video_id, fatal=True, headers=None,
note='Downloading API JSON', errnote='Unable to download API page'):
data = self._DEFAULT_API_DATA.copy() data = self._DEFAULT_API_DATA.copy()
data.update(query) data.update(query)
headers = headers or {}
headers.update({'content-type': 'application/json'})
auth = self._generate_sapisidhash_header()
if auth is not None:
headers.update({'Authorization': auth, 'X-Origin': 'https://www.youtube.com'})
return self._download_json( return self._download_json(
'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id, 'https://www.youtube.com/youtubei/v1/%s' % ep,
note='Downloading API JSON', errnote='Unable to download API page', video_id=video_id, fatal=fatal, note=note, errnote=errnote,
data=json.dumps(data).encode('utf8'), fatal=fatal, data=json.dumps(data).encode('utf8'), headers=headers,
headers={'content-type': 'application/json'},
query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'}) query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
def _extract_yt_initial_data(self, video_id, webpage): def _extract_yt_initial_data(self, video_id, webpage):
@ -2699,7 +2713,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
ctp = continuation_ep.get('clickTrackingParams') ctp = continuation_ep.get('clickTrackingParams')
return YoutubeTabIE._build_continuation_query(continuation, ctp) return YoutubeTabIE._build_continuation_query(continuation, ctp)
def _entries(self, tab, identity_token): def _entries(self, tab, identity_token, item_id):
def extract_entries(parent_renderer): # this needs to called again for continuation to work with feeds def extract_entries(parent_renderer): # this needs to called again for continuation to work with feeds
contents = try_get(parent_renderer, lambda x: x['contents'], list) or [] contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
@ -2770,11 +2784,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
if last_error: if last_error:
self.report_warning('%s. Retrying ...' % last_error) self.report_warning('%s. Retrying ...' % last_error)
try: try:
browse = self._download_json( response = self._call_api(
'https://www.youtube.com/browse_ajax', None, ep="browse", fatal=True, headers=headers,
'Downloading page %d%s' video_id='%s page %s' % (item_id, page_num),
% (page_num, ' (retry #%d)' % count if count else ''), query={
headers=headers, query=continuation) 'continuation': continuation['continuation'],
'clickTracking': {'clickTrackingParams': continuation['itct']},
},
note='Downloading API JSON%s' % (' (retry #%d)' % count if count else ''))
except ExtractorError as e: except ExtractorError as e:
if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503, 404): if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503, 404):
# Downloading page may result in intermittent 5xx HTTP error # Downloading page may result in intermittent 5xx HTTP error
@ -2784,8 +2801,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
continue continue
raise raise
else: else:
response = try_get(browse, lambda x: x[1]['response'], dict)
# Youtube sometimes sends incomplete data # Youtube sometimes sends incomplete data
# See: https://github.com/ytdl-org/youtube-dl/issues/28194 # See: https://github.com/ytdl-org/youtube-dl/issues/28194
if response.get('continuationContents') or response.get('onResponseReceivedActions'): if response.get('continuationContents') or response.get('onResponseReceivedActions'):
@ -2793,7 +2808,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
last_error = 'Incomplete data recieved' last_error = 'Incomplete data recieved'
if count >= retries: if count >= retries:
self._downloader.report_error(last_error) self._downloader.report_error(last_error)
if not browse or not response:
if not response:
break break
known_continuation_renderers = { known_continuation_renderers = {
@ -2936,7 +2952,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
'channel_id': metadata['uploader_id'], 'channel_id': metadata['uploader_id'],
'channel_url': metadata['uploader_url']}) 'channel_url': metadata['uploader_url']})
return self.playlist_result( return self.playlist_result(
self._entries(selected_tab, identity_token), self._entries(selected_tab, identity_token, playlist_id),
**metadata) **metadata)
def _extract_from_playlist(self, item_id, url, data, playlist): def _extract_from_playlist(self, item_id, url, data, playlist):
@ -3223,26 +3239,14 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
_TESTS = [] _TESTS = []
def _entries(self, query, n): def _entries(self, query, n):
data = { data = {'query': query}
'context': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20201021.03.00',
}
},
'query': query,
}
if self._SEARCH_PARAMS: if self._SEARCH_PARAMS:
data['params'] = self._SEARCH_PARAMS data['params'] = self._SEARCH_PARAMS
total = 0 total = 0
for page_num in itertools.count(1): for page_num in itertools.count(1):
search = self._download_json( search = self._call_api(
'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', ep='search', video_id='query "%s"' % query, fatal=False,
video_id='query "%s"' % query, note='Downloading page %s' % page_num, query=data)
note='Downloading page %s' % page_num,
errnote='Unable to download API page', fatal=False,
data=json.dumps(data).encode('utf8'),
headers={'content-type': 'application/json'})
if not search: if not search:
break break
slr_contents = try_get( slr_contents = try_get(
@ -3394,8 +3398,8 @@ class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
class YoutubeHistoryIE(YoutubeFeedsInfoExtractor): class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)' IE_DESC = 'Youtube watch history, ":ythis" for short (requires authentication)'
_VALID_URL = r':ythistory' _VALID_URL = r':ythis(?:tory)?'
_FEED_NAME = 'history' _FEED_NAME = 'history'
_TESTS = [{ _TESTS = [{
'url': ':ythistory', 'url': ':ythistory',