kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[extractor/stacommu] Add extractors (#7432)

Browse source 
Authored by: urectanc
This commit is contained in:
urectanc 2023-07-01 03:27:07 +09:00 committed by GitHub
parent fcbc9ed760
commit af1fd12f67
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 173 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -1855,7 +1855,7 @@ The following extractors use this feature:
#### twitter
* `legacy_api`: Force usage of the legacy Twitter API instead of the GraphQL API for tweet extraction. Has no effect if login cookies are passed
#### wrestleuniverse
#### stacommu, wrestleuniverse
* `device_id`: UUID value assigned by the website and used to enforce device limits for paid livestream content. Can be found in browser local storage
#### twitch

4
yt_dlp/extractor/_extractors.py
View file

@ -1855,6 +1855,10 @@ from .srgssr import (
SRGSSRPlayIE,
)
from .srmediathek import SRMediathekIE
from .stacommu import (
StacommuLiveIE,
StacommuVODIE,
)
from .stanfordoc import StanfordOpenClassroomIE
from .startv import StarTVIE
from .steam import (

148
yt_dlp/extractor/stacommu.py Normal file
View file

@ -0,0 +1,148 @@
import time
from .wrestleuniverse import WrestleUniverseBaseIE
from ..utils import (
int_or_none,
traverse_obj,
url_or_none,
)
class StacommuBaseIE(WrestleUniverseBaseIE):
_NETRC_MACHINE = 'stacommu'
_API_HOST = 'api.stacommu.jp'
_LOGIN_QUERY = {'key': 'AIzaSyCR9czxhH2eWuijEhTNWBZ5MCcOYEUTAhg'}
_LOGIN_HEADERS = {
'Accept': '*/*',
'Content-Type': 'application/json',
'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
'Referer': 'https://www.stacommu.jp/',
'Origin': 'https://www.stacommu.jp',
}
@WrestleUniverseBaseIE._TOKEN.getter
def _TOKEN(self):
if self._REAL_TOKEN and self._TOKEN_EXPIRY <= int(time.time()):
self._refresh_token()
return self._REAL_TOKEN
def _get_formats(self, data, path, video_id=None):
if not traverse_obj(data, path) and not data.get('canWatch') and not self._TOKEN:
self.raise_login_required(method='password')
return super()._get_formats(data, path, video_id)
def _extract_hls_key(self, data, path, decrypt):
encryption_data = traverse_obj(data, path)
if traverse_obj(encryption_data, ('encryptType', {int})) == 0:
return None
return traverse_obj(encryption_data, {'key': ('key', {decrypt}), 'iv': ('iv', {decrypt})})
class StacommuVODIE(StacommuBaseIE):
_VALID_URL = r'https?://www\.stacommu\.jp/videos/episodes/(?P<id>[\da-zA-Z]+)'
_TESTS = [{
# not encrypted
'url': 'https://www.stacommu.jp/videos/episodes/aXcVKjHyAENEjard61soZZ',
'info_dict': {
'id': 'aXcVKjHyAENEjard61soZZ',
'ext': 'mp4',
'title': 'スタコミュAWARDの裏側、ほぼ全部見せます！〜晴れ舞台の直前ドキドキ編〜',
'description': 'md5:6400275c57ae75c06da36b06f96beb1c',
'timestamp': 1679652000,
'upload_date': '20230324',
'thumbnail': 'https://image.stacommu.jp/6eLobQan8PFtBoU4RL4uGg/6eLobQan8PFtBoU4RL4uGg',
'cast': 'count:11',
'duration': 250,
},
'params': {
'skip_download': 'm3u8',
},
}, {
# encrypted; requires a premium account
'url': 'https://www.stacommu.jp/videos/episodes/3hybMByUvzMEqndSeu5LpD',
'info_dict': {
'id': '3hybMByUvzMEqndSeu5LpD',
'ext': 'mp4',
'title': 'スタプラフェス2023〜裏側ほぼ全部見せます〜＃10',
'description': 'md5:85494488ccf1dfa1934accdeadd7b340',
'timestamp': 1682506800,
'upload_date': '20230426',
'thumbnail': 'https://image.stacommu.jp/eMdXtEefR4kEyJJMpAFi7x/eMdXtEefR4kEyJJMpAFi7x',
'cast': 'count:55',
'duration': 312,
'hls_aes': {
'key': '6bbaf241b8e1fd9f59ecf546a70e4ae7',
'iv': '1fc9002a23166c3bb1d240b953d09de9',
},
},
'params': {
'skip_download': 'm3u8',
},
}]
_API_PATH = 'videoEpisodes'
def _real_extract(self, url):
video_id = self._match_id(url)
video_info = self._download_metadata(
url, video_id, 'ja', ('dehydratedState', 'queries', 0, 'state', 'data'))
hls_info, decrypt = self._call_encrypted_api(
video_id, ':watch', 'stream information', data={'method': 1})
return {
'id': video_id,
'formats': self._get_formats(hls_info, ('protocolHls', 'url', {url_or_none}), video_id),
'hls_aes': self._extract_hls_key(hls_info, 'protocolHls', decrypt),
**traverse_obj(video_info, {
'title': ('displayName', {str}),
'description': ('description', {str}),
'timestamp': ('watchStartTime', {int_or_none}),
'thumbnail': ('keyVisualUrl', {url_or_none}),
'cast': ('casts', ..., 'displayName', {str}),
'duration': ('duration', {int}),
}),
}
class StacommuLiveIE(StacommuBaseIE):
_VALID_URL = r'https?://www\.stacommu\.jp/live/(?P<id>[\da-zA-Z]+)'
_TESTS = [{
'url': 'https://www.stacommu.jp/live/d2FJ3zLnndegZJCAEzGM3m',
'info_dict': {
'id': 'd2FJ3zLnndegZJCAEzGM3m',
'ext': 'mp4',
'title': '仲村悠菜 2023/05/04',
'timestamp': 1683195647,
'upload_date': '20230504',
'thumbnail': 'https://image.stacommu.jp/pHGF57SPEHE2ke83FS92FN/pHGF57SPEHE2ke83FS92FN',
'duration': 5322,
'hls_aes': {
'key': 'efbb3ec0b8246f61adf1764c5a51213a',
'iv': '80621d19a1f19167b64cedb415b05d1c',
},
},
'params': {
'skip_download': 'm3u8',
},
}]
_API_PATH = 'events'
def _real_extract(self, url):
video_id = self._match_id(url)
video_info = self._call_api(video_id, msg='video information', query={'al': 'ja'}, auth=False)
hls_info, decrypt = self._call_encrypted_api(
video_id, ':watchArchive', 'stream information', data={'method': 1})
return {
'id': video_id,
'formats': self._get_formats(hls_info, ('hls', 'urls', ..., {url_or_none}), video_id),
'hls_aes': self._extract_hls_key(hls_info, 'hls', decrypt),
**traverse_obj(video_info, {
'title': ('displayName', {str}),
'timestamp': ('startTime', {int_or_none}),
'thumbnail': ('keyVisualUrl', {url_or_none}),
'duration': ('duration', {int_or_none}),
}),
}

33
yt_dlp/extractor/wrestleuniverse.py
View file

@ -14,12 +14,14 @@ from ..utils import (
try_call,
url_or_none,
urlencode_postdata,
variadic,
)
class WrestleUniverseBaseIE(InfoExtractor):
_NETRC_MACHINE = 'wrestleuniverse'
_VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
_API_HOST = 'api.wrestle-universe.com'
_API_PATH = None
_REAL_TOKEN = None
_TOKEN_EXPIRY = None
@ -67,24 +69,28 @@ class WrestleUniverseBaseIE(InfoExtractor):
'returnSecureToken': True,
'email': username,
'password': password,
}, separators=(',', ':')).encode())
}, separators=(',', ':')).encode(), expected_status=400)
token = traverse_obj(login, ('idToken', {str}))
if not token:
raise ExtractorError(
f'Unable to log in: {traverse_obj(login, ("error", "message"))}', expected=True)
self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
if not self._REFRESH_TOKEN:
self.report_warning('No refresh token was granted')
self._TOKEN = traverse_obj(login, ('idToken', {str}))
self._TOKEN = token
def _real_initialize(self):
if WrestleUniverseBaseIE._DEVICE_ID:
if self._DEVICE_ID:
return
WrestleUniverseBaseIE._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key='WrestleUniverse')[0]
if not WrestleUniverseBaseIE._DEVICE_ID:
WrestleUniverseBaseIE._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
if WrestleUniverseBaseIE._DEVICE_ID:
self._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key=self._NETRC_MACHINE)[0]
if not self._DEVICE_ID:
self._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
if self._DEVICE_ID:
return
WrestleUniverseBaseIE._DEVICE_ID = str(uuid.uuid4())
self._DEVICE_ID = str(uuid.uuid4())
self.cache.store(self._NETRC_MACHINE, 'device_id', WrestleUniverseBaseIE._DEVICE_ID)
self.cache.store(self._NETRC_MACHINE, 'device_id', self._DEVICE_ID)
def _refresh_token(self):
refresh = self._download_json(
@ -108,10 +114,10 @@ class WrestleUniverseBaseIE(InfoExtractor):
if data:
headers['Content-Type'] = 'application/json;charset=utf-8'
data = json.dumps(data, separators=(',', ':')).encode()
if auth:
if auth and self._TOKEN:
headers['Authorization'] = f'Bearer {self._TOKEN}'
return self._download_json(
f'https://api.wrestle-universe.com/v1/{self._API_PATH}/{video_id}{param}', video_id,
f'https://{self._API_HOST}/v1/{self._API_PATH}/{video_id}{param}', video_id,
note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
data=data, headers=headers, query=query, fatal=fatal)
@ -137,12 +143,13 @@ class WrestleUniverseBaseIE(InfoExtractor):
}, query=query, fatal=fatal)
return api_json, decrypt
def _download_metadata(self, url, video_id, lang, props_key):
def _download_metadata(self, url, video_id, lang, props_keys):
metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
if not metadata:
webpage = self._download_webpage(url, video_id)
nextjs_data = self._search_nextjs_data(webpage, video_id)
metadata = traverse_obj(nextjs_data, ('props', 'pageProps', props_key, {dict})) or {}
metadata = traverse_obj(nextjs_data, (
'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
return metadata
def _get_formats(self, data, path, video_id=None):