[puhutv] Improve extraction (closes #16269)
This commit is contained in:
Sergey M․
parent
6de82b4476
commit
8fd2a7be37
1 changed files with 164 additions and 149 deletions
|
|
@ -2,53 +2,54 @@
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_str
|
from ..compat import (
|
||||||
|
compat_HTTPError,
|
||||||
|
compat_str,
|
||||||
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
float_or_none,
|
float_or_none,
|
||||||
determine_ext,
|
parse_resolution,
|
||||||
str_or_none,
|
str_or_none,
|
||||||
url_or_none,
|
|
||||||
unified_strdate,
|
|
||||||
unified_timestamp,
|
|
||||||
try_get,
|
try_get,
|
||||||
url_basename,
|
unified_timestamp,
|
||||||
remove_end
|
url_or_none,
|
||||||
|
urljoin,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class PuhuTVIE(InfoExtractor):
|
class PuhuTVIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[a-z0-9-]+)-izle'
|
_VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-izle'
|
||||||
IE_NAME = 'puhutv'
|
IE_NAME = 'puhutv'
|
||||||
_TESTS = [
|
_TESTS = [{
|
||||||
{
|
# film
|
||||||
# A Film
|
'url': 'https://puhutv.com/sut-kardesler-izle',
|
||||||
'url': 'https://puhutv.com/sut-kardesler-izle',
|
'md5': 'fbd8f2d8e7681f8bcd51b592475a6ae7',
|
||||||
'md5': 'a347470371d56e1585d1b2c8dab01c96',
|
'info_dict': {
|
||||||
'info_dict': {
|
'id': '5085',
|
||||||
'id': 'sut-kardesler',
|
'display_id': 'sut-kardesler',
|
||||||
'display_id': '5085',
|
'ext': 'mp4',
|
||||||
'ext': 'mp4',
|
'title': 'Süt Kardeşler',
|
||||||
'title': 'Süt Kardeşler',
|
'description': 'md5:405fd024df916ca16731114eb18e511a',
|
||||||
'thumbnail': r're:^https?://.*\.jpg$',
|
'thumbnail': r're:^https?://.*\.jpg$',
|
||||||
'uploader': 'Arzu Film',
|
'duration': 4832.44,
|
||||||
'description': 'md5:405fd024df916ca16731114eb18e511a',
|
'creator': 'Arzu Film',
|
||||||
'uploader_id': '43',
|
'timestamp': 1469778212,
|
||||||
'upload_date': '20160729',
|
'upload_date': '20160729',
|
||||||
'timestamp': int,
|
'release_year': 1976,
|
||||||
},
|
'view_count': int,
|
||||||
|
'tags': ['Aile', 'Komedi', 'Klasikler'],
|
||||||
},
|
},
|
||||||
{
|
}, {
|
||||||
# An Episode and geo restricted
|
# episode, geo restricted, bypassable with --geo-verification-proxy
|
||||||
'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
|
'url': 'https://puhutv.com/jet-sosyete-1-bolum-izle',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
},
|
}, {
|
||||||
{
|
# 4k, with subtitles
|
||||||
# Has subtitle
|
'url': 'https://puhutv.com/dip-1-bolum-izle',
|
||||||
'url': 'https://puhutv.com/dip-1-bolum-izle',
|
'only_matching': True,
|
||||||
'only_matching': True,
|
}]
|
||||||
}
|
|
||||||
]
|
|
||||||
_SUBTITLE_LANGS = {
|
_SUBTITLE_LANGS = {
|
||||||
'English': 'en',
|
'English': 'en',
|
||||||
'Deutsch': 'de',
|
'Deutsch': 'de',
|
||||||
|
|
@ -56,47 +57,103 @@ class PuhuTVIE(InfoExtractor):
|
||||||
}
|
}
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
info = self._download_json(
|
|
||||||
'https://puhutv.com/api/slug/%s-izle' % video_id, video_id)['data']
|
|
||||||
|
|
||||||
display_id = compat_str(info['id'])
|
info = self._download_json(
|
||||||
title = info['title']['name']
|
urljoin(url, '/api/slug/%s-izle' % display_id),
|
||||||
|
display_id)['data']
|
||||||
|
|
||||||
|
video_id = compat_str(info['id'])
|
||||||
|
title = info.get('name') or info['title']['name']
|
||||||
if info.get('display_name'):
|
if info.get('display_name'):
|
||||||
title = '%s %s' % (title, info.get('display_name'))
|
title = '%s %s' % (title, info.get('display_name'))
|
||||||
|
|
||||||
description = try_get(info, lambda x: x['title']['description'], compat_str) or info.get('description')
|
try:
|
||||||
|
videos = self._download_json(
|
||||||
|
'https://puhutv.com/api/assets/%s/videos' % video_id,
|
||||||
|
display_id, 'Downloading video JSON',
|
||||||
|
headers=self.geo_verification_headers())
|
||||||
|
except ExtractorError as e:
|
||||||
|
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
|
||||||
|
self.raise_geo_restricted()
|
||||||
|
raise
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
for video in videos['data']['videos']:
|
||||||
|
media_url = url_or_none(video.get('url'))
|
||||||
|
if not media_url:
|
||||||
|
continue
|
||||||
|
playlist = video.get('is_playlist')
|
||||||
|
if video.get('stream_type') == 'hls' and playlist is True:
|
||||||
|
formats.extend(self._extract_m3u8_formats(
|
||||||
|
media_url, video_id, 'mp4', entry_protocol='m3u8_native',
|
||||||
|
m3u8_id='hls', fatal=False))
|
||||||
|
continue
|
||||||
|
quality = int_or_none(video.get('quality'))
|
||||||
|
f = {
|
||||||
|
'url': media_url,
|
||||||
|
'ext': 'mp4',
|
||||||
|
'height': quality
|
||||||
|
}
|
||||||
|
video_format = video.get('video_format')
|
||||||
|
if video_format == 'hls' and playlist is False:
|
||||||
|
format_id = 'hls'
|
||||||
|
f['protocol'] = 'm3u8_native'
|
||||||
|
elif video_format == 'mp4':
|
||||||
|
format_id = 'http'
|
||||||
|
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
if quality:
|
||||||
|
format_id += '-%sp' % quality
|
||||||
|
f['format_id'] = format_id
|
||||||
|
formats.append(f)
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
description = try_get(
|
||||||
|
info, lambda x: x['title']['description'],
|
||||||
|
compat_str) or info.get('description')
|
||||||
timestamp = unified_timestamp(info.get('created_at'))
|
timestamp = unified_timestamp(info.get('created_at'))
|
||||||
upload_date = unified_strdate(info.get('created_at'))
|
creator = try_get(
|
||||||
uploader = try_get(info, lambda x: x['title']['producer']['name'], compat_str)
|
info, lambda x: x['title']['producer']['name'], compat_str)
|
||||||
uploader_id = str_or_none(try_get(info, lambda x: x['title']['producer']['id']))
|
|
||||||
view_count = int_or_none(try_get(info, lambda x: x['content']['watch_count']))
|
duration = float_or_none(
|
||||||
duration = float_or_none(try_get(info, lambda x: x['content']['duration_in_ms']), scale=1000)
|
try_get(info, lambda x: x['content']['duration_in_ms'], int),
|
||||||
thumbnail = try_get(info, lambda x: x['content']['images']['wide']['main'], compat_str)
|
scale=1000)
|
||||||
release_year = int_or_none(try_get(info, lambda x: x['title']['released_at']))
|
view_count = try_get(info, lambda x: x['content']['watch_count'], int)
|
||||||
webpage_url = info.get('web_url')
|
|
||||||
|
images = try_get(
|
||||||
|
info, lambda x: x['content']['images']['wide'], dict) or {}
|
||||||
|
thumbnails = []
|
||||||
|
for image_id, image_url in images.items():
|
||||||
|
if not isinstance(image_url, compat_str):
|
||||||
|
continue
|
||||||
|
if not image_url.startswith(('http', '//')):
|
||||||
|
image_url = 'https://%s' % image_url
|
||||||
|
t = parse_resolution(image_id)
|
||||||
|
t.update({
|
||||||
|
'id': image_id,
|
||||||
|
'url': image_url
|
||||||
|
})
|
||||||
|
thumbnails.append(t)
|
||||||
|
|
||||||
|
release_year = try_get(info, lambda x: x['title']['released_at'], int)
|
||||||
|
|
||||||
season_number = int_or_none(info.get('season_number'))
|
season_number = int_or_none(info.get('season_number'))
|
||||||
season_id = int_or_none(info.get('season_id'))
|
season_id = str_or_none(info.get('season_id'))
|
||||||
episode_number = int_or_none(info.get('episode_number'))
|
episode_number = int_or_none(info.get('episode_number'))
|
||||||
|
|
||||||
tags = []
|
tags = []
|
||||||
for tag in try_get(info, lambda x: x['title']['genres'], list) or []:
|
for genre in try_get(info, lambda x: x['title']['genres'], list) or []:
|
||||||
if isinstance(tag.get('name'), compat_str):
|
if not isinstance(genre, dict):
|
||||||
tags.append(tag.get('name'))
|
|
||||||
|
|
||||||
thumbnails = []
|
|
||||||
thumbs_dict = try_get(info, lambda x: x['content']['images']['wide'], dict) or {}
|
|
||||||
for id, url in thumbs_dict.items():
|
|
||||||
if not url_or_none(url):
|
|
||||||
continue
|
continue
|
||||||
thumbnails.append({
|
genre_name = genre.get('name')
|
||||||
'url': 'https://%s' % url,
|
if genre_name and isinstance(genre_name, compat_str):
|
||||||
'id': id
|
tags.append(genre_name)
|
||||||
})
|
|
||||||
|
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
for subtitle in try_get(info, lambda x: x['content']['subtitles'], list) or []:
|
for subtitle in try_get(
|
||||||
|
info, lambda x: x['content']['subtitles'], list) or []:
|
||||||
if not isinstance(subtitle, dict):
|
if not isinstance(subtitle, dict):
|
||||||
continue
|
continue
|
||||||
lang = subtitle.get('language')
|
lang = subtitle.get('language')
|
||||||
|
|
@ -107,30 +164,6 @@ class PuhuTVIE(InfoExtractor):
|
||||||
'url': sub_url
|
'url': sub_url
|
||||||
}]
|
}]
|
||||||
|
|
||||||
# Some of videos are geo restricted upon request copyright owner and returns 403
|
|
||||||
req_formats = self._download_json(
|
|
||||||
'https://puhutv.com/api/assets/%s/videos' % display_id,
|
|
||||||
video_id, 'Downloading video JSON')
|
|
||||||
|
|
||||||
formats = []
|
|
||||||
for format in req_formats['data']['videos']:
|
|
||||||
media_url = url_or_none(format.get('url'))
|
|
||||||
if not media_url:
|
|
||||||
continue
|
|
||||||
ext = format.get('video_format') or determine_ext(media_url)
|
|
||||||
quality = format.get('quality')
|
|
||||||
if format.get('stream_type') == 'hls' and format.get('is_playlist') is True:
|
|
||||||
m3u8_id = remove_end(url_basename(media_url), '.m3u8')
|
|
||||||
formats.append(self._m3u8_meta_format(media_url, ext, m3u8_id=m3u8_id))
|
|
||||||
elif ext == 'mp4' and format.get('is_playlist', False) is False:
|
|
||||||
formats.append({
|
|
||||||
'url': media_url,
|
|
||||||
'format_id': 'http-%s' % quality,
|
|
||||||
'ext': ext,
|
|
||||||
'height': quality
|
|
||||||
})
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
|
|
@ -140,93 +173,75 @@ class PuhuTVIE(InfoExtractor):
|
||||||
'season_number': season_number,
|
'season_number': season_number,
|
||||||
'episode_number': episode_number,
|
'episode_number': episode_number,
|
||||||
'release_year': release_year,
|
'release_year': release_year,
|
||||||
'upload_date': upload_date,
|
|
||||||
'timestamp': timestamp,
|
'timestamp': timestamp,
|
||||||
'uploader': uploader,
|
'creator': creator,
|
||||||
'uploader_id': uploader_id,
|
|
||||||
'view_count': view_count,
|
'view_count': view_count,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
'tags': tags,
|
'tags': tags,
|
||||||
'subtitles': subtitles,
|
'subtitles': subtitles,
|
||||||
'webpage_url': webpage_url,
|
|
||||||
'thumbnail': thumbnail,
|
|
||||||
'thumbnails': thumbnails,
|
'thumbnails': thumbnails,
|
||||||
'formats': formats
|
'formats': formats
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class PuhuTVSerieIE(InfoExtractor):
|
class PuhuTVSerieIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[a-z0-9-]+)-detay'
|
_VALID_URL = r'https?://(?:www\.)?puhutv\.com/(?P<id>[^/?#&]+)-detay'
|
||||||
IE_NAME = 'puhutv:serie'
|
IE_NAME = 'puhutv:serie'
|
||||||
_TESTS = [
|
_TESTS = [{
|
||||||
{
|
'url': 'https://puhutv.com/deniz-yildizi-detay',
|
||||||
'url': 'https://puhutv.com/deniz-yildizi-detay',
|
'info_dict': {
|
||||||
'info_dict': {
|
'title': 'Deniz Yıldızı',
|
||||||
'title': 'Deniz Yıldızı',
|
'id': 'deniz-yildizi',
|
||||||
'id': 'deniz-yildizi',
|
|
||||||
'uploader': 'Focus Film',
|
|
||||||
'uploader_id': 61,
|
|
||||||
},
|
|
||||||
'playlist_mincount': 234,
|
|
||||||
},
|
},
|
||||||
{
|
'playlist_mincount': 205,
|
||||||
# a film detail page which is using same url with serie page
|
}, {
|
||||||
'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
|
# a film detail page which is using same url with serie page
|
||||||
'info_dict': {
|
'url': 'https://puhutv.com/kaybedenler-kulubu-detay',
|
||||||
'title': 'Kaybedenler Kulübü',
|
'only_matching': True,
|
||||||
'id': 'kaybedenler-kulubu',
|
}]
|
||||||
'uploader': 'Tolga Örnek, Murat Dörtbudak, Neslihan Dörtbudak, Kemal Kaplanoğlu',
|
|
||||||
'uploader_id': 248,
|
|
||||||
},
|
|
||||||
'playlist_mincount': 1,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
def _extract_entries(self, playlist_id, seasons):
|
def _extract_entries(self, seasons):
|
||||||
for season in seasons:
|
for season in seasons:
|
||||||
season_id = season['id']
|
season_id = season.get('id')
|
||||||
season_number = season.get('position')
|
if not season_id:
|
||||||
pagenum = 1
|
continue
|
||||||
|
page = 1
|
||||||
has_more = True
|
has_more = True
|
||||||
while has_more is True:
|
while has_more is True:
|
||||||
season_info = self._download_json(
|
season = self._download_json(
|
||||||
'https://galadriel.puhutv.com/seasons/%s' % season_id,
|
'https://galadriel.puhutv.com/seasons/%s' % season_id,
|
||||||
playlist_id, 'Downloading season %s page %s' % (season_number, pagenum), query={
|
season_id, 'Downloading page %s' % page, query={
|
||||||
'page': pagenum,
|
'page': page,
|
||||||
'per': 40,
|
'per': 40,
|
||||||
})
|
})
|
||||||
for episode in season_info.get('episodes'):
|
episodes = season.get('episodes')
|
||||||
video_id = episode['slugPath'].replace('-izle', '')
|
if isinstance(episodes, list):
|
||||||
yield self.url_result(
|
for ep in episodes:
|
||||||
'https://puhutv.com/%s-izle' % video_id,
|
slug_path = str_or_none(ep.get('slugPath'))
|
||||||
PuhuTVIE.ie_key(), video_id)
|
if not slug_path:
|
||||||
pagenum = pagenum + 1
|
continue
|
||||||
has_more = season_info.get('hasMore', False)
|
video_id = str_or_none(int_or_none(ep.get('id')))
|
||||||
|
yield self.url_result(
|
||||||
|
'https://puhutv.com/%s' % slug_path,
|
||||||
|
ie=PuhuTVIE.ie_key(), video_id=video_id,
|
||||||
|
video_title=ep.get('name') or ep.get('eventLabel'))
|
||||||
|
page += 1
|
||||||
|
has_more = season.get('hasMore')
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
playlist_id = self._match_id(url)
|
playlist_id = self._match_id(url)
|
||||||
|
|
||||||
info = self._download_json(
|
info = self._download_json(
|
||||||
'https://puhutv.com/api/slug/%s-detay' % playlist_id, playlist_id)['data']
|
urljoin(url, '/api/slug/%s-detay' % playlist_id),
|
||||||
|
playlist_id)['data']
|
||||||
|
|
||||||
title = info.get('name')
|
|
||||||
uploader = try_get(info, lambda x: x['producer']['name'], compat_str)
|
|
||||||
uploader_id = try_get(info, lambda x: x['producer']['id'])
|
|
||||||
seasons = info.get('seasons')
|
seasons = info.get('seasons')
|
||||||
if seasons:
|
if seasons:
|
||||||
entries = self._extract_entries(playlist_id, seasons)
|
return self.playlist_result(
|
||||||
else:
|
self._extract_entries(seasons), playlist_id, info.get('name'))
|
||||||
# For films, these are using same url with series
|
|
||||||
video_id = info['assets'][0]['slug']
|
|
||||||
return self.url_result(
|
|
||||||
'https://puhutv.com/%s-izle' % video_id,
|
|
||||||
PuhuTVIE.ie_key(), video_id)
|
|
||||||
|
|
||||||
return {
|
# For films, these are using same url with series
|
||||||
'_type': 'playlist',
|
video_id = info.get('slug') or info['assets'][0]['slug']
|
||||||
'id': playlist_id,
|
return self.url_result(
|
||||||
'title': title,
|
'https://puhutv.com/%s-izle' % video_id,
|
||||||
'uploader': uploader,
|
PuhuTVIE.ie_key(), video_id)
|
||||||
'uploader_id': uploader_id,
|
|
||||||
'entries': entries,
|
|
||||||
}
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue