From feee67ae881b623c9fe4e35d9a6c7879c98bab19 Mon Sep 17 00:00:00 2001 From: nixxo Date: Sat, 6 Mar 2021 19:10:32 +0100 Subject: [PATCH] [gedi] Improvements from youtube-dl (#149) Authored-by: nixxo --- yt_dlp/extractor/extractors.py | 5 +- yt_dlp/extractor/gedi.py | 266 -------------------------------- yt_dlp/extractor/gedidigital.py | 210 +++++++++++++++++++++++++ yt_dlp/extractor/generic.py | 8 +- 4 files changed, 215 insertions(+), 274 deletions(-) delete mode 100644 yt_dlp/extractor/gedi.py create mode 100644 yt_dlp/extractor/gedidigital.py diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index 88dc2f1bf..580838d0f 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -450,10 +450,7 @@ from .gamestar import GameStarIE from .gaskrank import GaskrankIE from .gazeta import GazetaIE from .gdcvault import GDCVaultIE -from .gedi import ( - GediIE, - GediEmbedsIE, -) +from .gedidigital import GediDigitalIE from .generic import GenericIE from .gfycat import GfycatIE from .giantbomb import GiantBombIE diff --git a/yt_dlp/extractor/gedi.py b/yt_dlp/extractor/gedi.py deleted file mode 100644 index 9d9d4acc2..000000000 --- a/yt_dlp/extractor/gedi.py +++ /dev/null @@ -1,266 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..compat import compat_str -from ..utils import ( - base_url, - url_basename, - urljoin, -) - - -class GediBaseIE(InfoExtractor): - @staticmethod - def _clean_audio_fmts(formats): - unique_formats = [] - for f in formats: - if 'acodec' in f: - unique_formats.append(f) - formats[:] = unique_formats - - def _real_extract(self, url): - video_id = self._match_id(url) - - webpage = self._download_webpage(url, video_id) - player_data = re.findall( - r'PlayerFactory\.setParam\(\'(?P.+?)\',\s*\'(?P.+?)\',\s*\'(?P.+?)\'\);', - webpage) - - formats = [] - audio_fmts = [] - hls_fmts = [] - http_fmts = [] - title = '' - thumb = '' - - fmt_reg = r'(?Pvideo|audio)-(?P

rrtv|hls)-(?P[\w\d]+)(?:-(?P
[\w\d]+))?$' - br_reg = r'video-rrtv-(?P
\d+)-' - - for t, n, v in player_data: - if t == 'format': - m = re.match(fmt_reg, n) - if m: - # audio formats - if m.group('t') == 'audio': - if m.group('p') == 'hls': - audio_fmts.extend(self._extract_m3u8_formats( - v, video_id, 'm4a', m3u8_id='hls', fatal=False)) - elif m.group('p') == 'rrtv': - audio_fmts.append({ - 'format_id': 'mp3', - 'url': v, - 'tbr': 128, - 'ext': 'mp3', - 'vcodec': 'none', - 'acodec': 'mp3', - }) - - # video formats - elif m.group('t') == 'video': - # hls manifest video - if m.group('p') == 'hls': - hls_fmts.extend(self._extract_m3u8_formats( - v, video_id, 'mp4', m3u8_id='hls', fatal=False)) - # direct mp4 video - elif m.group('p') == 'rrtv': - if not m.group('br'): - mm = re.search(br_reg, v) - http_fmts.append({ - 'format_id': 'https-' + m.group('h'), - 'protocol': 'https', - 'url': v, - 'tbr': int(m.group('br')) if m.group('br') else - (int(mm.group('br')) if mm.group('br') else 0), - 'height': int(m.group('h')) - }) - - elif t == 'param': - if n == 'videotitle': - title = v - if n == 'image_full_play': - thumb = v - - title = self._og_search_title(webpage) if title == '' else title - - # clean weird char - title = compat_str(title).encode('utf8', 'replace').replace(b'\xc3\x82', b'').decode('utf8', 'replace') - - if audio_fmts: - self._clean_audio_fmts(audio_fmts) - self._sort_formats(audio_fmts) - if hls_fmts: - self._sort_formats(hls_fmts) - if http_fmts: - self._sort_formats(http_fmts) - - formats.extend(audio_fmts) - formats.extend(hls_fmts) - formats.extend(http_fmts) - - return { - 'id': video_id, - 'title': title, - 'description': self._html_search_meta('twitter:description', webpage), - 'thumbnail': thumb, - 'formats': formats, - } - - -class GediIE(GediBaseIE): - _VALID_URL = r'''(?x)https?://video\. - (?: - (?:espresso\.)?repubblica - |lastampa - |huffingtonpost - |ilsecoloxix - |iltirreno - |messaggeroveneto - |ilpiccolo - |gazzettadimantova - |mattinopadova - |laprovinciapavese - |tribunatreviso - |nuovavenezia - |gazzettadimodena - |lanuovaferrara - |corrierealpi - |lasentinella - ) - (?:\.gelocal)?\.it/(?!embed/).+?/(?P[\d/]+)(?:\?|\&|$)''' - _TESTS = [{ - 'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683', - 'md5': '84658d7fb9e55a6e57ecc77b73137494', - 'info_dict': { - 'id': '121559/121683', - 'ext': 'mp4', - 'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso', - 'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca', - 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', - }, - }, { - 'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963', - 'md5': 'e763b94b7920799a0e0e23ffefa2d157', - 'info_dict': { - 'id': '367415/367963', - 'ext': 'mp4', - 'title': 'Record della pista a Spa Francorchamps, la Pagani Huayra Roadster BC stupisce', - 'description': 'md5:5deb503cefe734a3eb3f07ed74303920', - 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', - }, - }, { - 'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267', - 'md5': 'e48108e97b1af137d22a8469f2019057', - 'info_dict': { - 'id': '66184/66267', - 'ext': 'mp4', - 'title': 'Cassani e i brividi azzurri ai Mondiali di Imola: \\"Qui mi sono innamorato del ciclismo da ragazzino, incredibile tornarci da ct\\"', - 'description': 'md5:fc9c50894f70a2469bb9b54d3d0a3d3b', - 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', - }, - }, { - 'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723', - 'md5': 'a6e39f3bdc1842bbd92abbbbef230817', - 'info_dict': { - 'id': '141059/142723', - 'ext': 'mp4', - 'title': 'Dentro la notizia - Ferrari, cosa succede a Maranello', - 'description': 'md5:9907d65b53765681fa3a0b3122617c1f', - 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', - }, - }] - - -class GediEmbedsIE(GediBaseIE): - _VALID_URL = r'''(?x)https?://video\. - (?: - (?:espresso\.)?repubblica - |lastampa - |huffingtonpost - |ilsecoloxix - |iltirreno - |messaggeroveneto - |ilpiccolo - |gazzettadimantova - |mattinopadova - |laprovinciapavese - |tribunatreviso - |nuovavenezia - |gazzettadimodena - |lanuovaferrara - |corrierealpi - |lasentinella - ) - (?:\.gelocal)?\.it/embed/.+?/(?P[\d/]+)(?:\?|\&|$)''' - _TESTS = [{ - 'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700', - 'md5': 'f4ac23cadfea7fef89bea536583fa7ed', - 'info_dict': { - 'id': '29312/29276', - 'ext': 'mp4', - 'title': 'Cotticelli: \\"Non so cosa mi sia successo. Sto cercando di capire se ho avuto un malore\\"', - 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e', - 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', - }, - }, { - 'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360', - 'md5': '0391c2c83c6506581003aaf0255889c0', - 'info_dict': { - 'id': '14772/14870', - 'ext': 'mp4', - 'title': 'Festival EMERGENCY, Villa: «La buona informazione aiuta la salute» (14772-14870)', - 'description': 'md5:2bce954d278248f3c950be355b7c2226', - 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-social-play\.jpg$', - }, - }] - - @staticmethod - def _sanitize_urls(urls): - # add protocol if missing - for i, e in enumerate(urls): - if e.startswith('//'): - urls[i] = 'https:%s' % e - # clean iframes urls - for i, e in enumerate(urls): - urls[i] = urljoin(base_url(e), url_basename(e)) - return urls - - @staticmethod - def _extract_urls(webpage): - entries = [ - mobj.group('url') - for mobj in re.finditer(r'''(?x) - (?: - data-frame-src=| - https?://video\. - (?: - (?:espresso\.)?repubblica - |lastampa - |huffingtonpost - |ilsecoloxix - |iltirreno - |messaggeroveneto - |ilpiccolo - |gazzettadimantova - |mattinopadova - |laprovinciapavese - |tribunatreviso - |nuovavenezia - |gazzettadimodena - |lanuovaferrara - |corrierealpi - |lasentinella - ) - (?:\.gelocal)?\.it/embed/.+?) - \1''', webpage)] - return GediEmbedsIE._sanitize_urls(entries) - - @staticmethod - def _extract_url(webpage): - urls = GediEmbedsIE._extract_urls(webpage) - return urls[0] if urls else None diff --git a/yt_dlp/extractor/gedidigital.py b/yt_dlp/extractor/gedidigital.py new file mode 100644 index 000000000..ef9e03e67 --- /dev/null +++ b/yt_dlp/extractor/gedidigital.py @@ -0,0 +1,210 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + base_url, + determine_ext, + int_or_none, + url_basename, + urljoin, +) + + +class GediDigitalIE(InfoExtractor): + _VALID_URL = r'''(?x)(?P(?:https?:)//video\. + (?: + (?: + (?:espresso\.)?repubblica + |lastampa + |ilsecoloxix + |huffingtonpost + )| + (?: + iltirreno + |messaggeroveneto + |ilpiccolo + |gazzettadimantova + |mattinopadova + |laprovinciapavese + |tribunatreviso + |nuovavenezia + |gazzettadimodena + |lanuovaferrara + |corrierealpi + |lasentinella + )\.gelocal + )\.it(?:/[^/]+){2,4}/(?P\d+))(?:$|[?&].*)''' + _TESTS = [{ + 'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683', + 'md5': '84658d7fb9e55a6e57ecc77b73137494', + 'info_dict': { + 'id': '121683', + 'ext': 'mp4', + 'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso', + 'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca', + 'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-full-.+?\.jpg$', + 'duration': 125, + }, + }, { + 'url': 'https://video.huffingtonpost.it/embed/politica/cotticelli-non-so-cosa-mi-sia-successo-sto-cercando-di-capire-se-ho-avuto-un-malore/29312/29276?responsive=true&el=video971040871621586700', + 'only_matching': True, + }, { + 'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360', + 'only_matching': True, + }, { + 'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963', + 'only_matching': True, + }, { + 'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267', + 'only_matching': True, + }, { + 'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723', + 'only_matching': True, + }, { + 'url': 'https://video.messaggeroveneto.gelocal.it/locale/maria-giovanna-elmi-covid-vaccino/138155/139268', + 'only_matching': True, + }, { + 'url': 'https://video.ilpiccolo.gelocal.it/dossier/big-john/dinosauro-big-john-al-via-le-visite-guidate-a-trieste/135226/135751', + 'only_matching': True, + }, { + 'url': 'https://video.gazzettadimantova.gelocal.it/locale/dal-ponte-visconteo-di-valeggio-l-and-8217sos-dei-ristoratori-aprire-anche-a-cena/137310/137818', + 'only_matching': True, + }, { + 'url': 'https://video.mattinopadova.gelocal.it/dossier/coronavirus-in-veneto/covid-a-vo-un-anno-dopo-un-cuore-tricolore-per-non-dimenticare/138402/138964', + 'only_matching': True, + }, { + 'url': 'https://video.laprovinciapavese.gelocal.it/locale/mede-zona-rossa-via-alle-vaccinazioni-per-gli-over-80/137545/138120', + 'only_matching': True, + }, { + 'url': 'https://video.tribunatreviso.gelocal.it/dossier/coronavirus-in-veneto/ecco-le-prima-vaccinazioni-di-massa-nella-marca/134485/135024', + 'only_matching': True, + }, { + 'url': 'https://video.nuovavenezia.gelocal.it/locale/camion-troppo-alto-per-il-ponte-ferroviario-perde-il-carico/135734/136266', + 'only_matching': True, + }, { + 'url': 'https://video.gazzettadimodena.gelocal.it/locale/modena-scoperta-la-proteina-che-predice-il-livello-di-gravita-del-covid/139109/139796', + 'only_matching': True, + }, { + 'url': 'https://video.lanuovaferrara.gelocal.it/locale/due-bombole-di-gpl-aperte-e-abbandonate-i-vigili-bruciano-il-gas/134391/134957', + 'only_matching': True, + }, { + 'url': 'https://video.corrierealpi.gelocal.it/dossier/cortina-2021-i-mondiali-di-sci-alpino/mondiali-di-sci-il-timelapse-sulla-splendida-olympia/133760/134331', + 'only_matching': True, + }, { + 'url': 'https://video.lasentinella.gelocal.it/locale/vestigne-centra-un-auto-e-si-ribalta/138931/139466', + 'only_matching': True, + }, { + 'url': 'https://video.espresso.repubblica.it/tutti-i-video/01-ted-villa/14772', + 'only_matching': True, + }] + + @staticmethod + def _sanitize_urls(urls): + # add protocol if missing + for i, e in enumerate(urls): + if e.startswith('//'): + urls[i] = 'https:%s' % e + # clean iframes urls + for i, e in enumerate(urls): + urls[i] = urljoin(base_url(e), url_basename(e)) + return urls + + @staticmethod + def _extract_urls(webpage): + entries = [ + mobj.group('eurl') + for mobj in re.finditer(r'''(?x) + (?: + data-frame-src=| + %s)\1''' % GediDigitalIE._VALID_URL, webpage)] + return GediDigitalIE._sanitize_urls(entries) + + @staticmethod + def _extract_url(webpage): + urls = GediDigitalIE._extract_urls(webpage) + return urls[0] if urls else None + + @staticmethod + def _clean_formats(formats): + format_urls = set() + clean_formats = [] + for f in formats: + if f['url'] not in format_urls: + if f.get('audio_ext') != 'none' and not f.get('acodec'): + continue + format_urls.add(f['url']) + clean_formats.append(f) + formats[:] = clean_formats + + def _real_extract(self, url): + video_id = self._match_id(url) + url = re.match(self._VALID_URL, url).group('url') + webpage = self._download_webpage(url, video_id) + title = self._html_search_meta( + ['twitter:title', 'og:title'], webpage, fatal=True) + player_data = re.findall( + r"PlayerFactory\.setParam\('(?Pformat|param)',\s*'(?P[^']+)',\s*'(?P[^']+)'\);", + webpage) + + formats = [] + duration = thumb = None + for t, n, v in player_data: + if t == 'format': + if n in ('video-hds-vod-ec', 'video-hls-vod-ec', 'video-viralize', 'video-youtube-pfp'): + continue + elif n.endswith('-vod-ak'): + formats.extend(self._extract_akamai_formats( + v, video_id, {'http': 'media.gedidigital.it'})) + else: + ext = determine_ext(v) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + v, video_id, 'mp4', 'm3u8_native', m3u8_id=n, fatal=False)) + continue + f = { + 'format_id': n, + 'url': v, + } + if ext == 'mp3': + abr = int_or_none(self._search_regex( + r'-mp3-audio-(\d+)', v, 'abr', default=None)) + f.update({ + 'abr': abr, + 'tbr': abr, + 'acodec': ext, + 'vcodec': 'none' + }) + else: + mobj = re.match(r'^video-rrtv-(\d+)(?:-(\d+))?$', n) + if mobj: + f.update({ + 'height': int(mobj.group(1)), + 'vbr': int_or_none(mobj.group(2)), + }) + if not f.get('vbr'): + f['vbr'] = int_or_none(self._search_regex( + r'-video-rrtv-(\d+)', v, 'abr', default=None)) + formats.append(f) + elif t == 'param': + if n in ['image_full', 'image']: + thumb = v + elif n == 'videoDuration': + duration = int_or_none(v) + + self._clean_formats(formats) + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': self._html_search_meta( + ['twitter:description', 'og:description', 'description'], webpage), + 'thumbnail': thumb or self._og_search_thumbnail(webpage), + 'formats': formats, + 'duration': duration, + } diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index 7ecc5b3bd..a3a8f6b0d 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -127,7 +127,7 @@ from .expressen import ExpressenIE from .zype import ZypeIE from .odnoklassniki import OdnoklassnikiIE from .kinja import KinjaEmbedIE -from .gedi import GediEmbedsIE +from .gedidigital import GediDigitalIE from .rcs import RCSEmbedsIE from .bitchute import BitChuteIE from .rumble import RumbleEmbedIE @@ -3339,12 +3339,12 @@ class GenericIE(InfoExtractor): return self.playlist_from_matches( zype_urls, video_id, video_title, ie=ZypeIE.ie_key()) - # Look for RCS media group embeds - gedi_urls = GediEmbedsIE._extract_urls(webpage) + gedi_urls = GediDigitalIE._extract_urls(webpage) if gedi_urls: return self.playlist_from_matches( - gedi_urls, video_id, video_title, ie=GediEmbedsIE.ie_key()) + gedi_urls, video_id, video_title, ie=GediDigitalIE.ie_key()) + # Look for RCS media group embeds rcs_urls = RCSEmbedsIE._extract_urls(webpage) if rcs_urls: return self.playlist_from_matches(