[extractor/generic:quoted-html] Add extractor (#5213)
Extracts embeds from escaped HTML within `data-html` attribute. Related: https://github.com/ytdl-org/youtube-dl/issues/21294, https://github.com/yt-dlp/yt-dlp/pull/5121 Authored by: coletdjnz Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
This commit is contained in:
Matthew
GitHub
parent
6678a4f0b3
commit
6dca2aa66d
4 changed files with 89 additions and 87 deletions
|
|
@ -698,7 +698,10 @@ from .hse import (
|
|||
HSEShowIE,
|
||||
HSEProductIE,
|
||||
)
|
||||
from .genericembeds import HTML5MediaEmbedIE
|
||||
from .genericembeds import (
|
||||
HTML5MediaEmbedIE,
|
||||
QuotedHTMLIE,
|
||||
)
|
||||
from .huajiao import HuajiaoIE
|
||||
from .huya import HuyaLiveIE
|
||||
from .huffpost import HuffPostIE
|
||||
|
|
@ -1884,7 +1887,6 @@ from .tv2 import (
|
|||
)
|
||||
from .tv24ua import (
|
||||
TV24UAVideoIE,
|
||||
TV24UAGenericPassthroughIE
|
||||
)
|
||||
from .tv2dk import (
|
||||
TV2DKIE,
|
||||
|
|
|
|||
|
|
@ -1980,22 +1980,6 @@ class GenericIE(InfoExtractor):
|
|||
},
|
||||
'playlist_count': 6,
|
||||
},
|
||||
{
|
||||
# Squarespace video embed, 2019-08-28
|
||||
'url': 'http://ootboxford.com',
|
||||
'info_dict': {
|
||||
'id': 'Tc7b_JGdZfw',
|
||||
'title': 'Out of the Blue, at Childish Things 10',
|
||||
'ext': 'mp4',
|
||||
'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
|
||||
'uploader_id': 'helendouglashouse',
|
||||
'uploader': 'Helen & Douglas House',
|
||||
'upload_date': '20140328',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
},
|
||||
# {
|
||||
# # Zype embed
|
||||
# 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites',
|
||||
|
|
@ -2784,12 +2768,6 @@ class GenericIE(InfoExtractor):
|
|||
# There probably should be a second run of generic extractor on unescaped webpage.
|
||||
# webpage = urllib.parse.unquote(webpage)
|
||||
|
||||
# Unescape squarespace embeds to be detected by generic extractor,
|
||||
# see https://github.com/ytdl-org/youtube-dl/issues/21294
|
||||
webpage = re.sub(
|
||||
r'<div[^>]+class=[^>]*?\bsqs-video-wrapper\b[^>]*>',
|
||||
lambda x: unescapeHTML(x.group(0)), webpage)
|
||||
|
||||
# TODO: Move to respective extractors
|
||||
bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
|
||||
if bc_urls:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import re
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import make_archive_id
|
||||
from ..utils import make_archive_id, unescapeHTML
|
||||
|
||||
|
||||
class HTML5MediaEmbedIE(InfoExtractor):
|
||||
|
|
@ -29,3 +32,84 @@ class HTML5MediaEmbedIE(InfoExtractor):
|
|||
})
|
||||
self._sort_formats(entry['formats'])
|
||||
yield entry
|
||||
|
||||
|
||||
class QuotedHTMLIE(InfoExtractor):
|
||||
"""For common cases of quoted/escaped html parts in the webpage"""
|
||||
_VALID_URL = False
|
||||
IE_NAME = 'generic:quoted-html'
|
||||
IE_DESC = False # Do not list
|
||||
_WEBPAGE_TESTS = [{
|
||||
# 2 YouTube embeds in data-html
|
||||
'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
|
||||
'info_dict': {
|
||||
'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
|
||||
'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g',
|
||||
'timestamp': float,
|
||||
'upload_date': str,
|
||||
'description': 'md5:6816e1e5a65304bd7898e4c7eb1b26f7',
|
||||
'age_limit': 0,
|
||||
},
|
||||
'playlist_count': 2
|
||||
}, {
|
||||
# Generic iframe embed of TV24UAPlayerIE within data-html
|
||||
'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
|
||||
'info_dict': {
|
||||
'id': '1887584',
|
||||
'ext': 'mp4',
|
||||
'title': 'Харків\'яни згадують місто до війни: щемливе відео',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g',
|
||||
},
|
||||
'params': {'skip_download': True}
|
||||
}, {
|
||||
# YouTube embeds on Squarespace (data-html): https://github.com/ytdl-org/youtube-dl/issues/21294
|
||||
'url': 'https://www.harvardballetcompany.org/past-productions',
|
||||
'info_dict': {
|
||||
'id': 'past-productions',
|
||||
'title': 'Productions — Harvard Ballet Company',
|
||||
'age_limit': 0,
|
||||
'description': 'Past Productions',
|
||||
},
|
||||
'playlist_mincount': 26
|
||||
}, {
|
||||
# Squarespace video embed, 2019-08-28, data-html
|
||||
'url': 'http://ootboxford.com',
|
||||
'info_dict': {
|
||||
'id': 'Tc7b_JGdZfw',
|
||||
'title': 'Out of the Blue, at Childish Things 10',
|
||||
'ext': 'mp4',
|
||||
'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
|
||||
'uploader_id': 'helendouglashouse',
|
||||
'uploader': 'Helen & Douglas House',
|
||||
'upload_date': '20140328',
|
||||
'availability': 'public',
|
||||
'view_count': int,
|
||||
'channel': 'Helen & Douglas House',
|
||||
'comment_count': int,
|
||||
'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
|
||||
'duration': 253,
|
||||
'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
|
||||
'playable_in_embed': True,
|
||||
'age_limit': 0,
|
||||
'channel_follower_count': int,
|
||||
'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
|
||||
'tags': 'count:6',
|
||||
'categories': ['Nonprofits & Activism'],
|
||||
'like_count': int,
|
||||
'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
}]
|
||||
|
||||
def _extract_from_webpage(self, url, webpage):
|
||||
combined = ''
|
||||
for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage):
|
||||
# unescapeHTML can handle " etc., unquote can handle percent encoding
|
||||
unquoted_html = unescapeHTML(urllib.parse.unquote(html))
|
||||
if unquoted_html != html:
|
||||
combined += unquoted_html
|
||||
if combined:
|
||||
yield from self._extract_generic_embeds(url, combined)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,10 @@
|
|||
import base64
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
extract_attributes,
|
||||
get_elements_html_by_class,
|
||||
js_to_json,
|
||||
mimetype2ext,
|
||||
smuggle_url,
|
||||
traverse_obj,
|
||||
)
|
||||
|
||||
|
|
@ -87,60 +82,3 @@ class TV24UAVideoIE(InfoExtractor):
|
|||
'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
|
||||
'description': self._og_search_description(webpage, default=None),
|
||||
}
|
||||
|
||||
|
||||
class TV24UAGenericPassthroughIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:[a-zA-Z0-9]+?\.)?24tv\.ua/(?P<id>[^/]+?_n\d+)'
|
||||
|
||||
_TESTS = [{
|
||||
# Generic iframe, not within media_embed
|
||||
'url': 'https://24tv.ua/vipalyuyut-nashi-mista-sela-dsns-pokazali-motoroshni-naslidki_n1883966',
|
||||
'info_dict': {
|
||||
'id': '1883966',
|
||||
'ext': 'mp4',
|
||||
'title': 'Випалюють наші міста та села, – моторошні наслідки обстрілів на Чернігівщині',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g',
|
||||
}
|
||||
}, {
|
||||
# Generic iframe embed of TV24UAPlayerIE, within media_embed
|
||||
'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
|
||||
'info_dict': {
|
||||
'id': 'harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
|
||||
'title': 'Харків\'яни згадують місто до війни: щемливе відео'
|
||||
},
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '1887584',
|
||||
'ext': 'mp4',
|
||||
'title': 'Харків\'яни згадують місто до війни: щемливе відео',
|
||||
'thumbnail': r're:^https?://.*\.jpe?g',
|
||||
},
|
||||
}]
|
||||
}, {
|
||||
# 2 media_embeds with YouTube iframes
|
||||
'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
|
||||
'info_dict': {
|
||||
'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
|
||||
'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
|
||||
},
|
||||
'playlist_count': 2
|
||||
}, {
|
||||
'url': 'https://men.24tv.ua/fitnes-bloger-sprobuvav-vikonati-trenuvannya-naysilnishoyi-lyudini_n2164538',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
data_urls = []
|
||||
# The site contains escaped iframe embeds within an attribute.
|
||||
# Once escaped, generic can handle them, so we use a data url to pass the escaped html back.
|
||||
for html in get_elements_html_by_class('media_embed', webpage):
|
||||
data = urllib.parse.unquote(extract_attributes(html).get('data-html'))
|
||||
data_urls.append(f'data:text/html;base64,{base64.b64encode(data.encode("utf-8")).decode("utf-8")}')
|
||||
|
||||
if not data_urls:
|
||||
return self.url_result(url, 'Generic')
|
||||
return self.playlist_from_matches(
|
||||
[smuggle_url(url, {'to_generic': True}) for url in data_urls], display_id, ie='Generic',
|
||||
playlist_title=self._og_search_title(webpage) or self._html_extract_title(webpage))
|
||||
|
|
|
|||
Loading…
Reference in a new issue