kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add regex to --match-filter

Browse source 
This does not fully deprecate `--match-title`/`--reject-title`
since `--match-filter` is only checked after the extraction is complete,
while `--match-title` can often be checked from the flat playlist.

Fixes: https://github.com/ytdl-org/youtube-dl/issues/9092, https://github.com/ytdl-org/youtube-dl/issues/23035
This commit is contained in:
pukkandan 2021-08-05 03:01:23 +05:30
parent 77b87f0519
commit a047eeb6d2
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
4 changed files with 82 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
README.md
View file

@ -340,19 +340,22 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
COUNT views
--match-filter FILTER Generic video filter. Any field (see
"OUTPUT TEMPLATE") can be compared with a
number or a quoted string using the
operators defined in "Filtering formats".
You can also simply specify a field to
match if the field is present and "!field"
to check if the field is not present.
Multiple filters can be checked using "&".
For example, to only match videos that are
not live, has a like count more than 100, a
dislike count less than 50 (or the dislike
number or a string using the operators
defined in "Filtering formats". You can
also simply specify a field to match if the
field is present and "!field" to check if
the field is not present. In addition,
Python style regular expression matching
can be done using "~=", and multiple
filters can be checked with "&". Use a "\"
to escape "&" or quotes if needed. Eg:
--match-filter "!is_live & like_count>?100
& description~=\'(?i)\bcats \& dogs\b\'"
matches only videos that are not live, has
a like count more than 100 (or the like
field is not available), and also has a
description that contains "python", use
--match-filter "!is_live & like_count>100 &
dislike_count<?50 & description*='python'"
description that contains the phrase "cats
& dogs" (ignoring case)
--no-match-filter Do not use generic video filter (default)
--no-playlist Download only the video, if the URL refers
to a video and a playlist

54
test/test_utils.py
View file

@ -1207,11 +1207,26 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
'9999 51')
def test_match_str(self):
# Unary
self.assertFalse(match_str('xy', {'x': 1200}))
self.assertTrue(match_str('!xy', {'x': 1200}))
self.assertTrue(match_str('x', {'x': 1200}))
self.assertFalse(match_str('!x', {'x': 1200}))
self.assertTrue(match_str('x', {'x': 0}))
self.assertTrue(match_str('is_live', {'is_live': True}))
self.assertFalse(match_str('is_live', {'is_live': False}))
self.assertFalse(match_str('is_live', {'is_live': None}))
self.assertFalse(match_str('is_live', {}))
self.assertFalse(match_str('!is_live', {'is_live': True}))
self.assertTrue(match_str('!is_live', {'is_live': False}))
self.assertTrue(match_str('!is_live', {'is_live': None}))
self.assertTrue(match_str('!is_live', {}))
self.assertTrue(match_str('title', {'title': 'abc'}))
self.assertTrue(match_str('title', {'title': ''}))
self.assertFalse(match_str('!title', {'title': 'abc'}))
self.assertFalse(match_str('!title', {'title': ''}))
# Numeric
self.assertFalse(match_str('x>0', {'x': 0}))
self.assertFalse(match_str('x>0', {}))
self.assertTrue(match_str('x>?0', {}))
@ -1219,6 +1234,8 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
self.assertFalse(match_str('x>2K', {'x': 1200}))
self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200}))
self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200}))
# String
self.assertFalse(match_str('y=a212', {'y': 'foobar42'}))
self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'}))
self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'}))
@ -1234,6 +1251,8 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
self.assertTrue(match_str('y!*=baz', {'y': 'foobar42'}))
self.assertTrue(match_str('y$=42', {'y': 'foobar42'}))
self.assertFalse(match_str('y$=43', {'y': 'foobar42'}))
# And
self.assertFalse(match_str(
'like_count > 100 & dislike_count <? 50 & description',
{'like_count': 90, 'description': 'foo'}))
@ -1246,18 +1265,29 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
self.assertFalse(match_str(
'like_count > 100 & dislike_count <? 50 & description',
{'like_count': 190, 'dislike_count': 10}))
self.assertTrue(match_str('is_live', {'is_live': True}))
self.assertFalse(match_str('is_live', {'is_live': False}))
self.assertFalse(match_str('is_live', {'is_live': None}))
self.assertFalse(match_str('is_live', {}))
self.assertFalse(match_str('!is_live', {'is_live': True}))
self.assertTrue(match_str('!is_live', {'is_live': False}))
self.assertTrue(match_str('!is_live', {'is_live': None}))
self.assertTrue(match_str('!is_live', {}))
self.assertTrue(match_str('title', {'title': 'abc'}))
self.assertTrue(match_str('title', {'title': ''}))
self.assertFalse(match_str('!title', {'title': 'abc'}))
self.assertFalse(match_str('!title', {'title': ''}))
# Regex
self.assertTrue(match_str(r'x~=\bbar', {'x': 'foo bar'}))
self.assertFalse(match_str(r'x~=\bbar.+', {'x': 'foo bar'}))
self.assertFalse(match_str(r'x~=^FOO', {'x': 'foo bar'}))
self.assertTrue(match_str(r'x~=(?i)^FOO', {'x': 'foo bar'}))
# Quotes
self.assertTrue(match_str(r'x^="foo"', {'x': 'foo "bar"'}))
self.assertFalse(match_str(r'x^="foo "', {'x': 'foo "bar"'}))
self.assertFalse(match_str(r'x$="bar"', {'x': 'foo "bar"'}))
self.assertTrue(match_str(r'x$=" \"bar\""', {'x': 'foo "bar"'}))
# Escaping &
self.assertFalse(match_str(r'x=foo & bar', {'x': 'foo & bar'}))
self.assertTrue(match_str(r'x=foo \& bar', {'x': 'foo & bar'}))
self.assertTrue(match_str(r'x=foo \& bar & x^=foo', {'x': 'foo & bar'}))
self.assertTrue(match_str(r'x="foo \& bar" & x^=foo', {'x': 'foo & bar'}))
# Example from docs
self.assertTrue(
r'!is_live & like_count>?100 & description~=\'(?i)\bcats \& dogs\b\'',
{'description': 'Raining Cats & Dogs'})
def test_parse_dfxp_time_expr(self):
self.assertEqual(parse_dfxp_time_expr(None), None)

15
yt_dlp/options.py
View file

@ -378,13 +378,14 @@ def parseOpts(overrideArguments=None):
'Generic video filter. Any field (see "OUTPUT TEMPLATE") can be compared with a '
'number or a string using the operators defined in "Filtering formats". '
'You can also simply specify a field to match if the field is present '
'and "!field" to check if the field is not present. '
'Multiple filters can be checked using "&". '
'For example, to only match videos that are not live, '
'has a like count more than 100, a dislike count less than 50 '
'(or the dislike field is not available), and also has a description '
'that contains "python", use --match-filter "!is_live & '
'like_count>100 & dislike_count<?50 & description*=\'python\'"'))
'and "!field" to check if the field is not present. In addition, '
'Python style regular expression matching can be done using "~=", '
'and multiple filters can be checked with "&". '
'Use a "\\" to escape "&" or quotes if needed. Eg: --match-filter '
r'"!is_live & like_count>?100 & description~=\'(?i)\bcats \& dogs\b\'" '
'matches only videos that are not live, has a like count more than 100 '
'(or the like field is not available), and also has a description '
'that contains the phrase "cats & dogs" (ignoring case)'))
selection.add_option(
'--no-match-filter',
metavar='FILTER', dest='match_filter', action='store_const', const=None,

29
yt_dlp/utils.py
View file

@ -4664,23 +4664,28 @@ def render_table(header_row, data, delim=False, extraGap=0, hideEmpty=False):
def _match_one(filter_part, dct):
# TODO: Generalize code with YoutubeDL._build_format_filter
COMPARISON_OPERATORS = {
'<': operator.lt,
'<=': operator.le,
'>': operator.gt,
'>=': operator.ge,
'=': operator.eq,
STRING_OPERATORS = {
'*=': operator.contains,
'^=': lambda attr, value: attr.startswith(value),
'$=': lambda attr, value: attr.endswith(value),
'~=': lambda attr, value: re.search(value, attr),
}
COMPARISON_OPERATORS = {
**STRING_OPERATORS,
'<=': operator.le, # "<=" must be defined above "<"
'<': operator.lt,
'>=': operator.ge,
'>': operator.gt,
'=': operator.eq,
}
operator_rex = re.compile(r'''(?x)\s*
(?P<key>[a-z_]+)
\s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
(?:
(?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
(?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
(?P<strval>(?![0-9.])[a-z0-9A-Z]*)
(?P<quote>["\'])(?P<quotedstrval>.+?)(?P=quote)|
(?P<strval>.+?)
)
\s*$
''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
@ -4705,9 +4710,8 @@ def _match_one(filter_part, dct):
if quote is not None:
comparison_value = comparison_value.replace(r'\%s' % quote, quote)
else:
if m.group('op') in ('*=', '^=', '$='):
raise ValueError(
'Operator %s only supports string values!' % m.group('op'))
if m.group('op') in STRING_OPERATORS:
raise ValueError('Operator %s only supports string values!' % m.group('op'))
try:
comparison_value = int(m.group('intval'))
except ValueError:
@ -4743,7 +4747,8 @@ def match_str(filter_str, dct):
""" Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
return all(
_match_one(filter_part, dct) for filter_part in filter_str.split('&'))
_match_one(filter_part.replace(r'\&', '&'), dct)
for filter_part in re.split(r'(?<!\\)&', filter_str))
def match_filter_func(filter_str):