kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[cleanup] Misc

Browse source 
Closes #7030, closes #6967
This commit is contained in:
pukkandan 2023-05-20 03:06:23 +05:30
parent 1d7656184c
commit 6f2287cb18
No known key found for this signature in database
GPG key ID: 7EEE9E1E817D0A39
6 changed files with 243 additions and 402 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
test/helper.py
View file

@ -194,8 +194,8 @@ def sanitize_got_info_dict(got_dict):
'formats', 'thumbnails', 'subtitles', 'automatic_captions', 'comments', 'entries', 'formats', 'thumbnails', 'subtitles', 'automatic_captions', 'comments', 'entries',
# Auto-generated # Auto-generated
'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch', 'autonumber', 'playlist', 'format_index', 'video_ext', 'audio_ext', 'duration_string', 'epoch', 'n_entries',
'fulltitle', 'extractor', 'extractor_key', 'filepath', 'infojson_filename', 'original_url', 'n_entries', 'fulltitle', 'extractor', 'extractor_key', 'filename', 'filepath', 'infojson_filename', 'original_url',
# Only live_status needs to be checked # Only live_status needs to be checked
'is_live', 'was_live', 'is_live', 'was_live',

8
test/test_YoutubeDL.py
View file

@ -757,7 +757,7 @@ class TestYoutubeDL(unittest.TestCase):
test('%(id)r %(height)r', "'1234' 1080") test('%(id)r %(height)r', "'1234' 1080")
test('%(ext)s-%(ext|def)d', 'mp4-def') test('%(ext)s-%(ext|def)d', 'mp4-def')
test('%(width|0)04d', '0000') test('%(width|0)04d', '0000')
test('a%(width|)d', 'a', outtmpl_na_placeholder='none') test('a%(width|b)d', 'ab', outtmpl_na_placeholder='none')
FORMATS = self.outtmpl_info['formats'] FORMATS = self.outtmpl_info['formats']
sanitize = lambda x: x.replace(':', '：').replace('"', "＂").replace('\n', ' ') sanitize = lambda x: x.replace(':', '：').replace('"', "＂").replace('\n', ' ')
@ -871,12 +871,12 @@ class TestYoutubeDL(unittest.TestCase):
class SimplePP(PostProcessor): class SimplePP(PostProcessor):
def run(self, info): def run(self, info):
with open(audiofile, 'wt') as f: with open(audiofile, 'w') as f:
f.write('EXAMPLE') f.write('EXAMPLE')
return [info['filepath']], info return [info['filepath']], info
def run_pp(params, PP): def run_pp(params, PP):
with open(filename, 'wt') as f: with open(filename, 'w') as f:
f.write('EXAMPLE') f.write('EXAMPLE')
ydl = YoutubeDL(params) ydl = YoutubeDL(params)
ydl.add_post_processor(PP()) ydl.add_post_processor(PP())
@ -895,7 +895,7 @@ class TestYoutubeDL(unittest.TestCase):
class ModifierPP(PostProcessor): class ModifierPP(PostProcessor):
def run(self, info): def run(self, info):
with open(info['filepath'], 'wt') as f: with open(info['filepath'], 'w') as f:
f.write('MODIFIED') f.write('MODIFIED')
return [], info return [], info

558
test/test_jsinterp.py
View file

@ -14,462 +14,302 @@ from yt_dlp.jsinterp import JS_Undefined, JSInterpreter
class TestJSInterpreter(unittest.TestCase): class TestJSInterpreter(unittest.TestCase):
def _test(self, code, ret, func='f', args=()):
self.assertEqual(JSInterpreter(code).call_function(func, *args), ret)
def test_basic(self): def test_basic(self):
jsi = JSInterpreter('function x(){;}') jsi = JSInterpreter('function f(){;}')
self.assertEqual(jsi.call_function('x'), None) self.assertEqual(repr(jsi.extract_function('f')), 'F<f>')
jsi = JSInterpreter('function x3(){return 42;}')
self.assertEqual(jsi.call_function('x3'), 42)
jsi = JSInterpreter('function x3(){42}')
self.assertEqual(jsi.call_function('x3'), None)
jsi = JSInterpreter('var x5 = function(){return 42;}')
self.assertEqual(jsi.call_function('x5'), 42)
def test_calc(self):
jsi = JSInterpreter('function x4(a){return 2*a+1;}')
self.assertEqual(jsi.call_function('x4', 3), 7)
def test_empty_return(self):
jsi = JSInterpreter('function f(){return; y()}')
self.assertEqual(jsi.call_function('f'), None) self.assertEqual(jsi.call_function('f'), None)
def test_morespace(self): self._test('function f(){return 42;}', 42)
jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }') self._test('function f(){42}', None)
self.assertEqual(jsi.call_function('x', 3), 7) self._test('var f = function(){return 42;}', 42)
jsi = JSInterpreter('function f () { x = 2 ; return x; }') def test_calc(self):
self.assertEqual(jsi.call_function('f'), 2) self._test('function f(a){return 2*a+1;}', 7, args=[3])
def test_empty_return(self):
self._test('function f(){return; y()}', None)
def test_morespace(self):
self._test('function f (a) { return 2 * a + 1 ; }', 7, args=[3])
self._test('function f () { x = 2 ; return x; }', 2)
def test_strange_chars(self): def test_strange_chars(self):
jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }') self._test('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }',
self.assertEqual(jsi.call_function('$_xY1', 20), 21) 21, args=[20], func='$_xY1')
def test_operators(self): def test_operators(self):
jsi = JSInterpreter('function f(){return 1 << 5;}') self._test('function f(){return 1 << 5;}', 32)
self.assertEqual(jsi.call_function('f'), 32) self._test('function f(){return 2 ** 5}', 32)
self._test('function f(){return 19 & 21;}', 17)
jsi = JSInterpreter('function f(){return 2 ** 5}') self._test('function f(){return 11 >> 2;}', 2)
self.assertEqual(jsi.call_function('f'), 32) self._test('function f(){return []? 2+3: 4;}', 5)
self._test('function f(){return 1 == 2}', False)
jsi = JSInterpreter('function f(){return 19 & 21;}') self._test('function f(){return 0 && 1 || 2;}', 2)
self.assertEqual(jsi.call_function('f'), 17) self._test('function f(){return 0 ?? 42;}', 0)
self._test('function f(){return "life, the universe and everything" < 42;}', False)
jsi = JSInterpreter('function f(){return 11 >> 2;}')
self.assertEqual(jsi.call_function('f'), 2)
jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
self.assertEqual(jsi.call_function('f'), 5)
jsi = JSInterpreter('function f(){return 1 == 2}')
self.assertEqual(jsi.call_function('f'), False)
jsi = JSInterpreter('function f(){return 0 && 1 || 2;}')
self.assertEqual(jsi.call_function('f'), 2)
jsi = JSInterpreter('function f(){return 0 ?? 42;}')
self.assertEqual(jsi.call_function('f'), 0)
jsi = JSInterpreter('function f(){return "life, the universe and everything" < 42;}')
self.assertFalse(jsi.call_function('f'))
def test_array_access(self): def test_array_access(self):
jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}') self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7])
self.assertEqual(jsi.call_function('f'), [5, 2, 7])
def test_parens(self): def test_parens(self):
jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}') self._test('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}', 7)
self.assertEqual(jsi.call_function('f'), 7) self._test('function f(){return (1 + 2) * 3;}', 9)
jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
self.assertEqual(jsi.call_function('f'), 9)
def test_quotes(self): def test_quotes(self):
jsi = JSInterpreter(R'function f(){return "a\"\\("}') self._test(R'function f(){return "a\"\\("}', R'a"\(')
self.assertEqual(jsi.call_function('f'), R'a"\(')
def test_assignments(self): def test_assignments(self):
jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}') self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31)
self.assertEqual(jsi.call_function('f'), 31) self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
self.assertEqual(jsi.call_function('f'), 51)
jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
self.assertEqual(jsi.call_function('f'), -11)
def test_comments(self): def test_comments(self):
'Skipping: Not yet fully implemented' 'Skipping: Not yet fully implemented'
return return
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
var x = /* 1 + */ 2; var x = /* 1 + */ 2;
var y = /* 30 var y = /* 30
* 40 */ 50; * 40 */ 50;
return x + y; return x + y;
} }
''') ''', 52)
self.assertEqual(jsi.call_function('x'), 52)
jsi = JSInterpreter(''' self._test('''
function f() { function f() {
var x = "/*"; var x = "/*";
var y = 1 /* comment */ + 2; var y = 1 /* comment */ + 2;
return y; return y;
} }
''') ''', 3)
self.assertEqual(jsi.call_function('f'), 3)
def test_precedence(self): def test_precedence(self):
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
var a = [10, 20, 30, 40, 50]; var a = [10, 20, 30, 40, 50];
var b = 6; var b = 6;
a[0]=a[b%a.length]; a[0]=a[b%a.length];
return a; return a;
}''') }
self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50]) ''', [20, 20, 30, 40, 50])
def test_builtins(self): def test_builtins(self):
jsi = JSInterpreter(''' jsi = JSInterpreter('function f() { return NaN }')
function x() { return NaN } self.assertTrue(math.isnan(jsi.call_function('f')))
''')
self.assertTrue(math.isnan(jsi.call_function('x')))
jsi = JSInterpreter(''' self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }',
function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; } 86000)
''') self._test('function f(dt) { return new Date(dt) - 0; }',
self.assertEqual(jsi.call_function('x'), 86000) 86000, args=['Wednesday 31 December 1969 18:01:26 MDT'])
jsi = JSInterpreter('''
function x(dt) { return new Date(dt) - 0; }
''')
self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
def test_call(self): def test_call(self):
jsi = JSInterpreter(''' jsi = JSInterpreter('''
function x() { return 2; } function x() { return 2; }
function y(a) { return x() + (a?a:0); } function y(a) { return x() + (a?a:0); }
function z() { return y(3); } function z() { return y(3); }
''') ''')
self.assertEqual(jsi.call_function('z'), 5) self.assertEqual(jsi.call_function('z'), 5)
self.assertEqual(jsi.call_function('y'), 2) self.assertEqual(jsi.call_function('y'), 2)
def test_if(self): def test_if(self):
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
let a = 9; let a = 9;
if (0==0) {a++} if (0==0) {a++}
return a return a
}''') }
self.assertEqual(jsi.call_function('x'), 10) ''', 10)
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
if (0==0) {return 10} if (0==0) {return 10}
}''') }
self.assertEqual(jsi.call_function('x'), 10) ''', 10)
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
if (0!=0) {return 1} if (0!=0) {return 1}
else {return 10} else {return 10}
}''') }
self.assertEqual(jsi.call_function('x'), 10) ''', 10)
""" # Unsupported """ # Unsupported
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
if (0!=0) {return 1} if (0!=0) {return 1}
else if (1==0) {return 2} else if (1==0) {return 2}
else {return 10} else {return 10}
}''') }
self.assertEqual(jsi.call_function('x'), 10) ''', 10)
""" """
def test_for_loop(self): def test_for_loop(self):
jsi = JSInterpreter(''' self._test('function f() { a=0; for (i=0; i-10; i++) {a++} return a }', 10)
function x() { a=0; for (i=0; i-10; i++) {a++} return a }
''')
self.assertEqual(jsi.call_function('x'), 10)
def test_switch(self): def test_switch(self):
jsi = JSInterpreter(''' jsi = JSInterpreter('''
function x(f) { switch(f){ function f(x) { switch(x){
case 1:f+=1; case 1:x+=1;
case 2:f+=2; case 2:x+=2;
case 3:f+=3;break; case 3:x+=3;break;
case 4:f+=4; case 4:x+=4;
default:f=0; default:x=0;
} return f } } return x }
''') ''')
self.assertEqual(jsi.call_function('x', 1), 7) self.assertEqual(jsi.call_function('f', 1), 7)
self.assertEqual(jsi.call_function('x', 3), 6) self.assertEqual(jsi.call_function('f', 3), 6)
self.assertEqual(jsi.call_function('x', 5), 0) self.assertEqual(jsi.call_function('f', 5), 0)
def test_switch_default(self): def test_switch_default(self):
jsi = JSInterpreter(''' jsi = JSInterpreter('''
function x(f) { switch(f){ function f(x) { switch(x){
case 2: f+=2; case 2: x+=2;
default: f-=1; default: x-=1;
case 5: case 5:
case 6: f+=6; case 6: x+=6;
case 0: break; case 0: break;
case 1: f+=1; case 1: x+=1;
} return f } } return x }
''') ''')
self.assertEqual(jsi.call_function('x', 1), 2) self.assertEqual(jsi.call_function('f', 1), 2)
self.assertEqual(jsi.call_function('x', 5), 11) self.assertEqual(jsi.call_function('f', 5), 11)
self.assertEqual(jsi.call_function('x', 9), 14) self.assertEqual(jsi.call_function('f', 9), 14)
def test_try(self): def test_try(self):
jsi = JSInterpreter(''' self._test('function f() { try{return 10} catch(e){return 5} }', 10)
function x() { try{return 10} catch(e){return 5} }
''')
self.assertEqual(jsi.call_function('x'), 10)
def test_catch(self): def test_catch(self):
jsi = JSInterpreter(''' self._test('function f() { try{throw 10} catch(e){return 5} }', 5)
function x() { try{throw 10} catch(e){return 5} }
''')
self.assertEqual(jsi.call_function('x'), 5)
def test_finally(self): def test_finally(self):
jsi = JSInterpreter(''' self._test('function f() { try{throw 10} finally {return 42} }', 42)
function x() { try{throw 10} finally {return 42} } self._test('function f() { try{throw 10} catch(e){return 5} finally {return 42} }', 42)
''')
self.assertEqual(jsi.call_function('x'), 42)
jsi = JSInterpreter('''
function x() { try{throw 10} catch(e){return 5} finally {return 42} }
''')
self.assertEqual(jsi.call_function('x'), 42)
def test_nested_try(self): def test_nested_try(self):
jsi = JSInterpreter(''' self._test('''
function x() {try { function f() {try {
try{throw 10} finally {throw 42} try{throw 10} finally {throw 42}
} catch(e){return 5} } } catch(e){return 5} }
''') ''', 5)
self.assertEqual(jsi.call_function('x'), 5)
def test_for_loop_continue(self): def test_for_loop_continue(self):
jsi = JSInterpreter(''' self._test('function f() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }', 0)
function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
''')
self.assertEqual(jsi.call_function('x'), 0)
def test_for_loop_break(self): def test_for_loop_break(self):
jsi = JSInterpreter(''' self._test('function f() { a=0; for (i=0; i-10; i++) { break; a++ } return a }', 0)
function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
''')
self.assertEqual(jsi.call_function('x'), 0)
def test_for_loop_try(self): def test_for_loop_try(self):
jsi = JSInterpreter(''' self._test('''
function x() { function f() {
for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} }; for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
return 42 } return 42 }
''') ''', 42)
self.assertEqual(jsi.call_function('x'), 42)
def test_literal_list(self): def test_literal_list(self):
jsi = JSInterpreter(''' self._test('function f() { return [1, 2, "asdf", [5, 6, 7]][3] }', [5, 6, 7])
function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
''')
self.assertEqual(jsi.call_function('x'), [5, 6, 7])
def test_comma(self): def test_comma(self):
jsi = JSInterpreter(''' self._test('function f() { a=5; a -= 1, a+=3; return a }', 7)
function x() { a=5; a -= 1, a+=3; return a } self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7)
''') self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
self.assertEqual(jsi.call_function('x'), 7)
jsi = JSInterpreter('''
function x() { a=5; return (a -= 1, a+=3, a); }
''')
self.assertEqual(jsi.call_function('x'), 7)
jsi = JSInterpreter('''
function x() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }
''')
self.assertEqual(jsi.call_function('x'), 5)
def test_void(self): def test_void(self):
jsi = JSInterpreter(''' self._test('function f() { return void 42; }', None)
function x() { return void 42; }
''')
self.assertEqual(jsi.call_function('x'), None)
def test_return_function(self): def test_return_function(self):
jsi = JSInterpreter(''' jsi = JSInterpreter('''
function x() { return [1, function(){return 1}][1] } function f() { return [1, function(){return 1}][1] }
''') ''')
self.assertEqual(jsi.call_function('x')([]), 1) self.assertEqual(jsi.call_function('f')([]), 1)
def test_null(self): def test_null(self):
jsi = JSInterpreter(''' self._test('function f() { return null; }', None)
function x() { return null; } self._test('function f() { return [null > 0, null < 0, null == 0, null === 0]; }',
''') [False, False, False, False])
self.assertEqual(jsi.call_function('x'), None) self._test('function f() { return [null >= 0, null <= 0]; }', [True, True])
jsi = JSInterpreter('''
function x() { return [null > 0, null < 0, null == 0, null === 0]; }
''')
self.assertEqual(jsi.call_function('x'), [False, False, False, False])
jsi = JSInterpreter('''
function x() { return [null >= 0, null <= 0]; }
''')
self.assertEqual(jsi.call_function('x'), [True, True])
def test_undefined(self): def test_undefined(self):
jsi = JSInterpreter(''' self._test('function f() { return undefined === undefined; }', True)
function x() { return undefined === undefined; } self._test('function f() { return undefined; }', JS_Undefined)
''') self._test('function f() {return undefined ?? 42; }', 42)
self.assertEqual(jsi.call_function('x'), True) self._test('function f() { let v; return v; }', JS_Undefined)
self._test('function f() { let v; return v**0; }', 1)
self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }',
[False, False, JS_Undefined, JS_Undefined])
self._test('''
function f() { return [
undefined === undefined,
undefined == undefined,
undefined == null,
undefined < undefined,
undefined > undefined,
undefined === 0,
undefined == 0,
undefined < 0,
undefined > 0,
undefined >= 0,
undefined <= 0,
undefined > null,
undefined < null,
undefined === null
]; }
''', list(map(bool, (1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))))
jsi = JSInterpreter(''' jsi = JSInterpreter('''
function x() { return undefined; } function f() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
''') ''')
self.assertEqual(jsi.call_function('x'), JS_Undefined) for y in jsi.call_function('f'):
jsi = JSInterpreter('''
function x() { let v; return v; }
''')
self.assertEqual(jsi.call_function('x'), JS_Undefined)
jsi = JSInterpreter('''
function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
''')
self.assertEqual(jsi.call_function('x'), [True, True, False, False])
jsi = JSInterpreter('''
function x() { return [undefined === 0, undefined == 0, undefined < 0, undefined > 0]; }
''')
self.assertEqual(jsi.call_function('x'), [False, False, False, False])
jsi = JSInterpreter('''
function x() { return [undefined >= 0, undefined <= 0]; }
''')
self.assertEqual(jsi.call_function('x'), [False, False])
jsi = JSInterpreter('''
function x() { return [undefined > null, undefined < null, undefined == null, undefined === null]; }
''')
self.assertEqual(jsi.call_function('x'), [False, False, True, False])
jsi = JSInterpreter('''
function x() { return [undefined === null, undefined == null, undefined < null, undefined > null]; }
''')
self.assertEqual(jsi.call_function('x'), [False, True, False, False])
jsi = JSInterpreter('''
function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
''')
for y in jsi.call_function('x'):
self.assertTrue(math.isnan(y)) self.assertTrue(math.isnan(y))
jsi = JSInterpreter('''
function x() { let v; return v**0; }
''')
self.assertEqual(jsi.call_function('x'), 1)
jsi = JSInterpreter('''
function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
''')
self.assertEqual(jsi.call_function('x'), [False, False, JS_Undefined, JS_Undefined])
jsi = JSInterpreter('function x(){return undefined ?? 42; }')
self.assertEqual(jsi.call_function('x'), 42)
def test_object(self): def test_object(self):
jsi = JSInterpreter(''' self._test('function f() { return {}; }', {})
function x() { return {}; } self._test('function f() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }', [42, 0])
''') self._test('function f() { let a; return a?.qq; }', JS_Undefined)
self.assertEqual(jsi.call_function('x'), {}) self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined)
jsi = JSInterpreter('''
function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
''')
self.assertEqual(jsi.call_function('x'), [42, 0])
jsi = JSInterpreter('''
function x() { let a; return a?.qq; }
''')
self.assertEqual(jsi.call_function('x'), JS_Undefined)
jsi = JSInterpreter('''
function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
''')
self.assertEqual(jsi.call_function('x'), JS_Undefined)
def test_regex(self): def test_regex(self):
jsi = JSInterpreter(''' self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
function x() { let a=/,,[/,913,/](,)}/; }
''')
self.assertEqual(jsi.call_function('x'), None)
jsi = JSInterpreter(''' jsi = JSInterpreter('function f() { let a=/,,[/,913,/](,)}/; return a; }')
function x() { let a=/,,[/,913,/](,)}/; return a; } self.assertIsInstance(jsi.call_function('f'), re.Pattern)
''')
self.assertIsInstance(jsi.call_function('x'), re.Pattern)
jsi = JSInterpreter(''' jsi = JSInterpreter('function f() { let a=/,,[/,913,/](,)}/i; return a; }')
function x() { let a=/,,[/,913,/](,)}/i; return a; } self.assertEqual(jsi.call_function('f').flags & re.I, re.I)
''')
self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
jsi = JSInterpreter(R''' jsi = JSInterpreter(R'function f() { let a=/,][}",],()}(\[)/; return a; }')
function x() { let a=/,][}",],()}(\[)/; return a; } self.assertEqual(jsi.call_function('f').pattern, r',][}",],()}(\[)')
''')
self.assertEqual(jsi.call_function('x').pattern, r',][}",],()}(\[)')
jsi = JSInterpreter(R''' jsi = JSInterpreter(R'function f() { let a=[/[)\\]/]; return a[0]; }')
function x() { let a=[/[)\\]/]; return a[0]; } self.assertEqual(jsi.call_function('f').pattern, r'[)\\]')
''')
self.assertEqual(jsi.call_function('x').pattern, r'[)\\]')
def test_char_code_at(self): def test_char_code_at(self):
jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}') jsi = JSInterpreter('function f(i){return "test".charCodeAt(i)}')
self.assertEqual(jsi.call_function('x', 0), 116) self.assertEqual(jsi.call_function('f', 0), 116)
self.assertEqual(jsi.call_function('x', 1), 101) self.assertEqual(jsi.call_function('f', 1), 101)
self.assertEqual(jsi.call_function('x', 2), 115) self.assertEqual(jsi.call_function('f', 2), 115)
self.assertEqual(jsi.call_function('x', 3), 116) self.assertEqual(jsi.call_function('f', 3), 116)
self.assertEqual(jsi.call_function('x', 4), None) self.assertEqual(jsi.call_function('f', 4), None)
self.assertEqual(jsi.call_function('x', 'not_a_number'), 116) self.assertEqual(jsi.call_function('f', 'not_a_number'), 116)
def test_bitwise_operators_overflow(self): def test_bitwise_operators_overflow(self):
jsi = JSInterpreter('function x(){return -524999584 << 5}') self._test('function f(){return -524999584 << 5}', 379882496)
self.assertEqual(jsi.call_function('x'), 379882496) self._test('function f(){return 1236566549 << 5}', 915423904)
jsi = JSInterpreter('function x(){return 1236566549 << 5}')
self.assertEqual(jsi.call_function('x'), 915423904)
def test_bitwise_operators_typecast(self): def test_bitwise_operators_typecast(self):
jsi = JSInterpreter('function x(){return null << 5}') self._test('function f(){return null << 5}', 0)
self.assertEqual(jsi.call_function('x'), 0) self._test('function f(){return undefined >> 5}', 0)
self._test('function f(){return 42 << NaN}', 42)
jsi = JSInterpreter('function x(){return undefined >> 5}')
self.assertEqual(jsi.call_function('x'), 0)
jsi = JSInterpreter('function x(){return 42 << NaN}')
self.assertEqual(jsi.call_function('x'), 42)
def test_negative(self): def test_negative(self):
jsi = JSInterpreter("function f(){return 2 * -2.0;}") self._test('function f(){return 2 * -2.0 ;}', -4)
self.assertEqual(jsi.call_function('f'), -4) self._test('function f(){return 2 - - -2 ;}', 0)
self._test('function f(){return 2 - - - -2 ;}', 4)
jsi = JSInterpreter('function f(){return 2 - - -2;}') self._test('function f(){return 2 - + + - -2;}', 0)
self.assertEqual(jsi.call_function('f'), 0) self._test('function f(){return 2 + - + - -2;}', 0)
jsi = JSInterpreter('function f(){return 2 - - - -2;}')
self.assertEqual(jsi.call_function('f'), 4)
jsi = JSInterpreter('function f(){return 2 - + + - -2;}')
self.assertEqual(jsi.call_function('f'), 0)
jsi = JSInterpreter('function f(){return 2 + - + - -2;}')
self.assertEqual(jsi.call_function('f'), 0)
if __name__ == '__main__': if __name__ == '__main__':

65
yt_dlp/YoutubeDL.py
View file

@ -13,6 +13,7 @@ import os
import random import random
import re import re
import shutil import shutil
import string
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
@ -21,7 +22,6 @@ import tokenize
import traceback import traceback
import unicodedata import unicodedata
import urllib.request import urllib.request
from string import Formatter, ascii_letters
from .cache import Cache from .cache import Cache
from .compat import compat_os_name, compat_shlex_quote from .compat import compat_os_name, compat_shlex_quote
@ -1079,7 +1079,7 @@ class YoutubeDL:
# correspondingly that is not what we want since we need to keep # correspondingly that is not what we want since we need to keep
# '%%' intact for template dict substitution step. Working around # '%%' intact for template dict substitution step. Working around
# with boundary-alike separator hack. # with boundary-alike separator hack.
sep = ''.join(random.choices(ascii_letters, k=32)) sep = ''.join(random.choices(string.ascii_letters, k=32))
outtmpl = outtmpl.replace('%%', f'%{sep}%').replace('$$', f'${sep}$') outtmpl = outtmpl.replace('%%', f'%{sep}%').replace('$$', f'${sep}$')
# outtmpl should be expand_path'ed before template dict substitution # outtmpl should be expand_path'ed before template dict substitution
@ -1238,7 +1238,7 @@ class YoutubeDL:
return list(obj) return list(obj)
return repr(obj) return repr(obj)
class _ReplacementFormatter(Formatter): class _ReplacementFormatter(string.Formatter):
def get_field(self, field_name, args, kwargs): def get_field(self, field_name, args, kwargs):
if field_name.isdigit(): if field_name.isdigit():
return args[0], -1 return args[0], -1
@ -2068,86 +2068,86 @@ class YoutubeDL:
def _parse_filter(tokens): def _parse_filter(tokens):
filter_parts = [] filter_parts = []
for type, string, start, _, _ in tokens: for type, string_, start, _, _ in tokens:
if type == tokenize.OP and string == ']': if type == tokenize.OP and string_ == ']':
return ''.join(filter_parts) return ''.join(filter_parts)
else: else:
filter_parts.append(string) filter_parts.append(string_)
def _remove_unused_ops(tokens): def _remove_unused_ops(tokens):
# Remove operators that we don't use and join them with the surrounding strings. # Remove operators that we don't use and join them with the surrounding strings.
# E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9' # E.g. 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
ALLOWED_OPS = ('/', '+', ',', '(', ')') ALLOWED_OPS = ('/', '+', ',', '(', ')')
last_string, last_start, last_end, last_line = None, None, None, None last_string, last_start, last_end, last_line = None, None, None, None
for type, string, start, end, line in tokens: for type, string_, start, end, line in tokens:
if type == tokenize.OP and string == '[': if type == tokenize.OP and string_ == '[':
if last_string: if last_string:
yield tokenize.NAME, last_string, last_start, last_end, last_line yield tokenize.NAME, last_string, last_start, last_end, last_line
last_string = None last_string = None
yield type, string, start, end, line yield type, string_, start, end, line
# everything inside brackets will be handled by _parse_filter # everything inside brackets will be handled by _parse_filter
for type, string, start, end, line in tokens: for type, string_, start, end, line in tokens:
yield type, string, start, end, line yield type, string_, start, end, line
if type == tokenize.OP and string == ']': if type == tokenize.OP and string_ == ']':
break break
elif type == tokenize.OP and string in ALLOWED_OPS: elif type == tokenize.OP and string_ in ALLOWED_OPS:
if last_string: if last_string:
yield tokenize.NAME, last_string, last_start, last_end, last_line yield tokenize.NAME, last_string, last_start, last_end, last_line
last_string = None last_string = None
yield type, string, start, end, line yield type, string_, start, end, line
elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]: elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
if not last_string: if not last_string:
last_string = string last_string = string_
last_start = start last_start = start
last_end = end last_end = end
else: else:
last_string += string last_string += string_
if last_string: if last_string:
yield tokenize.NAME, last_string, last_start, last_end, last_line yield tokenize.NAME, last_string, last_start, last_end, last_line
def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False): def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
selectors = [] selectors = []
current_selector = None current_selector = None
for type, string, start, _, _ in tokens: for type, string_, start, _, _ in tokens:
# ENCODING is only defined in python 3.x # ENCODING is only defined in python 3.x
if type == getattr(tokenize, 'ENCODING', None): if type == getattr(tokenize, 'ENCODING', None):
continue continue
elif type in [tokenize.NAME, tokenize.NUMBER]: elif type in [tokenize.NAME, tokenize.NUMBER]:
current_selector = FormatSelector(SINGLE, string, []) current_selector = FormatSelector(SINGLE, string_, [])
elif type == tokenize.OP: elif type == tokenize.OP:
if string == ')': if string_ == ')':
if not inside_group: if not inside_group:
# ')' will be handled by the parentheses group # ')' will be handled by the parentheses group
tokens.restore_last_token() tokens.restore_last_token()
break break
elif inside_merge and string in ['/', ',']: elif inside_merge and string_ in ['/', ',']:
tokens.restore_last_token() tokens.restore_last_token()
break break
elif inside_choice and string == ',': elif inside_choice and string_ == ',':
tokens.restore_last_token() tokens.restore_last_token()
break break
elif string == ',': elif string_ == ',':
if not current_selector: if not current_selector:
raise syntax_error('"," must follow a format selector', start) raise syntax_error('"," must follow a format selector', start)
selectors.append(current_selector) selectors.append(current_selector)
current_selector = None current_selector = None
elif string == '/': elif string_ == '/':
if not current_selector: if not current_selector:
raise syntax_error('"/" must follow a format selector', start) raise syntax_error('"/" must follow a format selector', start)
first_choice = current_selector first_choice = current_selector
second_choice = _parse_format_selection(tokens, inside_choice=True) second_choice = _parse_format_selection(tokens, inside_choice=True)
current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), []) current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
elif string == '[': elif string_ == '[':
if not current_selector: if not current_selector:
current_selector = FormatSelector(SINGLE, 'best', []) current_selector = FormatSelector(SINGLE, 'best', [])
format_filter = _parse_filter(tokens) format_filter = _parse_filter(tokens)
current_selector.filters.append(format_filter) current_selector.filters.append(format_filter)
elif string == '(': elif string_ == '(':
if current_selector: if current_selector:
raise syntax_error('Unexpected "("', start) raise syntax_error('Unexpected "("', start)
group = _parse_format_selection(tokens, inside_group=True) group = _parse_format_selection(tokens, inside_group=True)
current_selector = FormatSelector(GROUP, group, []) current_selector = FormatSelector(GROUP, group, [])
elif string == '+': elif string_ == '+':
if not current_selector: if not current_selector:
raise syntax_error('Unexpected "+"', start) raise syntax_error('Unexpected "+"', start)
selector_1 = current_selector selector_1 = current_selector
@ -2156,7 +2156,7 @@ class YoutubeDL:
raise syntax_error('Expected a selector', start) raise syntax_error('Expected a selector', start)
current_selector = FormatSelector(MERGE, (selector_1, selector_2), []) current_selector = FormatSelector(MERGE, (selector_1, selector_2), [])
else: else:
raise syntax_error(f'Operator not recognized: "{string}"', start) raise syntax_error(f'Operator not recognized: "{string_}"', start)
elif type == tokenize.ENDMARKER: elif type == tokenize.ENDMARKER:
break break
if current_selector: if current_selector:
@ -2898,7 +2898,7 @@ class YoutubeDL:
fmt = '%({})s' fmt = '%({})s'
if tmpl.startswith('{'): if tmpl.startswith('{'):
tmpl = f'.{tmpl}' tmpl, fmt = f'.{tmpl}', '%({})j'
if tmpl.endswith('='): if tmpl.endswith('='):
tmpl, fmt = tmpl[:-1], '{0} = %({0})#j' tmpl, fmt = tmpl[:-1], '{0} = %({0})#j'
return '\n'.join(map(fmt.format, [tmpl] if mobj.group('dict') else tmpl.split(','))) return '\n'.join(map(fmt.format, [tmpl] if mobj.group('dict') else tmpl.split(',')))
@ -2937,7 +2937,8 @@ class YoutubeDL:
print_field('url', 'urls') print_field('url', 'urls')
print_field('thumbnail', optional=True) print_field('thumbnail', optional=True)
print_field('description', optional=True) print_field('description', optional=True)
print_field('filename', optional=True) if filename:
print_field('filename')
if self.params.get('forceduration') and info_copy.get('duration') is not None: if self.params.get('forceduration') and info_copy.get('duration') is not None:
self.to_stdout(formatSeconds(info_copy['duration'])) self.to_stdout(formatSeconds(info_copy['duration']))
print_field('format') print_field('format')
@ -3419,8 +3420,8 @@ class YoutubeDL:
if remove_private_keys: if remove_private_keys:
reject = lambda k, v: v is None or k.startswith('__') or k in { reject = lambda k, v: v is None or k.startswith('__') or k in {
'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries', 'requested_downloads', 'requested_formats', 'requested_subtitles', 'requested_entries',
'entries', 'filepath', '_filename', 'infojson_filename', 'original_url', 'playlist_autonumber', 'entries', 'filepath', '_filename', 'filename', 'infojson_filename', 'original_url',
'_format_sort_fields', 'playlist_autonumber', '_format_sort_fields',
} }
else: else:
reject = lambda k, v: False reject = lambda k, v: False
@ -3489,7 +3490,7 @@ class YoutubeDL:
*files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)') *files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)')
return infodict return infodict
def run_all_pps(self, key, info, *, additional_pps=None, fatal=True): def run_all_pps(self, key, info, *, additional_pps=None):
if key != 'video': if key != 'video':
self._forceprint(key, info) self._forceprint(key, info)
for pp in (additional_pps or []) + self._pps[key]: for pp in (additional_pps or []) + self._pps[key]:

2
yt_dlp/jsinterp.py
View file

@ -248,7 +248,7 @@ class JSInterpreter:
return return
counters = {k: 0 for k in _MATCHING_PARENS.values()} counters = {k: 0 for k in _MATCHING_PARENS.values()}
start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1 start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
in_quote, escaping, after_op, in_regex_char_group, in_unary_op = None, False, True, False, False in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
for idx, char in enumerate(expr): for idx, char in enumerate(expr):
if not in_quote and char in _MATCHING_PARENS: if not in_quote and char in _MATCHING_PARENS:
counters[_MATCHING_PARENS[char]] += 1 counters[_MATCHING_PARENS[char]] += 1

8
yt_dlp/utils.py
View file

@ -3281,7 +3281,7 @@ def is_iterable_like(x, allowed_types=collections.abc.Iterable, blocked_types=NO
def variadic(x, allowed_types=NO_DEFAULT): def variadic(x, allowed_types=NO_DEFAULT):
return x if is_iterable_like(x, blocked_types=allowed_types) else (x,) return x if is_iterable_like(x, blocked_types=allowed_types) else (x, )
def dict_get(d, key_or_keys, default=None, skip_false_values=True): def dict_get(d, key_or_keys, default=None, skip_false_values=True):
@ -5404,7 +5404,7 @@ def to_high_limit_path(path):
def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY): def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY):
val = traverse_obj(obj, *variadic(field)) val = traverse_obj(obj, *variadic(field))
if (not val and val != 0) if ignore is NO_DEFAULT else val in variadic(ignore): if not val if ignore is NO_DEFAULT else val in variadic(ignore):
return default return default
return template % func(val) return template % func(val)
@ -5704,8 +5704,8 @@ def traverse_dict(dictn, keys, casesense=True):
return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True) return traverse_obj(dictn, keys, casesense=casesense, is_user_input=True, traverse_string=True)
def get_first(obj, keys, **kwargs): def get_first(obj, *paths, **kwargs):
return traverse_obj(obj, (..., *variadic(keys)), **kwargs, get_all=False) return traverse_obj(obj, *((..., *variadic(keys)) for keys in paths), **kwargs, get_all=False)
def time_seconds(**kwargs): def time_seconds(**kwargs):