kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added --force-overwrites option (https://github.com/ytdl-org/youtube-dl/pull/20405)

Browse source 
Co-authored by alxnull
This commit is contained in:
alxnull 2019-10-13 18:00:48 +02:00 committed by pukkandan
parent f5546c0b3c
commit 0c3d0f5177
8 changed files with 95 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -47,6 +47,7 @@ offlinetest: codetest
--exclude test_age_restriction.py \
--exclude test_download.py \
--exclude test_iqiyi_sdk_interpreter.py \
--exclude test_overwrites.py \
--exclude test_socks.py \
--exclude test_subtitles.py \
--exclude test_write_annotations.py \

2
devscripts/run_tests.sh
View file

@ -1,7 +1,7 @@
#!/bin/bash
# Keep this list in sync with the `offlinetest` target in Makefile
DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|socks|subtitles|write_annotations|youtube_lists|youtube_signature|post_hooks"
DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|overwrites|socks|subtitles|write_annotations|youtube_lists|youtube_signature|post_hooks"
test_set=""
multiprocess_args=""

2
test/parameters.json
View file

@ -14,7 +14,7 @@
"logtostderr": false,
"matchtitle": null,
"max_downloads": null,
"nooverwrites": false,
"overwrites": null,
"nopart": false,
"noprogress": false,
"outtmpl": "%(id)s.%(ext)s",

52
test/test_overwrites.py Normal file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env python
from __future__ import unicode_literals
import os
from os.path import join
import subprocess
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from test.helper import try_rm
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
download_file = join(root_dir, 'test.webm')
class TestOverwrites(unittest.TestCase):
def setUp(self):
# create an empty file
open(download_file, 'a').close()
def test_default_overwrites(self):
outp = subprocess.Popen(
[
sys.executable, 'youtube_dlc/__main__.py',
'-o', 'test.webm',
'https://www.youtube.com/watch?v=jNQXAC9IVRw'
], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sout, serr = outp.communicate()
self.assertTrue(b'has already been downloaded' in sout)
# if the file has no content, it has not been redownloaded
self.assertTrue(os.path.getsize(download_file) < 1)
def test_yes_overwrites(self):
outp = subprocess.Popen(
[
sys.executable, 'youtube_dlc/__main__.py', '--yes-overwrites',
'-o', 'test.webm',
'https://www.youtube.com/watch?v=jNQXAC9IVRw'
], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sout, serr = outp.communicate()
self.assertTrue(b'has already been downloaded' not in sout)
# if the file has no content, it has not been redownloaded
self.assertTrue(os.path.getsize(download_file) > 1)
def tearDown(self):
try_rm(join(root_dir, 'test.webm'))
if __name__ == '__main__':
unittest.main()

32
youtube_dlc/YoutubeDL.py
View file

@ -181,7 +181,9 @@ class YoutubeDL(object):
trim_file_name: Limit length of filename (extension excluded).
ignoreerrors: Do not stop on download errors. (Default True when running youtube-dlc, but False when directly accessing YoutubeDL class)
force_generic_extractor: Force downloader to use the generic extractor
nooverwrites: Prevent overwriting files.
overwrites: Overwrite all video and metadata files if True,
overwrite only non-video files if None
and don't overwrite any file if False
playliststart: Playlist item to start at.
playlistend: Playlist item to end at.
playlist_items: Specific indices of playlist to download.
@ -686,6 +688,13 @@ class YoutubeDL(object):
except UnicodeEncodeError:
self.to_screen('[download] The file has already been downloaded')
def report_file_delete(self, file_name):
"""Report that existing file will be deleted."""
try:
self.to_screen('Deleting already existent file %s' % file_name)
except UnicodeEncodeError:
self.to_screen('Deleting already existent file')
def prepare_filename(self, info_dict):
"""Generate the output filename."""
try:
@ -1898,7 +1907,7 @@ class YoutubeDL(object):
if self.params.get('writedescription', False):
descfn = replace_extension(filename, 'description', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
self.to_screen('[info] Video description is already present')
elif info_dict.get('description') is None:
self.report_warning('There\'s no description to write.')
@ -1913,7 +1922,7 @@ class YoutubeDL(object):
if self.params.get('writeannotations', False):
annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)):
self.to_screen('[info] Video annotations are already present')
elif not info_dict.get('annotations'):
self.report_warning('There are no annotations to write.')
@ -1947,7 +1956,7 @@ class YoutubeDL(object):
for sub_lang, sub_info in subtitles.items():
sub_format = sub_info['ext']
sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
else:
self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@ -2002,7 +2011,7 @@ class YoutubeDL(object):
if self.params.get('writeinfojson', False):
infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
self.to_screen('[info] Video description metadata is already present')
else:
self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
@ -2110,11 +2119,15 @@ class YoutubeDL(object):
'Requested formats are incompatible for merge and will be merged into mkv.')
# Ensure filename always has a correct extension for successful merge
filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
if os.path.exists(encodeFilename(filename)):
file_exists = os.path.exists(encodeFilename(filename))
if not self.params.get('overwrites', False) and file_exists:
self.to_screen(
'[download] %s has already been downloaded and '
'merged' % filename)
else:
if file_exists:
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
for f in requested_formats:
new_info = dict(info_dict)
new_info.update(f)
@ -2131,6 +2144,11 @@ class YoutubeDL(object):
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
# Delete existing file with --yes-overwrites
if self.params.get('overwrites', False):
if os.path.exists(encodeFilename(filename)):
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
# Just a single file
success, real_download = dl(filename, info_dict)
info_dict['__real_download'] = real_download
@ -2661,7 +2679,7 @@ class YoutubeDL(object):
thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
self.to_screen('[%s] %s: Thumbnail %sis already present' %
(info_dict['extractor'], info_dict['id'], thumb_display_id))
else:

5
youtube_dlc/__init__.py
View file

@ -176,6 +176,9 @@ def _real_main(argv=None):
opts.max_sleep_interval = opts.sleep_interval
if opts.ap_mso and opts.ap_mso not in MSO_INFO:
parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
if opts.overwrites:
# --yes-overwrites implies --no-continue
opts.continue_dl = False
def parse_retries(retries):
if retries in ('inf', 'infinite'):
@ -391,7 +394,7 @@ def _real_main(argv=None):
'ignoreerrors': opts.ignoreerrors,
'force_generic_extractor': opts.force_generic_extractor,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
'overwrites': opts.overwrites,
'retries': opts.retries,
'fragment_retries': opts.fragment_retries,
'skip_unavailable_fragments': opts.skip_unavailable_fragments,

2
youtube_dlc/downloader/common.py
View file

@ -332,7 +332,7 @@ class FileDownloader(object):
"""
nooverwrites_and_exists = (
self.params.get('nooverwrites', False)
not self.params.get('overwrites', True)
and os.path.exists(encodeFilename(filename))
)

12
youtube_dlc/options.py
View file

@ -834,8 +834,16 @@ def parseOpts(overrideArguments=None):
help=optparse.SUPPRESS_HELP)
filesystem.add_option(
'-w', '--no-overwrites',
action='store_true', dest='nooverwrites', default=False,
help='Do not overwrite files')
action='store_false', dest='overwrites', default=None,
help='Do not overwrite any files')
filesystem.add_option(
'--force-overwrites', '--yes-overwrites',
action='store_true', dest='overwrites',
help='Overwrite all video and metadata files. This option includes --no-continue')
filesystem.add_option(
'--no-force-overwrites',
action='store_const', dest='overwrites', const=None,
help='Do not overwrite the video, but overwrite related files (default)')
filesystem.add_option(
'-c', '--continue',
action='store_true', dest='continue_dl', default=True,