kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

#29 New option -P/--paths to give different paths for different types of files

Browse source 
Syntax: `-P "type:path" -P "type:path"`
Types: home, temp, description, annotation, subtitle, infojson, thumbnail
This commit is contained in:
pukkandan 2021-01-23 17:48:12 +05:30 committed by pukkandan
parent b8f6bbe68a
commit 0202b52a0c
7 changed files with 321 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
README.md
View file

@ -150,9 +150,9 @@ Then simply type this
compatibility) if this option is found
inside the system configuration file, the
user configuration is not loaded
--config-location PATH Location of the configuration file; either
the path to the config or its containing
directory
--config-location PATH Location of the main configuration file;
either the path to the config or its
containing directory
--flat-playlist Do not extract the videos of a playlist,
only list them
--flat-videos Do not resolve the video urls
@ -316,6 +316,17 @@ Then simply type this
stdin), one URL per line. Lines starting
with '#', ';' or ']' are considered as
comments and ignored
-P, --paths TYPE:PATH The paths where the files should be
downloaded. Specify the type of file and
the path separated by a colon ":"
(supported: description|annotation|subtitle
|infojson|thumbnail). Additionally, you can
also provide "home" and "temp" paths. All
intermediary files are first downloaded to
the temp path and then the final files are
moved over to the home path after download
is finished. Note that this option is
ignored if --output is an absolute path
-o, --output TEMPLATE Output filename template, see "OUTPUT
TEMPLATE" for details
--autonumber-start NUMBER Specify the start value for %(autonumber)s
@ -651,8 +662,9 @@ Then simply type this
You can configure youtube-dlc by placing any supported command line option to a configuration file. The configuration is loaded from the following locations:
1. The file given by `--config-location`
1. **Main Configuration**: The file given by `--config-location`
1. **Portable Configuration**: `yt-dlp.conf` or `youtube-dlc.conf` in the same directory as the bundled binary. If you are running from source-code (`<root dir>/youtube_dlc/__main__.py`), the root directory is used instead.
1. **Home Configuration**: `yt-dlp.conf` or `youtube-dlc.conf` in the home path given by `-P "home:<path>"`, or in the current directory if no such path is given
1. **User Configuration**:
* `%XDG_CONFIG_HOME%/yt-dlp/config` (recommended on Linux/macOS)
* `%XDG_CONFIG_HOME%/yt-dlp.conf`
@ -710,7 +722,7 @@ set HOME=%USERPROFILE%
# OUTPUT TEMPLATE
The `-o` option allows users to indicate a template for the output file names.
The `-o` option is used to indicate a template for the output file names while `-P` option is used to specify the path each type of file should be saved to.
**tl;dr:** [navigate me to examples](#output-template-examples).

229
youtube_dlc/YoutubeDL.py
View file

@ -69,6 +69,7 @@ from .utils import (
iri_to_uri,
ISO3166Utils,
locked_file,
make_dir,
make_HTTPS_handler,
MaxDownloadsReached,
orderedSet,
@ -114,8 +115,9 @@ from .postprocessor import (
FFmpegFixupStretchedPP,
FFmpegMergerPP,
FFmpegPostProcessor,
FFmpegSubtitlesConvertorPP,
# FFmpegSubtitlesConvertorPP,
get_postprocessor,
MoveFilesAfterDownloadPP,
)
from .version import __version__
@ -257,6 +259,8 @@ class YoutubeDL(object):
postprocessors: A list of dictionaries, each with an entry
* key: The name of the postprocessor. See
youtube_dlc/postprocessor/__init__.py for a list.
* _after_move: Optional. If True, run this post_processor
after 'MoveFilesAfterDownload'
as well as any further keyword arguments for the
postprocessor.
post_hooks: A list of functions that get called as the final step
@ -369,6 +373,8 @@ class YoutubeDL(object):
params = None
_ies = []
_pps = []
_pps_end = []
__prepare_filename_warned = False
_download_retcode = None
_num_downloads = None
_playlist_level = 0
@ -382,6 +388,8 @@ class YoutubeDL(object):
self._ies = []
self._ies_instances = {}
self._pps = []
self._pps_end = []
self.__prepare_filename_warned = False
self._post_hooks = []
self._progress_hooks = []
self._download_retcode = 0
@ -483,8 +491,11 @@ class YoutubeDL(object):
pp_class = get_postprocessor(pp_def_raw['key'])
pp_def = dict(pp_def_raw)
del pp_def['key']
after_move = pp_def.get('_after_move', False)
if '_after_move' in pp_def:
del pp_def['_after_move']
pp = pp_class(self, **compat_kwargs(pp_def))
self.add_post_processor(pp)
self.add_post_processor(pp, after_move=after_move)
for ph in self.params.get('post_hooks', []):
self.add_post_hook(ph)
@ -536,8 +547,11 @@ class YoutubeDL(object):
for ie in gen_extractor_classes():
self.add_info_extractor(ie)
def add_post_processor(self, pp):
def add_post_processor(self, pp, after_move=False):
"""Add a PostProcessor object to the end of the chain."""
if after_move:
self._pps_end.append(pp)
else:
self._pps.append(pp)
pp.set_downloader(self)
@ -702,7 +716,7 @@ class YoutubeDL(object):
except UnicodeEncodeError:
self.to_screen('Deleting already existent file')
def prepare_filename(self, info_dict):
def prepare_filename(self, info_dict, warn=False):
"""Generate the output filename."""
try:
template_dict = dict(info_dict)
@ -796,11 +810,33 @@ class YoutubeDL(object):
# to workaround encoding issues with subprocess on python2 @ Windows
if sys.version_info < (3, 0) and sys.platform == 'win32':
filename = encodeFilename(filename, True).decode(preferredencoding())
return sanitize_path(filename)
filename = sanitize_path(filename)
if warn and not self.__prepare_filename_warned:
if not self.params.get('paths'):
pass
elif filename == '-':
self.report_warning('--paths is ignored when an outputting to stdout')
elif os.path.isabs(filename):
self.report_warning('--paths is ignored since an absolute path is given in output template')
self.__prepare_filename_warned = True
return filename
except ValueError as err:
self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
return None
def prepare_filepath(self, filename, dir_type=''):
if filename == '-':
return filename
paths = self.params.get('paths', {})
assert isinstance(paths, dict)
homepath = expand_path(paths.get('home', '').strip())
assert isinstance(homepath, compat_str)
subdir = expand_path(paths.get(dir_type, '').strip()) if dir_type else ''
assert isinstance(subdir, compat_str)
return sanitize_path(os.path.join(homepath, subdir, filename))
def _match_entry(self, info_dict, incomplete):
""" Returns None if the file should be downloaded """
@ -972,7 +1008,8 @@ class YoutubeDL(object):
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
or extract_flat is True):
self.__forced_printings(
ie_result, self.prepare_filename(ie_result),
ie_result,
self.prepare_filepath(self.prepare_filename(ie_result)),
incomplete=True)
return ie_result
@ -1890,6 +1927,8 @@ class YoutubeDL(object):
assert info_dict.get('_type', 'video') == 'video'
info_dict.setdefault('__postprocessors', [])
max_downloads = self.params.get('max_downloads')
if max_downloads is not None:
if self._num_downloads >= int(max_downloads):
@ -1906,10 +1945,13 @@ class YoutubeDL(object):
self._num_downloads += 1
info_dict['_filename'] = filename = self.prepare_filename(info_dict)
filename = self.prepare_filename(info_dict, warn=True)
info_dict['_filename'] = full_filename = self.prepare_filepath(filename)
temp_filename = self.prepare_filepath(filename, 'temp')
files_to_move = {}
# Forced printings
self.__forced_printings(info_dict, filename, incomplete=False)
self.__forced_printings(info_dict, full_filename, incomplete=False)
if self.params.get('simulate', False):
if self.params.get('force_write_download_archive', False):
@ -1922,20 +1964,19 @@ class YoutubeDL(object):
return
def ensure_dir_exists(path):
try:
dn = os.path.dirname(path)
if dn and not os.path.exists(dn):
os.makedirs(dn)
return True
except (OSError, IOError) as err:
self.report_error('unable to create directory ' + error_to_compat_str(err))
return False
return make_dir(path, self.report_error)
if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
if not ensure_dir_exists(encodeFilename(full_filename)):
return
if not ensure_dir_exists(encodeFilename(temp_filename)):
return
if self.params.get('writedescription', False):
descfn = replace_extension(filename, 'description', info_dict.get('ext'))
descfn = replace_extension(
self.prepare_filepath(filename, 'description'),
'description', info_dict.get('ext'))
if not ensure_dir_exists(encodeFilename(descfn)):
return
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
self.to_screen('[info] Video description is already present')
elif info_dict.get('description') is None:
@ -1950,7 +1991,11 @@ class YoutubeDL(object):
return
if self.params.get('writeannotations', False):
annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
annofn = replace_extension(
self.prepare_filepath(filename, 'annotation'),
'annotations.xml', info_dict.get('ext'))
if not ensure_dir_exists(encodeFilename(annofn)):
return
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)):
self.to_screen('[info] Video annotations are already present')
elif not info_dict.get('annotations'):
@ -1984,9 +2029,13 @@ class YoutubeDL(object):
# ie = self.get_info_extractor(info_dict['extractor_key'])
for sub_lang, sub_info in subtitles.items():
sub_format = sub_info['ext']
sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext'))
sub_filename = subtitles_filename(temp_filename, sub_lang, sub_format, info_dict.get('ext'))
sub_filename_final = subtitles_filename(
self.prepare_filepath(filename, 'subtitle'),
sub_lang, sub_format, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
files_to_move[sub_filename] = sub_filename_final
else:
self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
if sub_info.get('data') is not None:
@ -1995,6 +2044,7 @@ class YoutubeDL(object):
# See https://github.com/ytdl-org/youtube-dl/issues/10268
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
subfile.write(sub_info['data'])
files_to_move[sub_filename] = sub_filename_final
except (OSError, IOError):
self.report_error('Cannot write subtitles file ' + sub_filename)
return
@ -2010,6 +2060,7 @@ class YoutubeDL(object):
with io.open(encodeFilename(sub_filename), 'wb') as subfile:
subfile.write(sub_data)
'''
files_to_move[sub_filename] = sub_filename_final
except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_warning('Unable to download subtitle for "%s": %s' %
(sub_lang, error_to_compat_str(err)))
@ -2017,15 +2068,14 @@ class YoutubeDL(object):
if self.params.get('skip_download', False):
if self.params.get('convertsubtitles', False):
subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
# subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
filename_real_ext = os.path.splitext(filename)[1][1:]
filename_wo_ext = (
os.path.splitext(filename)[0]
os.path.splitext(full_filename)[0]
if filename_real_ext == info_dict['ext']
else filename)
else full_filename)
afilename = '%s.%s' % (filename_wo_ext, self.params.get('convertsubtitles'))
if subconv.available:
info_dict.setdefault('__postprocessors', [])
# if subconv.available:
# info_dict['__postprocessors'].append(subconv)
if os.path.exists(encodeFilename(afilename)):
self.to_screen(
@ -2033,13 +2083,17 @@ class YoutubeDL(object):
'converted' % afilename)
else:
try:
self.post_process(filename, info_dict)
self.post_process(full_filename, info_dict, files_to_move)
except (PostProcessingError) as err:
self.report_error('postprocessing: %s' % str(err))
return
if self.params.get('writeinfojson', False):
infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
infofn = replace_extension(
self.prepare_filepath(filename, 'infojson'),
'info.json', info_dict.get('ext'))
if not ensure_dir_exists(encodeFilename(infofn)):
return
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
self.to_screen('[info] Video description metadata is already present')
else:
@ -2050,7 +2104,9 @@ class YoutubeDL(object):
self.report_error('Cannot write metadata to JSON file ' + infofn)
return
self._write_thumbnails(info_dict, filename)
thumbdir = os.path.dirname(self.prepare_filepath(filename, 'thumbnail'))
for thumbfn in self._write_thumbnails(info_dict, temp_filename):
files_to_move[thumbfn] = os.path.join(thumbdir, os.path.basename(thumbfn))
# Write internet shortcut files
url_link = webloc_link = desktop_link = False
@ -2075,7 +2131,7 @@ class YoutubeDL(object):
ascii_url = iri_to_uri(info_dict['webpage_url'])
def _write_link_file(extension, template, newline, embed_filename):
linkfn = replace_extension(filename, extension, info_dict.get('ext'))
linkfn = replace_extension(full_filename, extension, info_dict.get('ext'))
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(linkfn)):
self.to_screen('[info] Internet shortcut is already present')
else:
@ -2105,9 +2161,27 @@ class YoutubeDL(object):
must_record_download_archive = False
if not self.params.get('skip_download', False):
try:
def existing_file(filename, temp_filename):
file_exists = os.path.exists(encodeFilename(filename))
tempfile_exists = (
False if temp_filename == filename
else os.path.exists(encodeFilename(temp_filename)))
if not self.params.get('overwrites', False) and (file_exists or tempfile_exists):
existing_filename = temp_filename if tempfile_exists else filename
self.to_screen('[download] %s has already been downloaded and merged' % existing_filename)
return existing_filename
if tempfile_exists:
self.report_file_delete(temp_filename)
os.remove(encodeFilename(temp_filename))
if file_exists:
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
return None
success = True
if info_dict.get('requested_formats') is not None:
downloaded = []
success = True
merger = FFmpegMergerPP(self)
if not merger.available:
postprocessors = []
@ -2136,32 +2210,31 @@ class YoutubeDL(object):
# TODO: Check acodec/vcodec
return False
filename_real_ext = os.path.splitext(filename)[1][1:]
filename_wo_ext = (
os.path.splitext(filename)[0]
if filename_real_ext == info_dict['ext']
else filename)
requested_formats = info_dict['requested_formats']
old_ext = info_dict['ext']
if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
info_dict['ext'] = 'mkv'
self.report_warning(
'Requested formats are incompatible for merge and will be merged into mkv.')
def correct_ext(filename):
filename_real_ext = os.path.splitext(filename)[1][1:]
filename_wo_ext = (
os.path.splitext(filename)[0]
if filename_real_ext == old_ext
else filename)
return '%s.%s' % (filename_wo_ext, info_dict['ext'])
# Ensure filename always has a correct extension for successful merge
filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
file_exists = os.path.exists(encodeFilename(filename))
if not self.params.get('overwrites', False) and file_exists:
self.to_screen(
'[download] %s has already been downloaded and '
'merged' % filename)
else:
if file_exists:
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
full_filename = correct_ext(full_filename)
temp_filename = correct_ext(temp_filename)
dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None:
for f in requested_formats:
new_info = dict(info_dict)
new_info.update(f)
fname = prepend_extension(
self.prepare_filename(new_info),
self.prepare_filepath(self.prepare_filename(new_info), 'temp'),
'f%s' % f['format_id'], new_info['ext'])
if not ensure_dir_exists(fname):
return
@ -2173,14 +2246,17 @@ class YoutubeDL(object):
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
# Delete existing file with --yes-overwrites
if self.params.get('overwrites', False):
if os.path.exists(encodeFilename(filename)):
self.report_file_delete(filename)
os.remove(encodeFilename(filename))
# Just a single file
success, real_download = dl(filename, info_dict)
dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None:
success, real_download = dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download
# info_dict['__temp_filename'] = temp_filename
dl_filename = dl_filename or temp_filename
info_dict['__dl_filename'] = dl_filename
info_dict['__final_filename'] = full_filename
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_error('unable to download video data: %s' % error_to_compat_str(err))
return
@ -2206,7 +2282,6 @@ class YoutubeDL(object):
elif fixup_policy == 'detect_or_warn':
stretched_pp = FFmpegFixupStretchedPP(self)
if stretched_pp.available:
info_dict.setdefault('__postprocessors', [])
info_dict['__postprocessors'].append(stretched_pp)
else:
self.report_warning(
@ -2225,7 +2300,6 @@ class YoutubeDL(object):
elif fixup_policy == 'detect_or_warn':
fixup_pp = FFmpegFixupM4aPP(self)
if fixup_pp.available:
info_dict.setdefault('__postprocessors', [])
info_dict['__postprocessors'].append(fixup_pp)
else:
self.report_warning(
@ -2244,7 +2318,6 @@ class YoutubeDL(object):
elif fixup_policy == 'detect_or_warn':
fixup_pp = FFmpegFixupM3u8PP(self)
if fixup_pp.available:
info_dict.setdefault('__postprocessors', [])
info_dict['__postprocessors'].append(fixup_pp)
else:
self.report_warning(
@ -2254,13 +2327,13 @@ class YoutubeDL(object):
assert fixup_policy in ('ignore', 'never')
try:
self.post_process(filename, info_dict)
self.post_process(dl_filename, info_dict, files_to_move)
except (PostProcessingError) as err:
self.report_error('postprocessing: %s' % str(err))
return
try:
for ph in self._post_hooks:
ph(filename)
ph(full_filename)
except Exception as err:
self.report_error('post hooks: %s' % str(err))
return
@ -2326,27 +2399,41 @@ class YoutubeDL(object):
(k, v) for k, v in info_dict.items()
if k not in ['requested_formats', 'requested_subtitles'])
def post_process(self, filename, ie_info):
def post_process(self, filename, ie_info, files_to_move={}):
"""Run all the postprocessors on the given file."""
info = dict(ie_info)
info['filepath'] = filename
pps_chain = []
if ie_info.get('__postprocessors') is not None:
pps_chain.extend(ie_info['__postprocessors'])
pps_chain.extend(self._pps)
for pp in pps_chain:
def run_pp(pp):
files_to_delete = []
infodict = info
try:
files_to_delete, info = pp.run(info)
files_to_delete, infodict = pp.run(infodict)
except PostProcessingError as e:
self.report_error(e.msg)
if files_to_delete and not self.params.get('keepvideo', False):
if not files_to_delete:
return infodict
if self.params.get('keepvideo', False):
for f in files_to_delete:
files_to_move.setdefault(f, '')
else:
for old_filename in set(files_to_delete):
self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
try:
os.remove(encodeFilename(old_filename))
except (IOError, OSError):
self.report_warning('Unable to remove downloaded original file')
if old_filename in files_to_move:
del files_to_move[old_filename]
return infodict
for pp in ie_info.get('__postprocessors', []) + self._pps:
info = run_pp(pp)
info = run_pp(MoveFilesAfterDownloadPP(self, files_to_move))
files_to_move = {}
for pp in self._pps_end:
info = run_pp(pp)
def _make_archive_id(self, info_dict):
video_id = info_dict.get('id')
@ -2700,14 +2787,11 @@ class YoutubeDL(object):
if thumbnails:
thumbnails = [thumbnails[-1]]
elif self.params.get('write_all_thumbnails', False):
thumbnails = info_dict.get('thumbnails')
thumbnails = info_dict.get('thumbnails') or []
else:
return
if not thumbnails:
# No thumbnails present, so return immediately
return
thumbnails = []
ret = []
for t in thumbnails:
thumb_ext = determine_ext(t['url'], 'jpg')
suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
@ -2715,6 +2799,7 @@ class YoutubeDL(object):
t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext'))
if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
ret.append(thumb_filename)
self.to_screen('[%s] %s: Thumbnail %sis already present' %
(info_dict['extractor'], info_dict['id'], thumb_display_id))
else:
@ -2724,8 +2809,10 @@ class YoutubeDL(object):
uf = self.urlopen(t['url'])
with open(encodeFilename(thumb_filename), 'wb') as thumbf:
shutil.copyfileobj(uf, thumbf)
ret.append(thumb_filename)
self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
(info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_warning('Unable to download thumbnail "%s": %s' %
(t['url'], error_to_compat_str(err)))
return ret

6
youtube_dlc/__init__.py
View file

@ -244,6 +244,7 @@ def _real_main(argv=None):
parser.error('Cannot download a video and extract audio into the same'
' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
' template'.format(outtmpl))
for f in opts.format_sort:
if re.match(InfoExtractor.FormatSort.regex, f) is None:
parser.error('invalid format sort string "%s" specified' % f)
@ -318,12 +319,12 @@ def _real_main(argv=None):
'force': opts.sponskrub_force,
'ignoreerror': opts.sponskrub is None,
})
# Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
# So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
# ExecAfterDownload must be the last PP
if opts.exec_cmd:
postprocessors.append({
'key': 'ExecAfterDownload',
'exec_cmd': opts.exec_cmd,
'_after_move': True
})
_args_compat_warning = 'WARNING: %s given without specifying name. The arguments will be given to all %s\n'
@ -372,6 +373,7 @@ def _real_main(argv=None):
'listformats': opts.listformats,
'listformats_table': opts.listformats_table,
'outtmpl': outtmpl,
'paths': opts.paths,
'autonumber_size': opts.autonumber_size,
'autonumber_start': opts.autonumber_start,
'restrictfilenames': opts.restrictfilenames,

106
youtube_dlc/options.py
View file

@ -14,6 +14,7 @@ from .compat import (
compat_shlex_split,
)
from .utils import (
expand_path,
preferredencoding,
write_string,
)
@ -62,7 +63,7 @@ def parseOpts(overrideArguments=None):
userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
userConf = _readOptions(userConfFile, default=None)
if userConf is not None:
return userConf
return userConf, userConfFile
# appdata
appdata_dir = compat_getenv('appdata')
@ -70,19 +71,21 @@ def parseOpts(overrideArguments=None):
userConfFile = os.path.join(appdata_dir, package_name, 'config')
userConf = _readOptions(userConfFile, default=None)
if userConf is None:
userConf = _readOptions('%s.txt' % userConfFile, default=None)
userConfFile += '.txt'
userConf = _readOptions(userConfFile, default=None)
if userConf is not None:
return userConf
return userConf, userConfFile
# home
userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
userConf = _readOptions(userConfFile, default=None)
if userConf is None:
userConf = _readOptions('%s.txt' % userConfFile, default=None)
userConfFile += '.txt'
userConf = _readOptions(userConfFile, default=None)
if userConf is not None:
return userConf
return userConf, userConfFile
return default
return default, None
def _format_option_string(option):
''' ('-o', '--option') -> -o, --format METAVAR'''
@ -187,7 +190,7 @@ def parseOpts(overrideArguments=None):
general.add_option(
'--config-location',
dest='config_location', metavar='PATH',
help='Location of the configuration file; either the path to the config or its containing directory')
help='Location of the main configuration file; either the path to the config or its containing directory')
general.add_option(
'--flat-playlist',
action='store_const', dest='extract_flat', const='in_playlist', default=False,
@ -819,6 +822,21 @@ def parseOpts(overrideArguments=None):
filesystem.add_option(
'--id', default=False,
action='store_true', dest='useid', help=optparse.SUPPRESS_HELP)
filesystem.add_option(
'-P', '--paths',
metavar='TYPE:PATH', dest='paths', default={}, type='str',
action='callback', callback=_dict_from_multiple_values_options_callback,
callback_kwargs={
'allowed_keys': 'home|temp|config|description|annotation|subtitle|infojson|thumbnail',
'process': lambda x: x.strip()},
help=(
'The paths where the files should be downloaded. '
'Specify the type of file and the path separated by a colon ":" '
'(supported: description|annotation|subtitle|infojson|thumbnail). '
'Additionally, you can also provide "home" and "temp" paths. '
'All intermediary files are first downloaded to the temp path and '
'then the final files are moved over to the home path after download is finished. '
'Note that this option is ignored if --output is an absolute path'))
filesystem.add_option(
'-o', '--output',
dest='outtmpl', metavar='TEMPLATE',
@ -1171,59 +1189,79 @@ def parseOpts(overrideArguments=None):
return conf
configs = {
'command_line': compat_conf(sys.argv[1:]),
'custom': [], 'portable': [], 'user': [], 'system': []}
opts, args = parser.parse_args(configs['command_line'])
'command-line': compat_conf(sys.argv[1:]),
'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []}
paths = {'command-line': False}
opts, args = parser.parse_args(configs['command-line'])
def get_configs():
if '--config-location' in configs['command_line']:
if '--config-location' in configs['command-line']:
location = compat_expanduser(opts.config_location)
if os.path.isdir(location):
location = os.path.join(location, 'youtube-dlc.conf')
if not os.path.exists(location):
parser.error('config-location %s does not exist.' % location)
configs['custom'] = _readOptions(location)
if '--ignore-config' in configs['command_line']:
configs['custom'] = _readOptions(location, default=None)
if configs['custom'] is None:
configs['custom'] = []
else:
paths['custom'] = location
if '--ignore-config' in configs['command-line']:
return
if '--ignore-config' in configs['custom']:
return
def read_options(path, user=False):
func = _readUserConf if user else _readOptions
current_path = os.path.join(path, 'yt-dlp.conf')
config = func(current_path, default=None)
if user:
config, current_path = config
if config is None:
current_path = os.path.join(path, 'youtube-dlc.conf')
config = func(current_path, default=None)
if user:
config, current_path = config
if config is None:
return [], None
return config, current_path
def get_portable_path():
path = os.path.dirname(sys.argv[0])
if os.path.abspath(sys.argv[0]) != os.path.abspath(sys.executable): # Not packaged
path = os.path.join(path, '..')
return os.path.abspath(path)
run_path = get_portable_path()
configs['portable'] = _readOptions(os.path.join(run_path, 'yt-dlp.conf'), default=None)
if configs['portable'] is None:
configs['portable'] = _readOptions(os.path.join(run_path, 'youtube-dlc.conf'))
configs['portable'], paths['portable'] = read_options(get_portable_path())
if '--ignore-config' in configs['portable']:
return
configs['system'] = _readOptions('/etc/yt-dlp.conf', default=None)
if configs['system'] is None:
configs['system'] = _readOptions('/etc/youtube-dlc.conf')
def get_home_path():
opts = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])[0]
return expand_path(opts.paths.get('home', '')).strip()
configs['home'], paths['home'] = read_options(get_home_path())
if '--ignore-config' in configs['home']:
return
configs['system'], paths['system'] = read_options('/etc')
if '--ignore-config' in configs['system']:
return
configs['user'] = _readUserConf('yt-dlp', default=None)
if configs['user'] is None:
configs['user'] = _readUserConf('youtube-dlc')
configs['user'], paths['user'] = read_options('', True)
if '--ignore-config' in configs['user']:
configs['system'] = []
configs['system'], paths['system'] = [], None
get_configs()
argv = configs['system'] + configs['user'] + configs['portable'] + configs['custom'] + configs['command_line']
argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line']
opts, args = parser.parse_args(argv)
if opts.verbose:
for conf_label, conf in (
('System config', configs['system']),
('User config', configs['user']),
('Portable config', configs['portable']),
('Custom config', configs['custom']),
('Command-line args', configs['command_line'])):
write_string('[debug] %s: %s\n' % (conf_label, repr(_hide_login_info(conf))))
for label in ('System', 'User', 'Portable', 'Home', 'Custom', 'Command-line'):
key = label.lower()
if paths.get(key) is None:
continue
if paths[key]:
write_string('[debug] %s config file: %s\n' % (label, paths[key]))
write_string('[debug] %s config: %s\n' % (label, repr(_hide_login_info(configs[key]))))
return parser, opts, args

2
youtube_dlc/postprocessor/__init__.py
View file

@ -17,6 +17,7 @@ from .ffmpeg import (
from .xattrpp import XAttrMetadataPP
from .execafterdownload import ExecAfterDownloadPP
from .metadatafromtitle import MetadataFromTitlePP
from .movefilesafterdownload import MoveFilesAfterDownloadPP
from .sponskrub import SponSkrubPP
@ -39,6 +40,7 @@ __all__ = [
'FFmpegVideoConvertorPP',
'FFmpegVideoRemuxerPP',
'MetadataFromTitlePP',
'MoveFilesAfterDownloadPP',
'SponSkrubPP',
'XAttrMetadataPP',
]

52
youtube_dlc/postprocessor/movefilesafterdownload.py Normal file
View file

@ -0,0 +1,52 @@
from __future__ import unicode_literals
import os
import shutil
from .common import PostProcessor
from ..utils import (
encodeFilename,
make_dir,
PostProcessingError,
)
from ..compat import compat_str
class MoveFilesAfterDownloadPP(PostProcessor):
def __init__(self, downloader, files_to_move):
PostProcessor.__init__(self, downloader)
self.files_to_move = files_to_move
@classmethod
def pp_key(cls):
return 'MoveFiles'
def run(self, info):
if info.get('__dl_filename') is None:
return [], info
self.files_to_move.setdefault(info['__dl_filename'], '')
outdir = os.path.dirname(os.path.abspath(encodeFilename(info['__final_filename'])))
for oldfile, newfile in self.files_to_move.items():
if not os.path.exists(encodeFilename(oldfile)):
self.report_warning('File "%s" cannot be found' % oldfile)
continue
if not newfile:
newfile = compat_str(os.path.join(outdir, os.path.basename(encodeFilename(oldfile))))
if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):
continue
if os.path.exists(encodeFilename(newfile)):
if self.get_param('overwrites', True):
self.report_warning('Replacing existing file "%s"' % newfile)
os.path.remove(encodeFilename(newfile))
else:
self.report_warning(
'Cannot move file "%s" out of temporary directory since "%s" already exists. '
% (oldfile, newfile))
continue
make_dir(newfile, PostProcessingError)
self.to_screen('Moving file "%s" to "%s"' % (oldfile, newfile))
shutil.move(oldfile, newfile) # os.rename cannot move between volumes
info['filepath'] = info['__final_filename']
return [], info

12
youtube_dlc/utils.py
View file

@ -5893,3 +5893,15 @@ _HEX_TABLE = '0123456789abcdef'
def random_uuidv4():
return re.sub(r'[xy]', lambda x: _HEX_TABLE[random.randint(0, 15)], 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx')
def make_dir(path, to_screen=None):
try:
dn = os.path.dirname(path)
if dn and not os.path.exists(dn):
os.makedirs(dn)
return True
except (OSError, IOError) as err:
if callable(to_screen) is not None:
to_screen('unable to create directory ' + error_to_compat_str(err))
return False