Allow multiple and nested configuration files
This commit is contained in:
pukkandan
parent
b62fa6d75f
commit
06e57990f7
4 changed files with 172 additions and 141 deletions
|
|
@ -1,26 +0,0 @@
|
|||
# coding: utf-8
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from yt_dlp.options import _hide_login_info
|
||||
|
||||
|
||||
class TestOptions(unittest.TestCase):
|
||||
def test_hide_login_info(self):
|
||||
self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']),
|
||||
['-u', 'PRIVATE', '-p', 'PRIVATE'])
|
||||
self.assertEqual(_hide_login_info(['-u']), ['-u'])
|
||||
self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']),
|
||||
['-u', 'PRIVATE', '-u', 'PRIVATE'])
|
||||
self.assertEqual(_hide_login_info(['--username=foo']),
|
||||
['--username=PRIVATE'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -23,6 +23,7 @@ from yt_dlp.utils import (
|
|||
caesar,
|
||||
clean_html,
|
||||
clean_podcast_url,
|
||||
Config,
|
||||
date_from_str,
|
||||
datetime_from_str,
|
||||
DateRange,
|
||||
|
|
@ -1701,6 +1702,15 @@ Line 1
|
|||
self.assertEqual(format_bytes(1024**7), '1.00ZiB')
|
||||
self.assertEqual(format_bytes(1024**8), '1.00YiB')
|
||||
|
||||
def test_hide_login_info(self):
|
||||
self.assertEqual(Config.hide_login_info(['-u', 'foo', '-p', 'bar']),
|
||||
['-u', 'PRIVATE', '-p', 'PRIVATE'])
|
||||
self.assertEqual(Config.hide_login_info(['-u']), ['-u'])
|
||||
self.assertEqual(Config.hide_login_info(['-u', 'foo', '-u', 'bar']),
|
||||
['-u', 'PRIVATE', '-u', 'PRIVATE'])
|
||||
self.assertEqual(Config.hide_login_info(['--username=foo']),
|
||||
['--username=PRIVATE'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -13,11 +13,11 @@ from .compat import (
|
|||
compat_shlex_split,
|
||||
)
|
||||
from .utils import (
|
||||
Config,
|
||||
expand_path,
|
||||
get_executable_path,
|
||||
OUTTMPL_TYPES,
|
||||
POSTPROCESS_WHEN,
|
||||
preferredencoding,
|
||||
remove_end,
|
||||
write_string,
|
||||
)
|
||||
|
|
@ -35,39 +35,16 @@ from .postprocessor import (
|
|||
from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE
|
||||
|
||||
|
||||
def _hide_login_info(opts):
|
||||
PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
|
||||
eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
|
||||
def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
|
||||
parser = create_parser()
|
||||
root = Config(parser)
|
||||
|
||||
def _scrub_eq(o):
|
||||
m = eqre.match(o)
|
||||
if m:
|
||||
return m.group('key') + '=PRIVATE'
|
||||
if ignore_config_files == 'if_override':
|
||||
ignore_config_files = overrideArguments is not None
|
||||
if overrideArguments:
|
||||
root.append_config(overrideArguments, label='Override')
|
||||
else:
|
||||
return o
|
||||
|
||||
opts = list(map(_scrub_eq, opts))
|
||||
for idx, opt in enumerate(opts):
|
||||
if opt in PRIVATE_OPTS and idx + 1 < len(opts):
|
||||
opts[idx + 1] = 'PRIVATE'
|
||||
return opts
|
||||
|
||||
|
||||
def parseOpts(overrideArguments=None):
|
||||
def _readOptions(filename_bytes, default=[]):
|
||||
try:
|
||||
optionf = open(filename_bytes)
|
||||
except IOError:
|
||||
return default # silently skip if file is not present
|
||||
try:
|
||||
# FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
|
||||
contents = optionf.read()
|
||||
if sys.version_info < (3,):
|
||||
contents = contents.decode(preferredencoding())
|
||||
res = compat_shlex_split(contents, comments=True)
|
||||
finally:
|
||||
optionf.close()
|
||||
return res
|
||||
root.append_config(sys.argv[1:], label='Command-line')
|
||||
|
||||
def _readUserConf(package_name, default=[]):
|
||||
# .config
|
||||
|
|
@ -75,7 +52,7 @@ def parseOpts(overrideArguments=None):
|
|||
userConfFile = os.path.join(xdg_config_home, package_name, 'config')
|
||||
if not os.path.isfile(userConfFile):
|
||||
userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
|
||||
userConf = _readOptions(userConfFile, default=None)
|
||||
userConf = Config.read_file(userConfFile, default=None)
|
||||
if userConf is not None:
|
||||
return userConf, userConfFile
|
||||
|
||||
|
|
@ -83,24 +60,64 @@ def parseOpts(overrideArguments=None):
|
|||
appdata_dir = compat_getenv('appdata')
|
||||
if appdata_dir:
|
||||
userConfFile = os.path.join(appdata_dir, package_name, 'config')
|
||||
userConf = _readOptions(userConfFile, default=None)
|
||||
userConf = Config.read_file(userConfFile, default=None)
|
||||
if userConf is None:
|
||||
userConfFile += '.txt'
|
||||
userConf = _readOptions(userConfFile, default=None)
|
||||
userConf = Config.read_file(userConfFile, default=None)
|
||||
if userConf is not None:
|
||||
return userConf, userConfFile
|
||||
|
||||
# home
|
||||
userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
|
||||
userConf = _readOptions(userConfFile, default=None)
|
||||
userConf = Config.read_file(userConfFile, default=None)
|
||||
if userConf is None:
|
||||
userConfFile += '.txt'
|
||||
userConf = _readOptions(userConfFile, default=None)
|
||||
userConf = Config.read_file(userConfFile, default=None)
|
||||
if userConf is not None:
|
||||
return userConf, userConfFile
|
||||
|
||||
return default, None
|
||||
|
||||
def add_config(label, path, user=False):
|
||||
""" Adds config and returns whether to continue """
|
||||
if root.parse_args()[0].ignoreconfig:
|
||||
return False
|
||||
# Multiple package names can be given here
|
||||
# Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
|
||||
# the configuration file of any of these three packages
|
||||
for package in ('yt-dlp',):
|
||||
if user:
|
||||
args, current_path = _readUserConf(package, default=None)
|
||||
else:
|
||||
current_path = os.path.join(path, '%s.conf' % package)
|
||||
args = Config.read_file(current_path, default=None)
|
||||
if args is not None:
|
||||
root.append_config(args, current_path, label=label)
|
||||
return True
|
||||
return True
|
||||
|
||||
def load_configs():
|
||||
yield not ignore_config_files
|
||||
yield add_config('Portable', get_executable_path())
|
||||
yield add_config('Home', expand_path(root.parse_args()[0].paths.get('home', '')).strip())
|
||||
yield add_config('User', None, user=True)
|
||||
yield add_config('System', '/etc')
|
||||
|
||||
if all(load_configs()):
|
||||
# If ignoreconfig is found inside the system configuration file,
|
||||
# the user configuration is removed
|
||||
if root.parse_args()[0].ignoreconfig:
|
||||
user_conf = next((i for i, conf in enumerate(root.configs) if conf.label == 'User'), None)
|
||||
if user_conf is not None:
|
||||
root.configs.pop(user_conf)
|
||||
|
||||
opts, args = root.parse_args()
|
||||
if opts.verbose:
|
||||
write_string(f'\n{root}'.replace('\n| ', '\n[debug] ')[1:] + '\n')
|
||||
return parser, opts, args
|
||||
|
||||
|
||||
def create_parser():
|
||||
def _format_option_string(option):
|
||||
''' ('-o', '--option') -> -o, --format METAVAR'''
|
||||
|
||||
|
|
@ -244,14 +261,20 @@ def parseOpts(overrideArguments=None):
|
|||
'--ignore-config', '--no-config',
|
||||
action='store_true', dest='ignoreconfig',
|
||||
help=(
|
||||
'Disable loading any configuration files except the one provided by --config-location. '
|
||||
'When given inside a configuration file, no further configuration files are loaded. '
|
||||
'Additionally, (for backward compatibility) if this option is found inside the '
|
||||
'system configuration file, the user configuration is not loaded'))
|
||||
'Disable loading any further configuration files except the one provided by --config-locations. '
|
||||
'For backward compatibility, if this option is found inside the system configuration file, the user configuration is not loaded'))
|
||||
general.add_option(
|
||||
'--config-location',
|
||||
dest='config_location', metavar='PATH',
|
||||
help='Location of the main configuration file; either the path to the config or its containing directory')
|
||||
'--no-config-locations',
|
||||
action='store_const', dest='config_locations', const=[],
|
||||
help=(
|
||||
'Do not load any custom configuration files (default). When given inside a '
|
||||
'configuration file, ignore all previous --config-locations defined in the current file'))
|
||||
general.add_option(
|
||||
'--config-locations',
|
||||
dest='config_locations', metavar='PATH', action='append',
|
||||
help=(
|
||||
'Location of the main configuration file; either the path to the config or its containing directory. '
|
||||
'Can be used multiple times and inside other configuration files'))
|
||||
general.add_option(
|
||||
'--flat-playlist',
|
||||
action='store_const', dest='extract_flat', const='in_playlist', default=False,
|
||||
|
|
@ -1634,75 +1657,11 @@ def parseOpts(overrideArguments=None):
|
|||
parser.add_option_group(sponsorblock)
|
||||
parser.add_option_group(extractor)
|
||||
|
||||
if overrideArguments is not None:
|
||||
opts, args = parser.parse_args(overrideArguments)
|
||||
if opts.verbose:
|
||||
write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
|
||||
else:
|
||||
def compat_conf(conf):
|
||||
if sys.version_info < (3,):
|
||||
return [a.decode(preferredencoding(), 'replace') for a in conf]
|
||||
return conf
|
||||
return parser
|
||||
|
||||
configs = {
|
||||
'command-line': compat_conf(sys.argv[1:]),
|
||||
'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []}
|
||||
paths = {'command-line': False}
|
||||
|
||||
def read_options(name, path, user=False):
|
||||
''' loads config files and returns ignoreconfig '''
|
||||
# Multiple package names can be given here
|
||||
# Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for
|
||||
# the configuration file of any of these three packages
|
||||
for package in ('yt-dlp',):
|
||||
if user:
|
||||
config, current_path = _readUserConf(package, default=None)
|
||||
else:
|
||||
current_path = os.path.join(path, '%s.conf' % package)
|
||||
config = _readOptions(current_path, default=None)
|
||||
if config is not None:
|
||||
current_path = os.path.realpath(current_path)
|
||||
if current_path in paths.values():
|
||||
return False
|
||||
configs[name], paths[name] = config, current_path
|
||||
return parser.parse_args(config)[0].ignoreconfig
|
||||
return False
|
||||
|
||||
def get_configs():
|
||||
opts, _ = parser.parse_args(configs['command-line'])
|
||||
if opts.config_location is not None:
|
||||
location = compat_expanduser(opts.config_location)
|
||||
if os.path.isdir(location):
|
||||
location = os.path.join(location, 'yt-dlp.conf')
|
||||
if not os.path.exists(location):
|
||||
parser.error('config-location %s does not exist.' % location)
|
||||
config = _readOptions(location, default=None)
|
||||
if config:
|
||||
configs['custom'], paths['custom'] = config, location
|
||||
|
||||
if opts.ignoreconfig:
|
||||
return
|
||||
if parser.parse_args(configs['custom'])[0].ignoreconfig:
|
||||
return
|
||||
if read_options('portable', get_executable_path()):
|
||||
return
|
||||
opts, _ = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])
|
||||
if read_options('home', expand_path(opts.paths.get('home', '')).strip()):
|
||||
return
|
||||
if read_options('system', '/etc'):
|
||||
return
|
||||
if read_options('user', None, user=True):
|
||||
configs['system'], paths['system'] = [], None
|
||||
|
||||
get_configs()
|
||||
argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line']
|
||||
opts, args = parser.parse_args(argv)
|
||||
if opts.verbose:
|
||||
for label in ('Command-line', 'Custom', 'Portable', 'Home', 'User', 'System'):
|
||||
key = label.lower()
|
||||
if paths.get(key):
|
||||
write_string(f'[debug] {label} config file: {paths[key]}\n')
|
||||
if paths.get(key) is not None:
|
||||
write_string(f'[debug] {label} config: {_hide_login_info(configs[key])!r}\n')
|
||||
|
||||
return parser, opts, args
|
||||
def _hide_login_info(opts):
|
||||
write_string(
|
||||
'DeprecationWarning: "yt_dlp.options._hide_login_info" is deprecated and may be removed in a future version. '
|
||||
'Use "yt_dlp.utils.Config.hide_login_info" instead\n')
|
||||
return Config.hide_login_info(opts)
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ from .compat import (
|
|||
compat_kwargs,
|
||||
compat_os_name,
|
||||
compat_parse_qs,
|
||||
compat_shlex_split,
|
||||
compat_shlex_quote,
|
||||
compat_str,
|
||||
compat_struct_pack,
|
||||
|
|
@ -5100,3 +5101,90 @@ def join_nonempty(*values, delim='-', from_dict=None):
|
|||
if from_dict is not None:
|
||||
values = map(from_dict.get, values)
|
||||
return delim.join(map(str, filter(None, values)))
|
||||
|
||||
|
||||
class Config:
|
||||
own_args = None
|
||||
filename = None
|
||||
__initialized = False
|
||||
|
||||
def __init__(self, parser, label=None):
|
||||
self._parser, self.label = parser, label
|
||||
self._loaded_paths, self.configs = set(), []
|
||||
|
||||
def init(self, args=None, filename=None):
|
||||
assert not self.__initialized
|
||||
if filename:
|
||||
location = os.path.realpath(filename)
|
||||
if location in self._loaded_paths:
|
||||
return False
|
||||
self._loaded_paths.add(location)
|
||||
|
||||
self.__initialized = True
|
||||
self.own_args, self.filename = args, filename
|
||||
for location in self._parser.parse_args(args)[0].config_locations or []:
|
||||
location = compat_expanduser(location)
|
||||
if os.path.isdir(location):
|
||||
location = os.path.join(location, 'yt-dlp.conf')
|
||||
if not os.path.exists(location):
|
||||
self._parser.error(f'config location {location} does not exist')
|
||||
self.append_config(self.read_file(location), location)
|
||||
return True
|
||||
|
||||
def __str__(self):
|
||||
label = join_nonempty(
|
||||
self.label, 'config', f'"{self.filename}"' if self.filename else '',
|
||||
delim=' ')
|
||||
return join_nonempty(
|
||||
self.own_args is not None and f'{label[0].upper()}{label[1:]}: {self.hide_login_info(self.own_args)}',
|
||||
*(f'\n{c}'.replace('\n', '\n| ')[1:] for c in self.configs),
|
||||
delim='\n')
|
||||
|
||||
@staticmethod
|
||||
def read_file(filename, default=[]):
|
||||
try:
|
||||
optionf = open(filename)
|
||||
except IOError:
|
||||
return default # silently skip if file is not present
|
||||
try:
|
||||
# FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
|
||||
contents = optionf.read()
|
||||
if sys.version_info < (3,):
|
||||
contents = contents.decode(preferredencoding())
|
||||
res = compat_shlex_split(contents, comments=True)
|
||||
finally:
|
||||
optionf.close()
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def hide_login_info(opts):
|
||||
PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username'])
|
||||
eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
|
||||
|
||||
def _scrub_eq(o):
|
||||
m = eqre.match(o)
|
||||
if m:
|
||||
return m.group('key') + '=PRIVATE'
|
||||
else:
|
||||
return o
|
||||
|
||||
opts = list(map(_scrub_eq, opts))
|
||||
for idx, opt in enumerate(opts):
|
||||
if opt in PRIVATE_OPTS and idx + 1 < len(opts):
|
||||
opts[idx + 1] = 'PRIVATE'
|
||||
return opts
|
||||
|
||||
def append_config(self, *args, label=None):
|
||||
config = type(self)(self._parser, label)
|
||||
config._loaded_paths = self._loaded_paths
|
||||
if config.init(*args):
|
||||
self.configs.append(config)
|
||||
|
||||
@property
|
||||
def all_args(self):
|
||||
for config in reversed(self.configs):
|
||||
yield from config.all_args
|
||||
yield from self.own_args or []
|
||||
|
||||
def parse_args(self):
|
||||
return self._parser.parse_args(list(self.all_args))
|
||||
|
|
|
|||
Loading…
Reference in a new issue