[XAttrMetadata] Refactor and document dependencies
This commit is contained in:
pukkandan
parent
43d7f5a5d0
commit
6f7563beb7
5 changed files with 81 additions and 118 deletions
|
|
@ -273,6 +273,7 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
|
|||
* [**secretstorage**](https://github.com/mitya57/secretstorage) - For accessing the Gnome keyring while decrypting cookies of Chromium-based browsers on Linux. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
|
||||
* [**brotli**](https://github.com/google/brotli)\* or [**brotlicffi**](https://github.com/python-hyper/brotlicffi) - [Brotli](https://en.wikipedia.org/wiki/Brotli) content encoding support. Both licensed under MIT <sup>[1](https://github.com/google/brotli/blob/master/LICENSE) [2](https://github.com/python-hyper/brotlicffi/blob/master/LICENSE) </sup>
|
||||
* [**certifi**](https://github.com/certifi/python-certifi)\* - Provides Mozilla's root certificate bundle. Licensed under [MPLv2](https://github.com/certifi/python-certifi/blob/master/LICENSE)
|
||||
* [**xattr**](https://github.com/xattr/xattr), [**pyxattr**](https://github.com/iustin/pyxattr) or [**setfattr**](http://savannah.nongnu.org/projects/attr) - For writing xattr metadata on Linux. Licensed under [MIT](https://github.com/xattr/xattr/blob/master/LICENSE.txt), [LGPL2.1](https://github.com/iustin/pyxattr/blob/master/COPYING) and [GPLv2+](http://git.savannah.nongnu.org/cgit/attr.git/tree/doc/COPYING) respectively
|
||||
* [**AtomicParsley**](https://github.com/wez/atomicparsley) - For embedding thumbnail in mp4/m4a if mutagen/ffmpeg cannot. Licensed under [GPLv2+](https://github.com/wez/atomicparsley/blob/master/COPYING)
|
||||
* [**rtmpdump**](http://rtmpdump.mplayerhq.hu) - For downloading `rtmp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](http://rtmpdump.mplayerhq.hu)
|
||||
* [**mplayer**](http://mplayerhq.hu/design7/info.html) or [**mpv**](https://mpv.io) - For downloading `rstp` streams. ffmpeg will be used as a fallback. Licensed under [GPLv2+](https://github.com/mpv-player/mpv/blob/master/Copyright)
|
||||
|
|
|
|||
|
|
@ -75,6 +75,15 @@ except (ImportError, SyntaxError):
|
|||
websockets = None
|
||||
|
||||
|
||||
try:
|
||||
import xattr # xattr or pyxattr
|
||||
except ImportError:
|
||||
xattr = None
|
||||
else:
|
||||
if hasattr(xattr, 'set'): # pyxattr
|
||||
xattr._yt_dlp__identifier = 'pyxattr'
|
||||
|
||||
|
||||
all_dependencies = {k: v for k, v in globals().items() if not k.startswith('_')}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1422,7 +1422,7 @@ def create_parser():
|
|||
dest='parse_metadata', metavar='FIELDS REGEX REPLACE', action='append', nargs=3,
|
||||
help='Replace text in a metadata field using the given regex. This option can be used multiple times')
|
||||
postproc.add_option(
|
||||
'--xattrs',
|
||||
'--xattrs', '--xattr',
|
||||
action='store_true', dest='xattrs', default=False,
|
||||
help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
|
||||
postproc.add_option(
|
||||
|
|
|
|||
|
|
@ -12,68 +12,52 @@ from ..utils import (
|
|||
|
||||
|
||||
class XAttrMetadataPP(PostProcessor):
|
||||
#
|
||||
# More info about extended attributes for media:
|
||||
# http://freedesktop.org/wiki/CommonExtendedAttributes/
|
||||
# http://www.freedesktop.org/wiki/PhreedomDraft/
|
||||
# http://dublincore.org/documents/usageguide/elements.shtml
|
||||
#
|
||||
# TODO:
|
||||
# * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
|
||||
# * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
|
||||
#
|
||||
"""Set extended attributes on downloaded file (if xattr support is found)
|
||||
|
||||
More info about extended attributes for media:
|
||||
http://freedesktop.org/wiki/CommonExtendedAttributes/
|
||||
http://www.freedesktop.org/wiki/PhreedomDraft/
|
||||
http://dublincore.org/documents/usageguide/elements.shtml
|
||||
|
||||
TODO:
|
||||
* capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
|
||||
* figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
|
||||
"""
|
||||
|
||||
XATTR_MAPPING = {
|
||||
'user.xdg.referrer.url': 'webpage_url',
|
||||
# 'user.xdg.comment': 'description',
|
||||
'user.dublincore.title': 'title',
|
||||
'user.dublincore.date': 'upload_date',
|
||||
'user.dublincore.description': 'description',
|
||||
'user.dublincore.contributor': 'uploader',
|
||||
'user.dublincore.format': 'format',
|
||||
}
|
||||
|
||||
def run(self, info):
|
||||
""" Set extended attributes on downloaded file (if xattr support is found). """
|
||||
|
||||
# Write the metadata to the file's xattrs
|
||||
mtime = os.stat(info['filepath']).st_mtime
|
||||
self.to_screen('Writing metadata to file\'s xattrs')
|
||||
|
||||
filename = info['filepath']
|
||||
mtime = os.stat(filename).st_mtime
|
||||
|
||||
try:
|
||||
xattr_mapping = {
|
||||
'user.xdg.referrer.url': 'webpage_url',
|
||||
# 'user.xdg.comment': 'description',
|
||||
'user.dublincore.title': 'title',
|
||||
'user.dublincore.date': 'upload_date',
|
||||
'user.dublincore.description': 'description',
|
||||
'user.dublincore.contributor': 'uploader',
|
||||
'user.dublincore.format': 'format',
|
||||
}
|
||||
|
||||
num_written = 0
|
||||
for xattrname, infoname in xattr_mapping.items():
|
||||
|
||||
for xattrname, infoname in self.XATTR_MAPPING.items():
|
||||
value = info.get(infoname)
|
||||
|
||||
if value:
|
||||
if infoname == 'upload_date':
|
||||
value = hyphenate_date(value)
|
||||
|
||||
byte_value = value.encode('utf-8')
|
||||
write_xattr(filename, xattrname, byte_value)
|
||||
num_written += 1
|
||||
write_xattr(info['filepath'], xattrname, value.encode('utf-8'))
|
||||
|
||||
except XAttrUnavailableError as e:
|
||||
raise PostProcessingError(str(e))
|
||||
|
||||
except XAttrMetadataError as e:
|
||||
if e.reason == 'NO_SPACE':
|
||||
self.report_warning(
|
||||
'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. '
|
||||
+ (('Some ' if num_written else '') + 'extended attributes are not written.').capitalize())
|
||||
'Some extended attributes are not written')
|
||||
elif e.reason == 'VALUE_TOO_LONG':
|
||||
self.report_warning(
|
||||
'Unable to write extended attributes due to too long values.')
|
||||
self.report_warning('Unable to write extended attributes due to too long values.')
|
||||
else:
|
||||
msg = 'This filesystem doesn\'t support extended attributes. '
|
||||
if compat_os_name == 'nt':
|
||||
msg += 'You need to use NTFS.'
|
||||
else:
|
||||
msg += '(You may have to enable them in your /etc/fstab)'
|
||||
raise PostProcessingError(str(e))
|
||||
tip = ('You need to use NTFS' if compat_os_name == 'nt'
|
||||
else 'You may have to enable them in your "/etc/fstab"')
|
||||
raise PostProcessingError(f'This filesystem doesn\'t support extended attributes. {tip}')
|
||||
|
||||
self.try_utime(filename, mtime, mtime)
|
||||
self.try_utime(info['filepath'], mtime, mtime)
|
||||
return [], info
|
||||
|
|
|
|||
111
yt_dlp/utils.py
111
yt_dlp/utils.py
|
|
@ -4673,87 +4673,56 @@ def decode_png(png_data):
|
|||
|
||||
|
||||
def write_xattr(path, key, value):
|
||||
# This mess below finds the best xattr tool for the job
|
||||
try:
|
||||
# try the pyxattr module...
|
||||
import xattr
|
||||
# Windows: Write xattrs to NTFS Alternate Data Streams:
|
||||
# http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
|
||||
if compat_os_name == 'nt':
|
||||
assert ':' not in key
|
||||
assert os.path.exists(path)
|
||||
|
||||
if hasattr(xattr, 'set'): # pyxattr
|
||||
# Unicode arguments are not supported in python-pyxattr until
|
||||
# version 0.5.0
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/5498
|
||||
pyxattr_required_version = '0.5.0'
|
||||
if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
|
||||
# TODO: fallback to CLI tools
|
||||
raise XAttrUnavailableError(
|
||||
'python-pyxattr is detected but is too old. '
|
||||
'yt-dlp requires %s or above while your version is %s. '
|
||||
'Falling back to other xattr implementations' % (
|
||||
pyxattr_required_version, xattr.__version__))
|
||||
try:
|
||||
with open(f'{path}:{key}', 'wb') as f:
|
||||
f.write(value)
|
||||
except OSError as e:
|
||||
raise XAttrMetadataError(e.errno, e.strerror)
|
||||
return
|
||||
|
||||
# UNIX Method 1. Use xattrs/pyxattrs modules
|
||||
from .dependencies import xattr
|
||||
|
||||
setxattr = None
|
||||
if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
|
||||
# Unicode arguments are not supported in pyxattr until version 0.5.0
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/5498
|
||||
if version_tuple(xattr.__version__) >= (0, 5, 0):
|
||||
setxattr = xattr.set
|
||||
else: # xattr
|
||||
setxattr = xattr.setxattr
|
||||
elif xattr:
|
||||
setxattr = xattr.setxattr
|
||||
|
||||
if setxattr:
|
||||
try:
|
||||
setxattr(path, key, value)
|
||||
except OSError as e:
|
||||
raise XAttrMetadataError(e.errno, e.strerror)
|
||||
return
|
||||
|
||||
except ImportError:
|
||||
if compat_os_name == 'nt':
|
||||
# Write xattrs to NTFS Alternate Data Streams:
|
||||
# http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
|
||||
assert ':' not in key
|
||||
assert os.path.exists(path)
|
||||
# UNIX Method 2. Use setfattr/xattr executables
|
||||
exe = ('setfattr' if check_executable('setfattr', ['--version'])
|
||||
else 'xattr' if check_executable('xattr', ['-h']) else None)
|
||||
if not exe:
|
||||
raise XAttrUnavailableError(
|
||||
'Couldn\'t find a tool to set the xattrs. Install either the python "xattr" or "pyxattr" modules or the '
|
||||
+ ('"xattr" binary' if sys.platform != 'linux' else 'GNU "attr" package (which contains the "setfattr" tool)'))
|
||||
|
||||
ads_fn = path + ':' + key
|
||||
try:
|
||||
with open(ads_fn, 'wb') as f:
|
||||
f.write(value)
|
||||
except OSError as e:
|
||||
raise XAttrMetadataError(e.errno, e.strerror)
|
||||
else:
|
||||
user_has_setfattr = check_executable('setfattr', ['--version'])
|
||||
user_has_xattr = check_executable('xattr', ['-h'])
|
||||
|
||||
if user_has_setfattr or user_has_xattr:
|
||||
|
||||
value = value.decode('utf-8')
|
||||
if user_has_setfattr:
|
||||
executable = 'setfattr'
|
||||
opts = ['-n', key, '-v', value]
|
||||
elif user_has_xattr:
|
||||
executable = 'xattr'
|
||||
opts = ['-w', key, value]
|
||||
|
||||
cmd = ([encodeFilename(executable, True)]
|
||||
+ [encodeArgument(o) for o in opts]
|
||||
+ [encodeFilename(path, True)])
|
||||
|
||||
try:
|
||||
p = Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
|
||||
except OSError as e:
|
||||
raise XAttrMetadataError(e.errno, e.strerror)
|
||||
stdout, stderr = p.communicate_or_kill()
|
||||
stderr = stderr.decode('utf-8', 'replace')
|
||||
if p.returncode != 0:
|
||||
raise XAttrMetadataError(p.returncode, stderr)
|
||||
|
||||
else:
|
||||
# On Unix, and can't find pyxattr, setfattr, or xattr.
|
||||
if sys.platform.startswith('linux'):
|
||||
raise XAttrUnavailableError(
|
||||
"Couldn't find a tool to set the xattrs. "
|
||||
"Install either the python 'pyxattr' or 'xattr' "
|
||||
"modules, or the GNU 'attr' package "
|
||||
"(which contains the 'setfattr' tool).")
|
||||
else:
|
||||
raise XAttrUnavailableError(
|
||||
"Couldn't find a tool to set the xattrs. "
|
||||
"Install either the python 'xattr' module, "
|
||||
"or the 'xattr' binary.")
|
||||
value = value.decode('utf-8')
|
||||
try:
|
||||
p = Popen(
|
||||
[exe, '-w', key, value, path] if exe == 'xattr' else [exe, '-n', key, '-v', value, path],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
|
||||
except OSError as e:
|
||||
raise XAttrMetadataError(e.errno, e.strerror)
|
||||
stderr = p.communicate_or_kill()[1].decode('utf-8', 'replace')
|
||||
if p.returncode:
|
||||
raise XAttrMetadataError(p.returncode, stderr)
|
||||
|
||||
|
||||
def random_birthday(year_field, month_field, day_field):
|
||||
|
|
|
|||
Loading…
Reference in a new issue