[postprocessor] Add plugin support

Adds option `--use-postprocessor` to enable them
This commit is contained in:
pukkandan 2021-09-30 02:23:33 +05:30
parent 8e3fd7e034
commit 3ae5e79774
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
11 changed files with 95 additions and 49 deletions

View file

@ -837,6 +837,20 @@ ## Post-Processing Options:
around the cuts
--no-force-keyframes-at-cuts Do not force keyframes around the chapters
when cutting/splitting (default)
--use-postprocessor NAME[:ARGS] The (case sensitive) name of plugin
postprocessors to be enabled, and
(optionally) arguments to be passed to it,
seperated by a colon ":". ARGS are a
semicolon ";" delimited list of NAME=VALUE.
The "when" argument determines when the
postprocessor is invoked. It can be one of
"pre_process" (after extraction),
"before_dl" (before video download),
"post_process" (after video download;
default) or "after_move" (after moving file
to their final locations). This option can
be used multiple times to add different
postprocessors
## SponsorBlock Options:
Make chapter entries for, or remove various segments (sponsor,
@ -1465,9 +1479,16 @@ # EXTRACTOR ARGUMENTS
# PLUGINS
Plugins are loaded from `<root-dir>/ytdlp_plugins/<type>/__init__.py`. Currently only `extractor` plugins are supported. Support for `downloader` and `postprocessor` plugins may be added in the future. See [ytdlp_plugins](ytdlp_plugins) for example.
Plugins are loaded from `<root-dir>/ytdlp_plugins/<type>/__init__.py`; where `<root-dir>` is the directory of the binary (`<root-dir>/yt-dlp`), or the root directory of the module if you are running directly from source-code (`<root dir>/yt_dlp/__main__.py`). Plugins are currently not supported for the `pip` version
Plugins can be of `<type>`s `extractor` or `postprocessor`. Extractor plugins do not need to be enabled from the CLI and are automatically invoked when the input URL is suitable for it. Postprocessor plugins can be invoked using `--use-postprocessor NAME`.
See [ytdlp_plugins](ytdlp_plugins) for example plugins.
Note that **all** plugins are imported even if not invoked, and that **there are no checks** performed on plugin code. Use plugins at your own risk and only if you trust the code
If you are a plugin author, add [ytdlp-plugins](https://github.com/topics/ytdlp-plugins) as a topic to your repository for discoverability
**Note**: `<root-dir>` is the directory of the binary (`<root-dir>/yt-dlp`), or the root directory of the module if you are running directly from source-code (`<root dir>/yt_dlp/__main__.py`)
# DEPRECATED OPTIONS

View file

@ -123,7 +123,7 @@
gen_extractor_classes,
get_info_extractor,
_LAZY_LOADER,
_PLUGIN_CLASSES
_PLUGIN_CLASSES as plugin_extractors
)
from .extractor.openload import PhantomJSwrapper
from .downloader import (
@ -142,6 +142,7 @@
FFmpegMergerPP,
FFmpegPostProcessor,
MoveFilesAfterDownloadPP,
_PLUGIN_CLASSES as plugin_postprocessors
)
from .update import detect_variant
from .version import __version__
@ -3201,9 +3202,10 @@ def print_debug_header(self):
self._write_string('[debug] yt-dlp version %s%s\n' % (__version__, '' if source == 'unknown' else f' ({source})'))
if _LAZY_LOADER:
self._write_string('[debug] Lazy loading extractors enabled\n')
if _PLUGIN_CLASSES:
self._write_string(
'[debug] Plugin Extractors: %s\n' % [ie.ie_key() for ie in _PLUGIN_CLASSES])
if plugin_extractors or plugin_postprocessors:
self._write_string('[debug] Plugins: %s\n' % [
'%s%s' % (klass.__name__, '' if klass.__name__ == name else f' as {name}')
for name, klass in itertools.chain(plugin_extractors.items(), plugin_postprocessors.items())])
if self.params.get('compat_opts'):
self._write_string(
'[debug] Compatibility options: %s\n' % ', '.join(self.params.get('compat_opts')))

View file

@ -418,7 +418,7 @@ def report_conflict(arg1, arg2):
opts.sponskrub = False
# PostProcessors
postprocessors = []
postprocessors = list(opts.add_postprocessors)
if sponsorblock_query:
postprocessors.append({
'key': 'SponsorBlock',

View file

@ -6,7 +6,7 @@
from .lazy_extractors import *
from .lazy_extractors import _ALL_CLASSES
_LAZY_LOADER = True
_PLUGIN_CLASSES = []
_PLUGIN_CLASSES = {}
except ImportError:
_LAZY_LOADER = False
@ -20,7 +20,7 @@
_ALL_CLASSES.append(GenericIE)
_PLUGIN_CLASSES = load_plugins('extractor', 'IE', globals())
_ALL_CLASSES = _PLUGIN_CLASSES + _ALL_CLASSES
_ALL_CLASSES = list(_PLUGIN_CLASSES.values()) + _ALL_CLASSES
def gen_extractor_classes():

View file

@ -17,6 +17,7 @@
get_executable_path,
OUTTMPL_TYPES,
preferredencoding,
remove_end,
write_string,
)
from .cookies import SUPPORTED_BROWSERS
@ -1389,6 +1390,25 @@ def _dict_from_options_callback(
'--no-force-keyframes-at-cuts',
action='store_false', dest='force_keyframes_at_cuts',
help='Do not force keyframes around the chapters when cutting/splitting (default)')
_postprocessor_opts_parser = lambda key, val='': (
*(item.split('=', 1) for item in (val.split(';') if val else [])),
('key', remove_end(key, 'PP')))
postproc.add_option(
'--use-postprocessor',
metavar='NAME[:ARGS]', dest='add_postprocessors', default=[], type='str',
action='callback', callback=_list_from_options_callback,
callback_kwargs={
'delim': None,
'process': lambda val: dict(_postprocessor_opts_parser(*val.split(':', 1)))
}, help=(
'The (case sensitive) name of plugin postprocessors to be enabled, '
'and (optionally) arguments to be passed to it, seperated by a colon ":". '
'ARGS are a semicolon ";" delimited list of NAME=VALUE. '
'The "when" argument determines when the postprocessor is invoked. '
'It can be one of "pre_process" (after extraction), '
'"before_dl" (before video download), "post_process" (after video download; default) '
'or "after_move" (after moving file to their final locations). '
'This option can be used multiple times to add different postprocessors'))
sponsorblock = optparse.OptionGroup(parser, 'SponsorBlock Options', description=(
'Make chapter entries for, or remove various segments (sponsor, introductions, etc.) '

View file

@ -1,6 +1,9 @@
from __future__ import unicode_literals
# flake8: noqa: F401
from ..utils import load_plugins
from .embedthumbnail import EmbedThumbnailPP
from .exec import ExecPP, ExecAfterDownloadPP
from .ffmpeg import (
FFmpegPostProcessor,
FFmpegEmbedSubtitlePP,
@ -18,48 +21,23 @@
FFmpegVideoConvertorPP,
FFmpegVideoRemuxerPP,
)
from .xattrpp import XAttrMetadataPP
from .exec import ExecPP, ExecAfterDownloadPP
from .metadataparser import (
MetadataFromFieldPP,
MetadataFromTitlePP,
MetadataParserPP,
)
from .movefilesafterdownload import MoveFilesAfterDownloadPP
from .sponsorblock import SponsorBlockPP
from .sponskrub import SponSkrubPP
from .modify_chapters import ModifyChaptersPP
from .movefilesafterdownload import MoveFilesAfterDownloadPP
from .sponskrub import SponSkrubPP
from .sponsorblock import SponsorBlockPP
from .xattrpp import XAttrMetadataPP
_PLUGIN_CLASSES = load_plugins('postprocessor', 'PP', globals())
def get_postprocessor(key):
return globals()[key + 'PP']
__all__ = [
'FFmpegPostProcessor',
'EmbedThumbnailPP',
'ExecPP',
'ExecAfterDownloadPP',
'FFmpegEmbedSubtitlePP',
'FFmpegExtractAudioPP',
'FFmpegSplitChaptersPP',
'FFmpegFixupDurationPP',
'FFmpegFixupM3u8PP',
'FFmpegFixupM4aPP',
'FFmpegFixupStretchedPP',
'FFmpegFixupTimestampPP',
'FFmpegMergerPP',
'FFmpegMetadataPP',
'FFmpegSubtitlesConvertorPP',
'FFmpegThumbnailsConvertorPP',
'FFmpegVideoConvertorPP',
'FFmpegVideoRemuxerPP',
'MetadataParserPP',
'MetadataFromFieldPP',
'MetadataFromTitlePP',
'MoveFilesAfterDownloadPP',
'SponsorBlockPP',
'SponSkrubPP',
'ModifyChaptersPP',
'XAttrMetadataPP',
]
__all__ = [name for name in globals().keys() if name.endswith('IE')]
__all__.append('FFmpegPostProcessor')

View file

@ -6278,7 +6278,7 @@ def get_executable_path():
def load_plugins(name, suffix, namespace):
plugin_info = [None]
classes = []
classes = {}
try:
plugin_info = imp.find_module(
name, [os.path.join(get_executable_path(), 'ytdlp_plugins')])
@ -6289,8 +6289,7 @@ def load_plugins(name, suffix, namespace):
if not name.endswith(suffix):
continue
klass = getattr(plugins, name)
classes.append(klass)
namespace[name] = klass
classes[name] = namespace[name] = klass
except ImportError:
pass
finally:

View file

@ -1,3 +1,4 @@
# flake8: noqa
# flake8: noqa: F401
# The imported name must end in "IE"
from .sample import SamplePluginIE

View file

@ -1,7 +1,5 @@
# coding: utf-8
from __future__ import unicode_literals
# ⚠ Don't use relative imports
from yt_dlp.extractor.common import InfoExtractor

View file

@ -0,0 +1,4 @@
# flake8: noqa: F401
# The imported name must end in "PP" and is the name to be used in --use-postprocessor
from .sample import SamplePluginPP

View file

@ -0,0 +1,23 @@
# coding: utf-8
# ⚠ Don't use relative imports
from yt_dlp.postprocessor.common import PostProcessor
# See the docstring of yt_dlp.postprocessor.common.PostProcessor
class SamplePluginPP(PostProcessor):
def __init__(self, downloader=None, **kwargs):
# ⚠ Only kwargs can be passed from the CLI, and all argument values will be string
# Also, "downloader", "when" and "key" are reserved names
super().__init__(downloader)
self._kwargs = kwargs
# See docstring of yt_dlp.postprocessor.common.PostProcessor.run
def run(self, info):
filepath = info.get('filepath')
if filepath: # PP was called after download (default)
self.to_screen(f'Post-processed {filepath!r} with {self._kwargs}')
else: # PP was called before actual download
filepath = info.get('_filename')
self.to_screen(f'Pre-processed {filepath!r} with {self._kwargs}')
return [], info # return list_of_files_to_delete, info_dict