[ffmpeg] Download and merge in a single step if possible

This commit is contained in:
pukkandan 2021-05-23 03:47:44 +05:30
parent 8d68ab98a7
commit 18e674b4f6
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
4 changed files with 70 additions and 40 deletions

View file

@ -130,6 +130,7 @@ ### Differences in default behavior
* Youtube live chat (if available) is considered as a subtitle. Use `--sub-langs all,-live_chat` to download all subtitles except live chat. You can also use `--compat-options no-live-chat` to prevent live chat from downloading
* Youtube channel URLs are automatically redirected to `/video`. Append a `/featured` to the URL to download only the videos in the home page. If the channel does not have a videos tab, we try to download the equivalent `UU` playlist instead. Also, `/live` URLs raise an error if there are no live videos instead of silently downloading the entire channel. You may use `--compat-options no-youtube-channel-redirect` to revert all these redirections
* Unavailable videos are also listed for youtube playlists. Use `--compat-options no-youtube-unavailable-videos` to remove this
* If `ffmpeg` is used as the downloader, the downloading and merging of formats happen in a single step when possible. Use `--compat-options no-direct-merge` to revert this
For ease of use, a few more compat options are available:
* `--compat-options all`: Use all compat options

View file

@ -387,8 +387,9 @@ class YoutubeDL(object):
if True, otherwise use ffmpeg/avconv if False, otherwise
use downloader suggested by extractor if None.
compat_opts: Compatibility options. See "Differences in default behavior".
Note that only format-sort, format-spec, no-live-chat, no-attach-info-json
playlist-index, list-formats, no-youtube-channel-redirect
Note that only format-sort, format-spec, no-live-chat,
no-attach-info-json, playlist-index, list-formats,
no-direct-merge, no-youtube-channel-redirect,
and no-youtube-unavailable-videos works when used via the API
The following parameters are not used by YoutubeDL itself, they are used by
@ -2294,7 +2295,8 @@ def dl(self, name, info, subtitle=False, test=False):
if not test:
for ph in self._progress_hooks:
fd.add_progress_hook(ph)
self.write_debug('Invoking downloader on %r' % info.get('url'))
urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']])
self.write_debug('Invoking downloader on "%s"' % urls)
new_info = dict(info)
if new_info.get('http_headers') is None:
new_info['http_headers'] = self._calc_headers(new_info)
@ -2533,17 +2535,6 @@ def existing_file(*filepaths):
success = True
if info_dict.get('requested_formats') is not None:
downloaded = []
merger = FFmpegMergerPP(self)
if self.params.get('allow_unplayable_formats'):
self.report_warning(
'You have requested merging of multiple formats '
'while also allowing unplayable formats to be downloaded. '
'The formats won\'t be merged to prevent data corruption.')
elif not merger.available:
self.report_warning(
'You have requested merging of multiple formats but ffmpeg is not installed. '
'The formats won\'t be merged.')
def compatible_formats(formats):
# TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
@ -2591,27 +2582,57 @@ def correct_ext(filename):
temp_filename = correct_ext(temp_filename)
dl_filename = existing_file(full_filename, temp_filename)
info_dict['__real_download'] = False
if dl_filename is None:
for f in requested_formats:
new_info = dict(info_dict)
new_info.update(f)
fname = prepend_extension(
self.prepare_filename(new_info, 'temp'),
'f%s' % f['format_id'], new_info['ext'])
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
partial_success, real_download = self.dl(fname, new_info)
info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success
if merger.available and not self.params.get('allow_unplayable_formats'):
info_dict['__postprocessors'].append(merger)
info_dict['__files_to_merge'] = downloaded
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
for file in downloaded:
files_to_move[file] = None
_protocols = set(determine_protocol(f) for f in requested_formats)
if len(_protocols) == 1:
info_dict['protocol'] = _protocols.pop()
directly_mergable = (
'no-direct-merge' not in self.params.get('compat_opts', [])
and info_dict.get('protocol') is not None # All requested formats have same protocol
and not self.params.get('allow_unplayable_formats')
and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD')
if directly_mergable:
info_dict['url'] = requested_formats[0]['url']
# Treat it as a single download
dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None:
success, real_download = self.dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download
else:
downloaded = []
merger = FFmpegMergerPP(self)
if self.params.get('allow_unplayable_formats'):
self.report_warning(
'You have requested merging of multiple formats '
'while also allowing unplayable formats to be downloaded. '
'The formats won\'t be merged to prevent data corruption.')
elif not merger.available:
self.report_warning(
'You have requested merging of multiple formats but ffmpeg is not installed. '
'The formats won\'t be merged.')
if dl_filename is None:
for f in requested_formats:
new_info = dict(info_dict)
del new_info['requested_formats']
new_info.update(f)
fname = prepend_extension(
self.prepare_filename(new_info, 'temp'),
'f%s' % f['format_id'], new_info['ext'])
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
partial_success, real_download = self.dl(fname, new_info)
info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success
if merger.available and not self.params.get('allow_unplayable_formats'):
info_dict['__postprocessors'].append(merger)
info_dict['__files_to_merge'] = downloaded
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
for file in downloaded:
files_to_move[file] = None
else:
# Just a single file
dl_filename = existing_file(full_filename, temp_filename)

View file

@ -264,8 +264,8 @@ def parse_compat_opts():
return parsed_compat_opts
all_compat_opts = [
'filename', 'format-sort', 'abort-on-error', 'format-spec', 'multistreams',
'no-playlist-metafiles', 'no-live-chat', 'playlist-index', 'list-formats',
'filename', 'format-sort', 'abort-on-error', 'format-spec', 'no-playlist-metafiles',
'multistreams', 'no-live-chat', 'playlist-index', 'list-formats', 'no-direct-merge',
'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-attach-info-json',
]
compat_opts = parse_compat_opts()

View file

@ -346,7 +346,7 @@ def available(cls, path=None):
return FFmpegPostProcessor().available
def _call_downloader(self, tmpfilename, info_dict):
url = info_dict['url']
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
ffpp = FFmpegPostProcessor(downloader=self)
if not ffpp.available:
self.report_error('m3u8 download detected but ffmpeg could not be found. Please install')
@ -378,7 +378,7 @@ def _call_downloader(self, tmpfilename, info_dict):
# if end_time:
# args += ['-t', compat_str(end_time - start_time)]
if info_dict.get('http_headers') is not None and re.match(r'^https?://', url):
if info_dict.get('http_headers') is not None and re.match(r'^https?://', urls[0]):
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
headers = handle_youtubedl_headers(info_dict['http_headers'])
@ -436,7 +436,15 @@ def _call_downloader(self, tmpfilename, info_dict):
elif isinstance(conn, compat_str):
args += ['-rtmp_conn', conn]
args += ['-i', url, '-c', 'copy']
for url in urls:
args += ['-i', url]
args += ['-c', 'copy']
if info_dict.get('requested_formats'):
for (i, fmt) in enumerate(info_dict['requested_formats']):
if fmt.get('acodec') != 'none':
args.extend(['-map', '%d:a:0' % i])
if fmt.get('vcodec') != 'none':
args.extend(['-map', '%d:v:0' % i])
if self.params.get('test', False):
args += ['-fs', compat_str(self._TEST_FILE_SIZE)]