2005-07-31 19:17:43 +00:00
|
|
|
#ifndef RUN_COMMAND_H
|
|
|
|
#define RUN_COMMAND_H
|
|
|
|
|
2010-03-09 20:00:36 +00:00
|
|
|
#ifndef NO_PTHREADS
|
2010-03-06 15:40:42 +00:00
|
|
|
#include <pthread.h>
|
|
|
|
#endif
|
|
|
|
|
2014-05-15 08:33:26 +00:00
|
|
|
#include "argv-array.h"
|
|
|
|
|
2007-03-10 08:28:00 +00:00
|
|
|
struct child_process {
|
|
|
|
const char **argv;
|
2014-05-15 08:33:26 +00:00
|
|
|
struct argv_array args;
|
2014-10-19 11:13:55 +00:00
|
|
|
struct argv_array env_array;
|
2007-03-10 08:28:05 +00:00
|
|
|
pid_t pid;
|
2008-02-21 22:42:56 +00:00
|
|
|
/*
|
|
|
|
* Using .in, .out, .err:
|
|
|
|
* - Specify 0 for no redirections (child inherits stdin, stdout,
|
|
|
|
* stderr from parent).
|
|
|
|
* - Specify -1 to have a pipe allocated as follows:
|
|
|
|
* .in: returns the writable pipe end; parent writes to it,
|
|
|
|
* the readable pipe end becomes child's stdin
|
|
|
|
* .out, .err: returns the readable pipe end; parent reads from
|
|
|
|
* it, the writable pipe end becomes child's stdout/stderr
|
|
|
|
* The caller of start_command() must close the returned FDs
|
|
|
|
* after it has completed reading from/writing to it!
|
|
|
|
* - Specify > 0 to set a channel to a particular FD as follows:
|
|
|
|
* .in: a readable FD, becomes child's stdin
|
|
|
|
* .out: a writable FD, becomes child's stdout/stderr
|
2010-02-05 20:57:37 +00:00
|
|
|
* .err: a writable FD, becomes child's stderr
|
2008-02-21 22:42:56 +00:00
|
|
|
* The specified FD is closed by start_command(), even in case
|
|
|
|
* of errors!
|
|
|
|
*/
|
2007-03-10 08:28:08 +00:00
|
|
|
int in;
|
2007-03-12 18:37:45 +00:00
|
|
|
int out;
|
2007-10-19 19:47:58 +00:00
|
|
|
int err;
|
2007-05-22 21:48:23 +00:00
|
|
|
const char *dir;
|
2007-05-22 21:48:47 +00:00
|
|
|
const char *const *env;
|
2007-03-10 08:28:00 +00:00
|
|
|
unsigned no_stdin:1;
|
2007-03-12 18:37:55 +00:00
|
|
|
unsigned no_stdout:1;
|
2007-11-11 07:29:37 +00:00
|
|
|
unsigned no_stderr:1;
|
2007-03-10 08:28:00 +00:00
|
|
|
unsigned git_cmd:1; /* if this is to be git sub-command */
|
2009-07-04 19:26:42 +00:00
|
|
|
unsigned silent_exec_failure:1;
|
2007-03-10 08:28:00 +00:00
|
|
|
unsigned stdout_to_stderr:1;
|
2009-12-30 10:53:16 +00:00
|
|
|
unsigned use_shell:1;
|
run-command: optionally kill children on exit
When we spawn a helper process, it should generally be done
and finish_command called before we exit. However, if we
exit abnormally due to an early return or a signal, the
helper may continue to run in our absence.
In the best case, this may simply be wasted CPU cycles or a
few stray messages on a terminal. But it could also mean a
process that the user thought was aborted continues to run
to completion (e.g., a push's pack-objects helper will
complete the push, even though you killed the push process).
This patch provides infrastructure for run-command to keep
track of PIDs to be killed, and clean them on signal
reception or input, just as we do with tempfiles. PIDs can
be added in two ways:
1. If NO_PTHREADS is defined, async helper processes are
automatically marked. By definition this code must be
ready to die when the parent dies, since it may be
implemented as a thread of the parent process.
2. If the run-command caller specifies the "clean_on_exit"
option. This is not the default, as there are cases
where it is OK for the child to outlive us (e.g., when
spawning a pager).
PIDs are cleared from the kill-list automatically during
wait_or_whine, which is called from finish_command and
finish_async.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Clemens Buchacher <drizzd@aon.at>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-01-07 11:42:43 +00:00
|
|
|
unsigned clean_on_exit:1;
|
2007-03-10 08:28:00 +00:00
|
|
|
};
|
|
|
|
|
2014-10-19 11:13:55 +00:00
|
|
|
#define CHILD_PROCESS_INIT { NULL, ARGV_ARRAY_INIT, ARGV_ARRAY_INIT }
|
2014-08-19 19:10:48 +00:00
|
|
|
void child_process_init(struct child_process *);
|
2015-10-24 12:11:27 +00:00
|
|
|
void child_process_clear(struct child_process *);
|
2014-08-19 19:09:35 +00:00
|
|
|
|
2007-03-10 08:28:05 +00:00
|
|
|
int start_command(struct child_process *);
|
|
|
|
int finish_command(struct child_process *);
|
pager: don't use unsafe functions in signal handlers
Since the commit a3da8821208d (pager: do wait_for_pager on signal
death), we call wait_for_pager() in the pager's signal handler. The
recent bug report revealed that this causes a deadlock in glibc at
aborting "git log" [*1*]. When this happens, git process is left
unterminated, and it can't be killed by SIGTERM but only by SIGKILL.
The problem is that wait_for_pager() function does more than waiting
for pager process's termination, but it does cleanups and printing
errors. Unfortunately, the functions that may be used in a signal
handler are very limited [*2*]. Particularly, malloc(), free() and the
variants can't be used in a signal handler because they take a mutex
internally in glibc. This was the cause of the deadlock above. Other
than the direct calls of malloc/free, many functions calling
malloc/free can't be used. strerror() is such one, either.
Also the usage of fflush() and printf() in a signal handler is bad,
although it seems working so far. In a safer side, we should avoid
them, too.
This patch tries to reduce the calls of such functions in signal
handlers. wait_for_signal() takes a flag and avoids the unsafe
calls. Also, finish_command_in_signal() is introduced for the
same reason. There the free() calls are removed, and only waits for
the children without whining at errors.
[*1*] https://bugzilla.opensuse.org/show_bug.cgi?id=942297
[*2*] http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_04_03
Signed-off-by: Takashi Iwai <tiwai@suse.de>
Reviewed-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-09-04 09:35:57 +00:00
|
|
|
int finish_command_in_signal(struct child_process *);
|
2007-03-10 08:28:00 +00:00
|
|
|
int run_command(struct child_process *);
|
|
|
|
|
2015-08-10 09:37:45 +00:00
|
|
|
/*
|
|
|
|
* Returns the path to the hook file, or NULL if the hook is missing
|
|
|
|
* or disabled. Note that this points to static storage that will be
|
|
|
|
* overwritten by further calls to find_hook and run_hook_*.
|
|
|
|
*/
|
2014-11-30 08:24:27 +00:00
|
|
|
extern const char *find_hook(const char *name);
|
2013-07-18 20:02:12 +00:00
|
|
|
LAST_ARG_MUST_BE_NULL
|
2014-03-18 10:00:53 +00:00
|
|
|
extern int run_hook_le(const char *const *env, const char *name, ...);
|
|
|
|
extern int run_hook_ve(const char *const *env, const char *name, va_list args);
|
|
|
|
|
2006-12-31 02:55:22 +00:00
|
|
|
#define RUN_COMMAND_NO_STDIN 1
|
2006-01-11 02:12:17 +00:00
|
|
|
#define RUN_GIT_CMD 2 /*If this is to be git sub-command */
|
2006-12-31 02:55:19 +00:00
|
|
|
#define RUN_COMMAND_STDOUT_TO_STDERR 4
|
2009-07-04 19:26:42 +00:00
|
|
|
#define RUN_SILENT_EXEC_FAILURE 8
|
2009-12-30 10:53:16 +00:00
|
|
|
#define RUN_USING_SHELL 16
|
2012-01-08 20:41:09 +00:00
|
|
|
#define RUN_CLEAN_ON_EXIT 32
|
2006-12-31 02:55:15 +00:00
|
|
|
int run_command_v_opt(const char **argv, int opt);
|
2007-05-23 20:21:39 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* env (the environment) is to be formatted like environ: "VAR=VALUE".
|
|
|
|
* To unset an environment variable use just "VAR".
|
|
|
|
*/
|
2007-05-22 21:48:47 +00:00
|
|
|
int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const char *const *env);
|
2005-07-31 19:17:43 +00:00
|
|
|
|
run-command: introduce capture_command helper
Something as simple as reading the stdout from a command
turns out to be rather hard to do right. Doing:
cmd.out = -1;
run_command(&cmd);
strbuf_read(&buf, cmd.out, 0);
can result in deadlock if the child process produces a large
amount of output. What happens is:
1. The parent spawns the child with its stdout connected
to a pipe, of which the parent is the sole reader.
2. The parent calls wait(), blocking until the child exits.
3. The child writes to stdout. If it writes more data than
the OS pipe buffer can hold, the write() call will
block.
This is a deadlock; the parent is waiting for the child to
exit, and the child is waiting for the parent to call
read().
So we might try instead:
start_command(&cmd);
strbuf_read(&buf, cmd.out, 0);
finish_command(&cmd);
But that is not quite right either. We are examining cmd.out
and running finish_command whether start_command succeeded
or not, which is wrong. Moreover, these snippets do not do
any error handling. If our read() fails, we must make sure
to still call finish_command (to reap the child process).
And both snippets failed to close the cmd.out descriptor,
which they must do (provided start_command succeeded).
Let's introduce a run-command helper that can make this a
bit simpler for callers to get right.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-03-23 03:53:43 +00:00
|
|
|
/**
|
|
|
|
* Execute the given command, capturing its stdout in the given strbuf.
|
|
|
|
* Returns -1 if starting the command fails or reading fails, and otherwise
|
|
|
|
* returns the exit code of the command. The output collected in the
|
|
|
|
* buffer is kept even if the command returns a non-zero exit. The hint field
|
|
|
|
* gives a starting size for the strbuf allocation.
|
|
|
|
*
|
|
|
|
* The fields of "cmd" should be set up as they would for a normal run_command
|
|
|
|
* invocation. But note that there is no need to set cmd->out; the function
|
|
|
|
* sets it up for the caller.
|
|
|
|
*/
|
|
|
|
int capture_command(struct child_process *cmd, struct strbuf *buf, size_t hint);
|
|
|
|
|
2007-10-19 19:48:00 +00:00
|
|
|
/*
|
|
|
|
* The purpose of the following functions is to feed a pipe by running
|
|
|
|
* a function asynchronously and providing output that the caller reads.
|
|
|
|
*
|
|
|
|
* It is expected that no synchronization and mutual exclusion between
|
|
|
|
* the caller and the feed function is necessary so that the function
|
|
|
|
* can run in a thread without interfering with the caller.
|
|
|
|
*/
|
|
|
|
struct async {
|
|
|
|
/*
|
2010-02-05 20:57:38 +00:00
|
|
|
* proc reads from in; closes it before return
|
|
|
|
* proc writes to out; closes it before return
|
2007-10-19 19:48:00 +00:00
|
|
|
* returns 0 on success, non-zero on failure
|
|
|
|
*/
|
2010-02-05 20:57:38 +00:00
|
|
|
int (*proc)(int in, int out, void *data);
|
2007-10-19 19:48:00 +00:00
|
|
|
void *data;
|
2010-02-05 20:57:38 +00:00
|
|
|
int in; /* caller writes here and closes it */
|
2007-10-19 19:48:00 +00:00
|
|
|
int out; /* caller reads from here and closes it */
|
2010-03-09 20:00:36 +00:00
|
|
|
#ifdef NO_PTHREADS
|
2007-10-19 19:48:00 +00:00
|
|
|
pid_t pid;
|
2007-12-08 21:19:14 +00:00
|
|
|
#else
|
2010-03-06 15:40:42 +00:00
|
|
|
pthread_t tid;
|
2010-02-05 20:57:38 +00:00
|
|
|
int proc_in;
|
|
|
|
int proc_out;
|
2007-12-08 21:19:14 +00:00
|
|
|
#endif
|
run-command: teach async threads to ignore SIGPIPE
Async processes can be implemented as separate forked
processes, or as threads (depending on the NO_PTHREADS
setting). In the latter case, if an async thread gets
SIGPIPE, it takes down the whole process. This is obviously
bad if the main process was not otherwise going to die, but
even if we were going to die, it means the main process does
not have a chance to report a useful error message.
There's also the small matter that forked async processes
will not take the main process down on a signal, meaning git
will behave differently depending on the NO_PTHREADS
setting.
This patch fixes it by adding a new flag to "struct async"
to block SIGPIPE just in the async thread. In theory, this
should always be on (which makes async threads behave more
like async processes), but we would first want to make sure
that each async process we spawn is careful about checking
return codes from write() and would not spew endlessly into
a dead pipe. So let's start with it as optional, and we can
enable it for specific sites in future patches.
The natural name for this option would be "ignore_sigpipe",
since that's what it does for the threaded case. But since
that name might imply that we are ignoring it in all cases
(including the separate-process one), let's call it
"isolate_sigpipe". What we are really asking for is
isolation. I.e., not to have our main process taken down by
signals spawned by the async process. How that is
implemented is up to the run-command code.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-04-19 22:49:41 +00:00
|
|
|
int isolate_sigpipe;
|
2007-10-19 19:48:00 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
int start_async(struct async *async);
|
|
|
|
int finish_async(struct async *async);
|
2015-09-01 20:22:43 +00:00
|
|
|
int in_async(void);
|
write_or_die: handle EPIPE in async threads
When write_or_die() sees EPIPE, it treats it specially by
converting it into a SIGPIPE death. We obviously cannot
ignore it, as the write has failed and the caller expects us
to die. But likewise, we cannot just call die(), because
printing any message at all would be a nuisance during
normal operations.
However, this is a problem if write_or_die() is called from
a thread. Our raised signal ends up killing the whole
process, when logically we just need to kill the thread
(after all, if we are ignoring SIGPIPE, there is good reason
to think that the main thread is expecting to handle it).
Inside an async thread, the die() code already does the
right thing, because we use our custom die_async() routine,
which calls pthread_join(). So ideally we would piggy-back
on that, and simply call:
die_quietly_with_code(141);
or similar. But refactoring the die code to do this is
surprisingly non-trivial. The die_routines themselves handle
both printing and the decision of the exit code. Every one
of them would have to be modified to take new parameters for
the code, and to tell us to be quiet.
Instead, we can just teach write_or_die() to check for the
async case and handle it specially. We do have to build an
interface to abstract the async exit, but it's simple and
self-contained. If we had many call-sites that wanted to do
this die_quietly_with_code(), this approach wouldn't scale
as well, but we don't. This is the only place where do this
weird exit trick.
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2016-02-24 07:40:16 +00:00
|
|
|
void NORETURN async_exit(int code);
|
2007-10-19 19:48:00 +00:00
|
|
|
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
/**
|
|
|
|
* This callback should initialize the child process and preload the
|
|
|
|
* error channel if desired. The preloading of is useful if you want to
|
|
|
|
* have a message printed directly before the output of the child process.
|
|
|
|
* pp_cb is the callback cookie as passed to run_processes_parallel.
|
|
|
|
* You can store a child process specific callback cookie in pp_task_cb.
|
|
|
|
*
|
|
|
|
* Even after returning 0 to indicate that there are no more processes,
|
|
|
|
* this function will be called again until there are no more running
|
|
|
|
* child processes.
|
|
|
|
*
|
|
|
|
* Return 1 if the next child is ready to run.
|
|
|
|
* Return 0 if there are currently no more tasks to be processed.
|
|
|
|
* To send a signal to other child processes for abortion,
|
|
|
|
* return the negative signal number.
|
|
|
|
*/
|
|
|
|
typedef int (*get_next_task_fn)(struct child_process *cp,
|
2016-03-01 02:07:16 +00:00
|
|
|
struct strbuf *out,
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
void *pp_cb,
|
|
|
|
void **pp_task_cb);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This callback is called whenever there are problems starting
|
|
|
|
* a new process.
|
|
|
|
*
|
|
|
|
* You must not write to stdout or stderr in this function. Add your
|
2016-03-01 02:07:16 +00:00
|
|
|
* message to the strbuf out instead, which will be printed without
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
* messing up the output of the other parallel processes.
|
|
|
|
*
|
|
|
|
* pp_cb is the callback cookie as passed into run_processes_parallel,
|
|
|
|
* pp_task_cb is the callback cookie as passed into get_next_task_fn.
|
|
|
|
*
|
|
|
|
* Return 0 to continue the parallel processing. To abort return non zero.
|
|
|
|
* To send a signal to other child processes for abortion, return
|
|
|
|
* the negative signal number.
|
|
|
|
*/
|
2016-03-01 02:07:16 +00:00
|
|
|
typedef int (*start_failure_fn)(struct strbuf *out,
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
void *pp_cb,
|
|
|
|
void *pp_task_cb);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This callback is called on every child process that finished processing.
|
|
|
|
*
|
|
|
|
* You must not write to stdout or stderr in this function. Add your
|
2016-03-01 02:07:16 +00:00
|
|
|
* message to the strbuf out instead, which will be printed without
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
* messing up the output of the other parallel processes.
|
|
|
|
*
|
|
|
|
* pp_cb is the callback cookie as passed into run_processes_parallel,
|
|
|
|
* pp_task_cb is the callback cookie as passed into get_next_task_fn.
|
|
|
|
*
|
|
|
|
* Return 0 to continue the parallel processing. To abort return non zero.
|
|
|
|
* To send a signal to other child processes for abortion, return
|
|
|
|
* the negative signal number.
|
|
|
|
*/
|
|
|
|
typedef int (*task_finished_fn)(int result,
|
2016-03-01 02:07:16 +00:00
|
|
|
struct strbuf *out,
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
void *pp_cb,
|
|
|
|
void *pp_task_cb);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Runs up to n processes at the same time. Whenever a process can be
|
|
|
|
* started, the callback get_next_task_fn is called to obtain the data
|
|
|
|
* required to start another child process.
|
|
|
|
*
|
|
|
|
* The children started via this function run in parallel. Their output
|
|
|
|
* (both stdout and stderr) is routed to stderr in a manner that output
|
|
|
|
* from different tasks does not interleave.
|
|
|
|
*
|
2016-02-29 21:57:06 +00:00
|
|
|
* start_failure_fn and task_finished_fn can be NULL to omit any
|
|
|
|
* special handling.
|
run-command: add an asynchronous parallel child processor
This allows to run external commands in parallel with ordered output
on stderr.
If we run external commands in parallel we cannot pipe the output directly
to the our stdout/err as it would mix up. So each process's output will
flow through a pipe, which we buffer. One subprocess can be directly
piped to out stdout/err for a low latency feedback to the user.
Example:
Let's assume we have 5 submodules A,B,C,D,E and each fetch takes a
different amount of time as the different submodules vary in size, then
the output of fetches in sequential order might look like this:
time -->
output: |---A---| |-B-| |-------C-------| |-D-| |-E-|
When we schedule these submodules into maximal two parallel processes,
a schedule and sample output over time may look like this:
process 1: |---A---| |-D-| |-E-|
process 2: |-B-| |-------C-------|
output: |---A---|B|---C-------|DE
So A will be perceived as it would run normally in the single child
version. As B has finished by the time A is done, we can dump its whole
progress buffer on stderr, such that it looks like it finished in no
time. Once that is done, C is determined to be the visible child and
its progress will be reported in real time.
So this way of output is really good for human consumption, as it only
changes the timing, not the actual output.
For machine consumption the output needs to be prepared in the tasks,
by either having a prefix per line or per block to indicate whose tasks
output is displayed, because the output order may not follow the
original sequential ordering:
|----A----| |--B--| |-C-|
will be scheduled to be all parallel:
process 1: |----A----|
process 2: |--B--|
process 3: |-C-|
output: |----A----|CB
This happens because C finished before B did, so it will be queued for
output before B.
To detect when a child has finished executing, we check interleaved
with other actions (such as checking the liveliness of children or
starting new processes) whether the stderr pipe still exists. Once a
child closed its stderr stream, we assume it is terminating very soon,
and use `finish_command()` from the single external process execution
interface to collect the exit status.
By maintaining the strong assumption of stderr being open until the
very end of a child process, we can avoid other hassle such as an
implementation using `waitpid(-1)`, which is not implemented in Windows.
Signed-off-by: Stefan Beller <sbeller@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2015-12-16 00:04:10 +00:00
|
|
|
*/
|
|
|
|
int run_processes_parallel(int n,
|
|
|
|
get_next_task_fn,
|
|
|
|
start_failure_fn,
|
|
|
|
task_finished_fn,
|
|
|
|
void *pp_cb);
|
|
|
|
|
2005-07-31 19:17:43 +00:00
|
|
|
#endif
|