container/podman
container/podman
1
0
Fork 0
mirror of https://github.com/containers/podman synced 2024-10-21 09:45:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a backoff and retries to retrieving exited event

Browse source 
There's a potential race around extremely short-running
containers and events with journald. Events may not be written
for some time (small, but appreciable) after they are received,
and as such we can fail to retrieve it if there is a sufficiently
short time between us writing the event and trying to read it.

Work around this by just retrying, with a 0.25 second delay
between retries, up to 4 times.

[NO TESTS NEEDED] because I have no idea how to reproduce this
race in CI.

Fixes #11633

Signed-off-by: Matthew Heon <matthew.heon@pm.me>
This commit is contained in:
Matthew Heon 2021-09-21 14:24:23 -04:00
parent cd7b48198c
commit 4ecbc7caae
1 changed files with 25 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
pkg/domain/infra/abi/containers.go
View file

@ -830,21 +830,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri
}
return reports, errors.Wrapf(err, "unable to start container %s", ctr.ID())
}
if ecode, err := ctr.Wait(ctx); err != nil {
if errors.Cause(err) == define.ErrNoSuchCtr {
// Check events
event, err := ic.Libpod.GetLastContainerEvent(ctx, ctr.ID(), events.Exited)
if err != nil {
logrus.Errorf("Cannot get exit code: %v", err)
exitCode = define.ExecErrorCodeNotFound
} else {
exitCode = event.ContainerExitCode
}
}
} else {
exitCode = int(ecode)
}
exitCode = ic.GetContainerExitCode(ctx, ctr)
reports = append(reports, &entities.ContainerStartReport{
Id: ctr.ID(),
RawInput: rawInput,
@ -985,21 +971,7 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta
report.ExitCode = define.ExitCode(err)
return &report, err
}
if ecode, err := ctr.Wait(ctx); err != nil {
if errors.Cause(err) == define.ErrNoSuchCtr {
// Check events
event, err := ic.Libpod.GetLastContainerEvent(ctx, ctr.ID(), events.Exited)
if err != nil {
logrus.Errorf("Cannot get exit code: %v", err)
report.ExitCode = define.ExecErrorCodeNotFound
} else {
report.ExitCode = event.ContainerExitCode
}
}
} else {
report.ExitCode = int(ecode)
}
report.ExitCode = ic.GetContainerExitCode(ctx, ctr)
if opts.Rm && !ctr.ShouldRestart(ctx) {
if err := ic.Libpod.RemoveContainer(ctx, ctr, false, true); err != nil {
if errors.Cause(err) == define.ErrNoSuchCtr ||
@ -1013,6 +985,29 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta
return &report, nil
}
func (ic *ContainerEngine) GetContainerExitCode(ctx context.Context, ctr *libpod.Container) int {
exitCode, err := ctr.Wait(ctx)
if err == nil {
return int(exitCode)
}
if errors.Cause(err) != define.ErrNoSuchCtr {
logrus.Errorf("Could not retrieve exit code: %v", err)
return define.ExecErrorCodeNotFound
}
// Make 4 attempt with 0.25s backoff between each for 1 second total
var event *events.Event
for i := 0; i < 4; i++ {
event, err = ic.Libpod.GetLastContainerEvent(ctx, ctr.ID(), events.Exited)
if err != nil {
time.Sleep(250 * time.Millisecond)
continue
}
return int(event.ContainerExitCode)
}
logrus.Errorf("Could not retrieve exit code from event: %v", err)
return define.ExecErrorCodeNotFound
}
func (ic *ContainerEngine) ContainerLogs(ctx context.Context, containers []string, options entities.ContainerLogsOptions) error {
if options.StdoutWriter == nil && options.StderrWriter == nil {
return errors.New("no io.Writer set for container logs")