diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 21ddf9cbb2..cbf2a61b80 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -736,6 +736,12 @@ func (c *Container) cleanupCgroups() error {
 
 	cgroup, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(path))
 	if err != nil {
+		// It's fine for the cgroup to not exist
+		// We want it gone, it's gone
+		if err == cgroups.ErrCgroupDeleted {
+			return nil
+		}
+
 		return err
 	}