diff --git a/ipc/msg.c b/ipc/msg.c
index cd90bfde89a4..770342e1d327 100644
--- a/ipc/msg.c
+++ b/ipc/msg.c
@@ -117,8 +117,6 @@ static struct msg_queue *msg_alloc(void)
 	if (unlikely(!msq))
 		return NULL;
 
-	atomic_set(&msq->q_perm.refcount, 1);
-
 	return msq;
 }
 
diff --git a/ipc/sem.c b/ipc/sem.c
index 2b2ed56e0fde..5f137738819d 100644
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -465,7 +465,6 @@ static struct sem_array *sem_alloc(size_t nsems)
 		return NULL;
 
 	memset(sma, 0, size);
-	atomic_set(&sma->sem_perm.refcount, 1);
 
 	return sma;
 }
diff --git a/ipc/shm.c b/ipc/shm.c
index c5976d318ed1..d1988ef821a1 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -526,8 +526,6 @@ static struct shmid_kernel *shm_alloc(void)
 	if (unlikely(!shp))
 		return NULL;
 
-	atomic_set(&shp->shm_perm.refcount, 1);
-
 	return shp;
 }
 
diff --git a/ipc/util.c b/ipc/util.c
index 2428dd44ca97..1a2cb02467ab 100644
--- a/ipc/util.c
+++ b/ipc/util.c
@@ -232,6 +232,7 @@ int ipc_addid(struct ipc_ids *ids, struct kern_ipc_perm *new, int size)
 
 	idr_preload(GFP_KERNEL);
 
+	atomic_set(&new->refcount, 1);
 	spin_lock_init(&new->lock);
 	new->deleted = false;
 	rcu_read_lock();