diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
index 27d4864577e5..c8eca88f12e6 100644
--- a/arch/arm64/kernel/insn.c
+++ b/arch/arm64/kernel/insn.c
@@ -87,8 +87,10 @@ static void __kprobes *patch_map(void *addr, int fixmap)
 
 	if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
 		page = vmalloc_to_page(addr);
-	else
+	else if (!module && IS_ENABLED(CONFIG_DEBUG_RODATA))
 		page = virt_to_page(addr);
+	else
+		return addr;
 
 	BUG_ON(!page);
 	set_fixmap(fixmap, page_to_phys(page));