From d8132003f8d0f15a15ccd1ac00b8d72ca3b6c69a Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 16 Jan 2024 17:25:39 +0100 Subject: [PATCH 001/117] s390/diag: fix diag26c() physical vs virtual address confusion Fix virtual vs physical address confusion (which currently are the same). Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/diag.h | 4 ++-- arch/s390/kernel/diag.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index bed804137537..79ff1f940611 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -331,7 +331,7 @@ struct hypfs_diag0c_entry; */ struct diag_ops { int (*diag210)(struct diag210 *addr); - int (*diag26c)(void *req, void *resp, enum diag26c_sc subcode); + int (*diag26c)(unsigned long rx, unsigned long rx1, enum diag26c_sc subcode); int (*diag14)(unsigned long rx, unsigned long ry1, unsigned long subcode); int (*diag8c)(struct diag8c *addr, struct ccw_dev_id *devno, size_t len); void (*diag0c)(struct hypfs_diag0c_entry *entry); @@ -342,7 +342,7 @@ extern struct diag_ops diag_amode31_ops; extern struct diag210 *__diag210_tmp_amode31; int _diag210_amode31(struct diag210 *addr); -int _diag26c_amode31(void *req, void *resp, enum diag26c_sc subcode); +int _diag26c_amode31(unsigned long rx, unsigned long rx1, enum diag26c_sc subcode); int _diag14_amode31(unsigned long rx, unsigned long ry1, unsigned long subcode); void _diag0c_amode31(struct hypfs_diag0c_entry *entry); void _diag308_reset_amode31(void); diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index 92fdc35f028c..c33a95cfcc14 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -265,6 +265,6 @@ EXPORT_SYMBOL(diag224); int diag26c(void *req, void *resp, enum diag26c_sc subcode) { diag_stat_inc(DIAG_STAT_X26C); - return diag_amode31_ops.diag26c(req, resp, subcode); + return diag_amode31_ops.diag26c(virt_to_phys(req), virt_to_phys(resp), subcode); } EXPORT_SYMBOL(diag26c); From 0ec5117fa36b9c20b1bcb9b8faa40c3a2957cd7f Mon Sep 17 00:00:00 2001 From: Gerald Schaefer Date: Wed, 17 Jan 2024 13:58:12 +0100 Subject: [PATCH 002/117] s390/appldata: fix virtual vs physical address confusion Fix virtual vs physical address confusion (which currently are the same). Reviewed-by: Alexander Gordeev Signed-off-by: Gerald Schaefer Signed-off-by: Heiko Carstens --- arch/s390/include/asm/appldata.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/appldata.h b/arch/s390/include/asm/appldata.h index f2240392c708..a92ebbc7aa7a 100644 --- a/arch/s390/include/asm/appldata.h +++ b/arch/s390/include/asm/appldata.h @@ -54,13 +54,13 @@ static inline int appldata_asm(struct appldata_parameter_list *parm_list, parm_list->function = fn; parm_list->parlist_length = sizeof(*parm_list); parm_list->buffer_length = length; - parm_list->product_id_addr = (unsigned long) id; + parm_list->product_id_addr = virt_to_phys(id); parm_list->buffer_addr = virt_to_phys(buffer); diag_stat_inc(DIAG_STAT_X0DC); asm volatile( " diag %1,%0,0xdc" : "=d" (ry) - : "d" (parm_list), "m" (*parm_list), "m" (*id) + : "d" (virt_to_phys(parm_list)), "m" (*parm_list), "m" (*id) : "cc"); return ry; } From 49c372ae74b7c184b5aaedc3497735859f4c7d8e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 17 Jan 2024 15:35:16 +0100 Subject: [PATCH 003/117] s390/hypfs_sprp: fix virtual vs physical address confusion Add missing virt_to_phys() translation to __hypfs_sprp_diag304(). This doesn't fix a bug since virtual and physical addresses are currently the same. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/hypfs/hypfs_sprp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/hypfs/hypfs_sprp.c b/arch/s390/hypfs/hypfs_sprp.c index f5f7e78ddc0c..e66b4de8913f 100644 --- a/arch/s390/hypfs/hypfs_sprp.c +++ b/arch/s390/hypfs/hypfs_sprp.c @@ -25,7 +25,7 @@ static inline unsigned long __hypfs_sprp_diag304(void *data, unsigned long cmd) { - union register_pair r1 = { .even = (unsigned long)data, }; + union register_pair r1 = { .even = virt_to_phys(data), }; asm volatile("diag %[r1],%[r3],0x304\n" : [r1] "+&d" (r1.pair) From e98eda926b5d855a9513dcf742107d1e22d1089c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 17 Jan 2024 15:35:17 +0100 Subject: [PATCH 004/117] s390/hypfs_diag0c: fix virtual vs physical address confusion Add missing virt_to_phys() translation to diag0c(). This doesn't fix a bug since virtual and physical addresses are currently the same. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/hypfs/hypfs_diag0c.c | 3 +-- arch/s390/include/asm/diag.h | 11 +++++++++-- arch/s390/kernel/diag.c | 9 +++++++++ arch/s390/kernel/text_amode31.S | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/arch/s390/hypfs/hypfs_diag0c.c b/arch/s390/hypfs/hypfs_diag0c.c index 9a2786079e3a..4131f0daa5ea 100644 --- a/arch/s390/hypfs/hypfs_diag0c.c +++ b/arch/s390/hypfs/hypfs_diag0c.c @@ -20,8 +20,7 @@ */ static void diag0c_fn(void *data) { - diag_stat_inc(DIAG_STAT_X00C); - diag_amode31_ops.diag0c(((void **)data)[smp_processor_id()]); + diag0c(((void **)data)[smp_processor_id()]); } /* diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index 79ff1f940611..20b94220113b 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -44,6 +44,13 @@ enum diag_stat_enum { void diag_stat_inc(enum diag_stat_enum nr); void diag_stat_inc_norecursion(enum diag_stat_enum nr); +struct hypfs_diag0c_entry; + +/* + * Diagnose 0c: Pseudo Timer + */ +void diag0c(struct hypfs_diag0c_entry *data); + /* * Diagnose 10: Release page range */ @@ -334,7 +341,7 @@ struct diag_ops { int (*diag26c)(unsigned long rx, unsigned long rx1, enum diag26c_sc subcode); int (*diag14)(unsigned long rx, unsigned long ry1, unsigned long subcode); int (*diag8c)(struct diag8c *addr, struct ccw_dev_id *devno, size_t len); - void (*diag0c)(struct hypfs_diag0c_entry *entry); + void (*diag0c)(unsigned long rx); void (*diag308_reset)(void); }; @@ -344,7 +351,7 @@ extern struct diag210 *__diag210_tmp_amode31; int _diag210_amode31(struct diag210 *addr); int _diag26c_amode31(unsigned long rx, unsigned long rx1, enum diag26c_sc subcode); int _diag14_amode31(unsigned long rx, unsigned long ry1, unsigned long subcode); -void _diag0c_amode31(struct hypfs_diag0c_entry *entry); +void _diag0c_amode31(unsigned long rx); void _diag308_reset_amode31(void); int _diag8c_amode31(struct diag8c *addr, struct ccw_dev_id *devno, size_t len); diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index c33a95cfcc14..d8d97f970af4 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -146,6 +146,15 @@ void notrace diag_stat_inc_norecursion(enum diag_stat_enum nr) } EXPORT_SYMBOL(diag_stat_inc_norecursion); +/* + * Diagnose 0c: Pseudo Timer + */ +void diag0c(struct hypfs_diag0c_entry *data) +{ + diag_stat_inc(DIAG_STAT_X00C); + diag_amode31_ops.diag0c(virt_to_phys(data)); +} + /* * Diagnose 14: Input spool file manipulation */ diff --git a/arch/s390/kernel/text_amode31.S b/arch/s390/kernel/text_amode31.S index 14c6d25c035f..c0a70efa2426 100644 --- a/arch/s390/kernel/text_amode31.S +++ b/arch/s390/kernel/text_amode31.S @@ -90,7 +90,7 @@ SYM_FUNC_START(_diag26c_amode31) SYM_FUNC_END(_diag26c_amode31) /* - * void _diag0c_amode31(struct hypfs_diag0c_entry *entry) + * void _diag0c_amode31(unsigned long rx) */ SYM_FUNC_START(_diag0c_amode31) sam31 From 225d09d6e5f3870560665a1829d2db79330b4c58 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 18 Jan 2024 13:03:39 +0100 Subject: [PATCH 005/117] s390/pai: fix attr_event_free upper limit for pai device drivers When the device drivers are initialized, a sysfs directory is created. This contains many attributes which are allocated with kzalloc(). Should it fail, the memory for the attributes already created is freed in attr_event_free(). Its second parameter is number of attribute elements to delete. This parameter is off by one. When i. e. the 10th attribute fails to get created, attributes numbered 0 to 9 should be deleted. Currently only attributes numbered 0 to 8 are deleted. Fixes: 39d62336f5c1 ("s390/pai: add support for cryptography counters") Reported-by: Sumanth Korikkar Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 2 +- arch/s390/kernel/perf_pai_ext.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index bf8a672b15a4..522a5ea0a9f4 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -721,7 +721,7 @@ static int __init attr_event_init(void) for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) { ret = attr_event_init_one(attrs, i); if (ret) { - attr_event_free(attrs, i - 1); + attr_event_free(attrs, i); return ret; } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index af7f2b538c8f..95d1a890640a 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -611,7 +611,7 @@ static int __init attr_event_init(void) for (i = 0; i < ARRAY_SIZE(paiext_ctrnames); i++) { ret = attr_event_init_one(attrs, i); if (ret) { - attr_event_free(attrs, i - 1); + attr_event_free(attrs, i); return ret; } } From 3a5da4670dfad43e8e0b0c7135409c1f70230797 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 18 Jan 2024 13:24:45 +0100 Subject: [PATCH 006/117] s390/pai_crypto: emit error on too many counters When the device driver is initialized, it checks the number of possible counters. Should this number be too high, emit an error and return. Reported-by: Sumanth Korikkar Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 522a5ea0a9f4..69639ff1c5e0 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -742,8 +742,10 @@ static int __init paicrypt_init(void) paicrypt_cnt = ib.num_cc; if (paicrypt_cnt == 0) return 0; - if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) - paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1; + if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) { + pr_err("Too many PMU pai_crypto counters %d\n", paicrypt_cnt); + return -1; + } rc = attr_event_init(); /* Export known PAI crypto events */ if (rc) { From d414f4ecb240b994cba8c9666def0a4b9c953601 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 18 Jan 2024 13:46:08 +0100 Subject: [PATCH 007/117] s390/pai: export number of sysfs attribute files The number of sysfs files to be exported by the PAI device drivers depends on the hardware version level. Use the value returned by the hardware as the maximum number of counters to be exported in the sysfs attribute tree. Without the fix, older machine generation export counter names based on paiXXXX_ctrnames static array info, which can be inaccurate, as this array could also contain newer counter names in the future. This ensures proper pai counter sysfs attributes for both newer generation and older generation processors. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 11 ++++++++--- arch/s390/kernel/perf_pai_ext.c | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 69639ff1c5e0..ffde6d5001b7 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -694,6 +694,12 @@ static int __init attr_event_init_one(struct attribute **attrs, int num) { struct perf_pmu_events_attr *pa; + /* Index larger than array_size, no counter name available */ + if (num >= ARRAY_SIZE(paicrypt_ctrnames)) { + attrs[num] = NULL; + return 0; + } + pa = kzalloc(sizeof(*pa), GFP_KERNEL); if (!pa) return -ENOMEM; @@ -714,11 +720,10 @@ static int __init attr_event_init(void) struct attribute **attrs; int ret, i; - attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs), - GFP_KERNEL); + attrs = kmalloc_array(paicrypt_cnt + 2, sizeof(*attrs), GFP_KERNEL); if (!attrs) return -ENOMEM; - for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) { + for (i = 0; i <= paicrypt_cnt; i++) { ret = attr_event_init_one(attrs, i); if (ret) { attr_event_free(attrs, i); diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 95d1a890640a..4d86130bcf14 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -584,6 +584,12 @@ static int __init attr_event_init_one(struct attribute **attrs, int num) { struct perf_pmu_events_attr *pa; + /* Index larger than array_size, no counter name available */ + if (num >= ARRAY_SIZE(paiext_ctrnames)) { + attrs[num] = NULL; + return 0; + } + pa = kzalloc(sizeof(*pa), GFP_KERNEL); if (!pa) return -ENOMEM; @@ -604,11 +610,10 @@ static int __init attr_event_init(void) struct attribute **attrs; int ret, i; - attrs = kmalloc_array(ARRAY_SIZE(paiext_ctrnames) + 1, sizeof(*attrs), - GFP_KERNEL); + attrs = kmalloc_array(paiext_cnt + 2, sizeof(*attrs), GFP_KERNEL); if (!attrs) return -ENOMEM; - for (i = 0; i < ARRAY_SIZE(paiext_ctrnames); i++) { + for (i = 0; i <= paiext_cnt; i++) { ret = attr_event_init_one(attrs, i); if (ret) { attr_event_free(attrs, i); From 5d8cc70c36c42aa33e595836af2faaaaeb314fb5 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 22 Jan 2024 13:36:05 +0100 Subject: [PATCH 008/117] s390/pai_crypto: return proper error code in paicrypt_init paicrypt_init() return incorrect error code in case the number of PAI crypto counters is too high. Change the return code to -E2BIG. Please merge with d0b0efedc7fe Reported-by: Heiko Carstens Signed-off-by: Thomas Richter Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index ffde6d5001b7..7e7472aee4e5 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -749,7 +749,7 @@ static int __init paicrypt_init(void) return 0; if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR) { pr_err("Too many PMU pai_crypto counters %d\n", paicrypt_cnt); - return -1; + return -E2BIG; } rc = attr_event_init(); /* Export known PAI crypto events */ From fc17e992e1fdc5d2e3cf1049073aad0dd3933372 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Thu, 14 Sep 2023 08:58:54 +0200 Subject: [PATCH 009/117] s390/time: improve steering precision The common timekeeping code steers the clock by adjusting the multiplier value of the clock. With the current value of 1000 precision is lost when the clock is steered with a userspace daemon. Increase the multiplier and the shift values to increase precision. Signed-off-by: Sven Schnelle Acked-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/kernel/time.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 14abad953c02..8357306fb76e 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -251,8 +251,8 @@ static struct clocksource clocksource_tod = { .rating = 400, .read = read_tod_clock, .mask = CLOCKSOURCE_MASK(64), - .mult = 1000, - .shift = 12, + .mult = 4096000, + .shift = 24, .flags = CLOCK_SOURCE_IS_CONTINUOUS, .vdso_clock_mode = VDSO_CLOCKMODE_TOD, }; From 0ad92cbd5a55df4cf4610a9124b24e3f74b1ac50 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 25 Jan 2024 10:48:57 +0100 Subject: [PATCH 010/117] s390/vmur: fix virtual vs physical address confusion Add missing virt_to_phys() / phys_to_virt() translation to alloc_chan_prog() and free_chan_prog(). This doesn't fix a bug since virtual and physical addresses are currently the same. Signed-off-by: Thomas Richter Signed-off-by: Heiko Carstens --- drivers/s390/char/vmur.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 82efdd20ad01..1d17a83569ce 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -195,7 +195,7 @@ static void free_chan_prog(struct ccw1 *cpa) struct ccw1 *ptr = cpa; while (ptr->cda) { - kfree((void *)(addr_t) ptr->cda); + kfree(phys_to_virt(ptr->cda)); ptr++; } kfree(cpa); @@ -237,7 +237,7 @@ static struct ccw1 *alloc_chan_prog(const char __user *ubuf, int rec_count, free_chan_prog(cpa); return ERR_PTR(-ENOMEM); } - cpa[i].cda = (u32)(addr_t) kbuf; + cpa[i].cda = (u32)virt_to_phys(kbuf); if (copy_from_user(kbuf, ubuf, reclen)) { free_chan_prog(cpa); return ERR_PTR(-EFAULT); From eec561024b3e733ba4ed3515ba81b45f5e647d0d Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 25 Jan 2024 10:48:58 +0100 Subject: [PATCH 011/117] s390/diag: add missing virt_to_phys() translation to diag14() diag14() is currently only used by the vmur device driver. The third parameter, called subcommand, determines the type of the first parameter. For some subcommands the value of the first parameter is an address to a memory buffer and needs virtual to physical address conversion. Other subcommands interpret the first parameter is an integer. This doesn't fix a bug since virtual and physical addresses are currently the same. Suggested-by: Heiko Carstens Signed-off-by: Thomas Richter Signed-off-by: Heiko Carstens --- arch/s390/kernel/diag.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index d8d97f970af4..8dee9aa0ec95 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -157,10 +157,30 @@ void diag0c(struct hypfs_diag0c_entry *data) /* * Diagnose 14: Input spool file manipulation + * + * The subcode parameter determines the type of the first parameter rx. + * Currently used are the following 3 subcommands: + * 0x0: Read the Next Spool File Buffer (Data Record) + * 0x28: Position a Spool File to the Designated Record + * 0xfff: Retrieve Next File Descriptor + * + * For subcommands 0x0 and 0xfff, the value of the first parameter is + * a virtual address of a memory buffer and needs virtual to physical + * address translation. For other subcommands the rx parameter is not + * a virtual address. */ int diag14(unsigned long rx, unsigned long ry1, unsigned long subcode) { diag_stat_inc(DIAG_STAT_X014); + switch (subcode) { + case 0x0: + case 0xfff: + rx = virt_to_phys((void *)rx); + break; + default: + /* Do nothing */ + break; + } return diag_amode31_ops.diag14(rx, ry1, subcode); } EXPORT_SYMBOL(diag14); From 791833f22431aacdec6d489cc138e82c40709450 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 19 Jan 2024 17:19:46 +0100 Subject: [PATCH 012/117] s390/hypfs_sprp: remove unneeded DMA zone allocation Remove GFP_DMA flag when allocating memory to be used for diagnose 304. Diagnose 304 can access memory beyond the DMA zone. Suggested-by: Alexander Gordeev Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/hypfs/hypfs_sprp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/hypfs/hypfs_sprp.c b/arch/s390/hypfs/hypfs_sprp.c index e66b4de8913f..9fc3f0dae8f0 100644 --- a/arch/s390/hypfs/hypfs_sprp.c +++ b/arch/s390/hypfs/hypfs_sprp.c @@ -74,7 +74,7 @@ static int __hypfs_sprp_ioctl(void __user *user_area) int rc; rc = -ENOMEM; - data = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + data = (void *)get_zeroed_page(GFP_KERNEL); diag304 = kzalloc(sizeof(*diag304), GFP_KERNEL); if (!data || !diag304) goto out; From 86f48f922ba79ca32db1aabbcf8758148f925eb3 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Wed, 24 Jan 2024 09:22:07 +0100 Subject: [PATCH 013/117] s390/mmap: disable mmap alignment when randomize_va_space = 0 Stefan reported a test case fail in libc. The test runs with randomize_va_space set to zero, i.e. disabled randomization. Additionally, it runs the program with the dynamic loader. Looking at the failure showed that the heap was placed right before some pages mapped from the binary. This made memory allocation fail after a few allocations. Normally, when address randomization is switched off and the binary is loaded from the dynamic loader, the kernel places the binary below the 128MB top gap. So the address map would look like this: 3fff7fd1000-3fff7fd2000 r--p 00000000 5e:01 1447115 /lib/ld64.so.1 3fff7fd2000-3fff7ff2000 r-xp 00001000 5e:01 1447115 /lib/ld64.so.1 3fff7ff2000-3fff7ffc000 r--p 00021000 5e:01 1447115 /lib/ld64.so.1 3fff7ffc000-3fff7ffe000 r--p 0002a000 5e:01 1447115 /lib/ld64.so.1 3fff7ffe000-3fff8000000 rw-p 0002c000 5e:01 1447115 /lib/ld64.so.1 3fff8000000-3fff8021000 rw-p 00000000 00:00 0 [heap] 3fffffda000-3ffffffb000 rw-p 00000000 00:00 0 [stack] 3ffffffc000-3ffffffe000 r--p 00000000 00:00 0 [vvar] 3ffffffe000-40000000000 r-xp 00000000 00:00 0 [vdso] However, commit 1f6b83e5e4d3 ("s390: avoid z13 cache aliasing") introduced a mmap alignment mask of 8MB. With this commit, the memory map now looks like this: 3fff7f80000-3fff7f81000 r--p 00000000 5e:01 1447115 /lib/ld64.so.1 3fff7f81000-3fff7fa1000 r-xp 00001000 5e:01 1447115 /lib/ld64.so.1 3fff7fa1000-3fff7fab000 r--p 00021000 5e:01 1447115 /lib/ld64.so.1 3fff7fab000-3fff7fad000 r--p 0002a000 5e:01 1447115 /lib/ld64.so.1 3fff7fad000-3fff7faf000 rw-p 0002c000 5e:01 1447115 /lib/ld64.so.1 3fff7faf000-3fff7fd0000 rw-p 00000000 00:00 0 [heap] 3fff7fdc000-3fff8000000 rw-p 00000000 00:00 0 3fffffda000-3ffffffb000 rw-p 00000000 00:00 0 [stack] 3ffffffc000-3ffffffe000 r--p 00000000 00:00 0 [vvar] 3ffffffe000-40000000000 r-xp 00000000 00:00 0 [vdso] The reason for this placement is that the elf loader loads the binary to end at mmap_base (0x3fff8000000 on s390). This would result in a start address of 0x3fff7fd1000, but due to the alignment requirement of 8MB, mmap chooses 0x3fff7f80000. This causes a gap between the end of the mapped binary and mmap_base. When the next non-shared and non-file pages are mapped, mmap searches from top to bottom and the first free space it finds is the gap which is now present. This leaves only a few pages for the heap. With enabled address space randomization this doesn't happen because the binary is mapped to a completely different memory area. Fix this by disabling the mmap alignment when address space randomization is disabled. This is in line with what other architectures are doing. Reported-by: Stefan Liebler Tested-by: Stefan Liebler Signed-off-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/mm/mmap.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index fc9a7dc26c5e..b14fc0887654 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -71,6 +71,15 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(STACK_TOP - gap - rnd); } +static int get_align_mask(struct file *filp, unsigned long flags) +{ + if (!(current->flags & PF_RANDOMIZE)) + return 0; + if (filp || (flags & MAP_SHARED)) + return MMAP_ALIGN_MASK << PAGE_SHIFT; + return 0; +} + unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) @@ -97,10 +106,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; - if (filp || (flags & MAP_SHARED)) - info.align_mask = MMAP_ALIGN_MASK << PAGE_SHIFT; - else - info.align_mask = 0; + info.align_mask = get_align_mask(filp, flags); info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); if (offset_in_page(addr)) @@ -138,10 +144,7 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; - if (filp || (flags & MAP_SHARED)) - info.align_mask = MMAP_ALIGN_MASK << PAGE_SHIFT; - else - info.align_mask = 0; + info.align_mask = get_align_mask(filp, flags); info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); From a3a64a4def8daa9e73120a9f4b64fa9a91bcdc06 Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Thu, 18 Jan 2024 16:09:39 +0100 Subject: [PATCH 014/117] s390/cio: remove unneeded DMA zone allocation Remove GFP_DMA flag when allocating memory to be used for CHSC control blocks. The CHSC instruction can access memory beyond the DMA zone. Suggested-by: Heiko Carstens Reviewed-by: Vineeth Vijayan Signed-off-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc.c | 4 ++-- drivers/s390/cio/chsc_sch.c | 20 ++++++++++---------- drivers/s390/cio/scm.c | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 64ed55c3aed6..3d88899dff7c 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -1091,8 +1091,8 @@ int __init chsc_init(void) { int ret; - sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); - chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sei_page = (void *)get_zeroed_page(GFP_KERNEL); + chsc_page = (void *)get_zeroed_page(GFP_KERNEL); if (!sei_page || !chsc_page) { ret = -ENOMEM; goto out_err; diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index 902237d0baef..e6c800653f98 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -293,7 +293,7 @@ static int chsc_ioctl_start(void __user *user_area) if (!css_general_characteristics.dynio) /* It makes no sense to try. */ return -EOPNOTSUPP; - chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + chsc_area = (void *)get_zeroed_page(GFP_KERNEL); if (!chsc_area) return -ENOMEM; request = kzalloc(sizeof(*request), GFP_KERNEL); @@ -341,7 +341,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area) ret = -ENOMEM; goto out_unlock; } - on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + on_close_chsc_area = (void *)get_zeroed_page(GFP_KERNEL); if (!on_close_chsc_area) { ret = -ENOMEM; goto out_free_request; @@ -393,7 +393,7 @@ static int chsc_ioctl_start_sync(void __user *user_area) struct chsc_sync_area *chsc_area; int ret, ccode; - chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + chsc_area = (void *)get_zeroed_page(GFP_KERNEL); if (!chsc_area) return -ENOMEM; if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) { @@ -439,7 +439,7 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *scpcd_area; - scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpcd_area = (void *)get_zeroed_page(GFP_KERNEL); if (!scpcd_area) return -ENOMEM; cd = kzalloc(sizeof(*cd), GFP_KERNEL); @@ -501,7 +501,7 @@ static int chsc_ioctl_info_cu(void __user *user_cd) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *scucd_area; - scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scucd_area = (void *)get_zeroed_page(GFP_KERNEL); if (!scucd_area) return -ENOMEM; cd = kzalloc(sizeof(*cd), GFP_KERNEL); @@ -564,7 +564,7 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *sscud_area; - sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sscud_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sscud_area) return -ENOMEM; cud = kzalloc(sizeof(*cud), GFP_KERNEL); @@ -626,7 +626,7 @@ static int chsc_ioctl_conf_info(void __user *user_ci) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *sci_area; - sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sci_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sci_area) return -ENOMEM; ci = kzalloc(sizeof(*ci), GFP_KERNEL); @@ -697,7 +697,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) u32 res; } __attribute__ ((packed)) *cssids_parm; - sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sccl_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sccl_area) return -ENOMEM; ccl = kzalloc(sizeof(*ccl), GFP_KERNEL); @@ -757,7 +757,7 @@ static int chsc_ioctl_chpd(void __user *user_chpd) int ret; chpd = kzalloc(sizeof(*chpd), GFP_KERNEL); - scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpd_area = (void *)get_zeroed_page(GFP_KERNEL); if (!scpd_area || !chpd) { ret = -ENOMEM; goto out_free; @@ -797,7 +797,7 @@ static int chsc_ioctl_dcal(void __user *user_dcal) u8 data[PAGE_SIZE - 36]; } __attribute__ ((packed)) *sdcal_area; - sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sdcal_area = (void *)get_zeroed_page(GFP_KERNEL); if (!sdcal_area) return -ENOMEM; dcal = kzalloc(sizeof(*dcal), GFP_KERNEL); diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c index 6b21ba68c1fe..a734b323e063 100644 --- a/drivers/s390/cio/scm.c +++ b/drivers/s390/cio/scm.c @@ -228,7 +228,7 @@ int scm_update_information(void) size_t num; int ret; - scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA); + scm_info = (void *)__get_free_page(GFP_KERNEL); if (!scm_info) return -ENOMEM; From 343c8a564583d9166ddacd2ade1f917eb8ea37cc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 19 Jan 2024 12:19:29 +0100 Subject: [PATCH 015/117] s390/cmf: remove unneeded DMA zone allocation The address of the measurement block can be anywhere in 64 bit absolute space. See description of the schm instruction in the Principles of Operation. Therefore remove the GFP_DMA flag when allocating the block. Acked-by: Alexander Gordeev Reviewed-by: Vineeth Vijayan Signed-off-by: Heiko Carstens --- drivers/s390/cio/cmf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 5584aa46c94e..6a6e61ceb3c1 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -501,8 +501,7 @@ static int alloc_cmb(struct ccw_device *cdev) WARN_ON(!list_empty(&cmb_area.list)); spin_unlock(&cmb_area.lock); - mem = (void*)__get_free_pages(GFP_KERNEL | GFP_DMA, - get_order(size)); + mem = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); spin_lock(&cmb_area.lock); if (cmb_area.mem) { From 14edd0d73bfef320fd5be11495687ba537aa6341 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 19 Jan 2024 12:19:30 +0100 Subject: [PATCH 016/117] s390/cmf: fix virtual vs physical address confusion The measurement block origin address is an absolute address; therefore add a missing virt_to_phys() translation to the cmf_activate() inline assembly. This doesn't fix a bug, since virtual and physical addresses are currently identical. Acked-by: Alexander Gordeev Acked-by: Vineeth Vijayan Signed-off-by: Heiko Carstens --- drivers/s390/cio/cmf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 6a6e61ceb3c1..f80dc18e2a76 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -169,7 +169,8 @@ static inline void cmf_activate(void *area, unsigned int onoff) " lgr 2,%[mbo]\n" " schm\n" : - : [r1] "d" ((unsigned long)onoff), [mbo] "d" (area) + : [r1] "d" ((unsigned long)onoff), + [mbo] "d" (virt_to_phys(area)) : "1", "2"); } From 0628c03934187be33942580e10bb9afcc61adeed Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 30 Jan 2024 20:14:28 -0700 Subject: [PATCH 017/117] s390/vdso: drop '-fPIC' from LDFLAGS '-fPIC' as an option to the linker does not do what it seems like it should. With ld.bfd, it is treated as '-f PIC', which does not make sense based on the meaning of '-f': -f SHLIB, --auxiliary SHLIB Auxiliary filter for shared object symbol table When building with ld.lld (currently under review in a GitHub pull request), it just errors out because '-f' means nothing and neither does '-fPIC': ld.lld: error: unknown argument '-fPIC' '-fPIC' was blindly copied from CFLAGS when the vDSO stopped being linked with '$(CC)', it should not be needed. Remove it to clear up the build failure with ld.lld. Fixes: 2b2a25845d53 ("s390/vdso: Use $(LD) instead of $(CC) to link vDSO") Link: https://github.com/llvm/llvm-project/pull/75643 Signed-off-by: Nathan Chancellor Reviewed-by: Fangrui Song Link: https://lore.kernel.org/r/20240130-s390-vdso-drop-fpic-from-ldflags-v1-1-094ad104fc55@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/kernel/vdso32/Makefile | 2 +- arch/s390/kernel/vdso64/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index caec7db6f966..b12a274cbb47 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -22,7 +22,7 @@ KBUILD_CFLAGS_32 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_32 := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 += -m31 -fPIC -shared -fno-common -fno-builtin -LDFLAGS_vdso32.so.dbg += -fPIC -shared -soname=linux-vdso32.so.1 \ +LDFLAGS_vdso32.so.dbg += -shared -soname=linux-vdso32.so.1 \ --hash-style=both --build-id=sha1 -melf_s390 -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index e3c9085f8fa7..caa4ebff8a19 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -26,7 +26,7 @@ KBUILD_AFLAGS_64 += -m64 KBUILD_CFLAGS_64 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_64 := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_64)) KBUILD_CFLAGS_64 += -m64 -fPIC -fno-common -fno-builtin -ldflags-y := -fPIC -shared -soname=linux-vdso64.so.1 \ +ldflags-y := -shared -soname=linux-vdso64.so.1 \ --hash-style=both --build-id=sha1 -T $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_64) From 6695d792246eb69c77e5952a0b85c8684950ed71 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sun, 4 Feb 2024 11:25:45 -0300 Subject: [PATCH 018/117] s390/time: make stp_subsys const Now that the driver core can properly handle constant struct bus_type, move the stp_subsys variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240204-bus_cleanup-s390_time-v1-1-d2120156982a@marliere.net Signed-off-by: Heiko Carstens --- arch/s390/kernel/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 8357306fb76e..fb9f31f36628 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -716,7 +716,7 @@ static void stp_work_fn(struct work_struct *work) /* * STP subsys sysfs interface functions */ -static struct bus_type stp_subsys = { +static const struct bus_type stp_subsys = { .name = "stp", .dev_name = "stp", }; From 0d78df873a4e86235af42ea108c5c65fa94d2db8 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 11:57:58 -0300 Subject: [PATCH 019/117] s390/ccwgroup: make ccwgroup_bus_type const Now that the driver core can properly handle constant struct bus_type, move the ccwgroup_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-s390-v1-1-ac891afc7282@marliere.net Signed-off-by: Heiko Carstens --- drivers/s390/cio/ccwgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/ccwgroup.c b/drivers/s390/cio/ccwgroup.c index aa3292e57e38..6eb8bcd948dc 100644 --- a/drivers/s390/cio/ccwgroup.c +++ b/drivers/s390/cio/ccwgroup.c @@ -31,7 +31,7 @@ * to devices that use multiple subchannels. */ -static struct bus_type ccwgroup_bus_type; +static const struct bus_type ccwgroup_bus_type; static void __ccwgroup_remove_symlinks(struct ccwgroup_device *gdev) { @@ -465,7 +465,7 @@ static void ccwgroup_shutdown(struct device *dev) gdrv->shutdown(gdev); } -static struct bus_type ccwgroup_bus_type = { +static const struct bus_type ccwgroup_bus_type = { .name = "ccwgroup", .dev_groups = ccwgroup_dev_groups, .remove = ccwgroup_remove, From 42c4c8fdbd1a71970d2a7e71d682b6beeb9f92b1 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 11:57:59 -0300 Subject: [PATCH 020/117] s390/cio: make css_bus_type const Now that the driver core can properly handle constant struct bus_type, move the css_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-s390-v1-2-ac891afc7282@marliere.net Signed-off-by: Heiko Carstens --- drivers/s390/cio/css.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 28a88ed2c3aa..094431a62ad5 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -39,7 +39,7 @@ int max_ssid; #define MAX_CSS_IDX 0 struct channel_subsystem *channel_subsystems[MAX_CSS_IDX + 1]; -static struct bus_type css_bus_type; +static const struct bus_type css_bus_type; int for_each_subchannel(int(*fn)(struct subchannel_id, void *), void *data) @@ -1409,7 +1409,7 @@ static int css_uevent(const struct device *dev, struct kobj_uevent_env *env) return ret; } -static struct bus_type css_bus_type = { +static const struct bus_type css_bus_type = { .name = "css", .match = css_bus_match, .probe = css_probe, From fd2b4bfa5fb4e4aa1c7eab6fca92ac01c44a4ac1 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 11:58:00 -0300 Subject: [PATCH 021/117] s390/cio: make ccw_bus_type const Now that the driver core can properly handle constant struct bus_type, move the ccw_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-s390-v1-3-ac891afc7282@marliere.net Signed-off-by: Heiko Carstens --- drivers/s390/cio/device.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 0cfb179e1bcb..f95d12345d98 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -49,7 +49,7 @@ static const unsigned long recovery_delay[] = { 3, 30, 300 }; static atomic_t ccw_device_init_count = ATOMIC_INIT(0); static DECLARE_WAIT_QUEUE_HEAD(ccw_device_init_wq); -static struct bus_type ccw_bus_type; +static const struct bus_type ccw_bus_type; /******************* bus type handling ***********************/ @@ -1776,7 +1776,7 @@ static void ccw_device_shutdown(struct device *dev) __disable_cmf(cdev); } -static struct bus_type ccw_bus_type = { +static const struct bus_type ccw_bus_type = { .name = "ccw", .match = ccw_bus_match, .uevent = ccw_uevent, From 7090dadbe72399ff6cb0a648736fc77414878a96 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 11:58:01 -0300 Subject: [PATCH 022/117] s390/cio: make scm_bus_type const Now that the driver core can properly handle constant struct bus_type, move the scm_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-s390-v1-4-ac891afc7282@marliere.net Signed-off-by: Heiko Carstens --- drivers/s390/cio/scm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c index a734b323e063..c7894d61306d 100644 --- a/drivers/s390/cio/scm.c +++ b/drivers/s390/cio/scm.c @@ -42,7 +42,7 @@ static int scmdev_uevent(const struct device *dev, struct kobj_uevent_env *env) return add_uevent_var(env, "MODALIAS=scm:scmdev"); } -static struct bus_type scm_bus_type = { +static const struct bus_type scm_bus_type = { .name = "scm", .probe = scmdev_probe, .remove = scmdev_remove, From 5b431787548a46898be655d79420022c4f43274c Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 11:58:02 -0300 Subject: [PATCH 023/117] s390/ap: make ap_bus_type const Now that the driver core can properly handle constant struct bus_type, move the ap_bus_type variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Reviewed-by: Harald Freudenberger Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-s390-v1-5-ac891afc7282@marliere.net Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index f46dd6abacd7..2ecf4d36e78b 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -135,7 +135,7 @@ static int ap_max_domain_id = 15; /* Maximum adapter id, if not given via qci */ static int ap_max_adapter_id = 63; -static struct bus_type ap_bus_type; +static const struct bus_type ap_bus_type; /* Adapter interrupt definitions */ static void ap_interrupt_handler(struct airq_struct *airq, @@ -1603,7 +1603,7 @@ static struct attribute *ap_bus_attrs[] = { }; ATTRIBUTE_GROUPS(ap_bus); -static struct bus_type ap_bus_type = { +static const struct bus_type ap_bus_type = { .name = "ap", .bus_groups = ap_bus_groups, .match = &ap_bus_match, From 9e99049a80b1a251dc4581ff3031196015b8ad41 Mon Sep 17 00:00:00 2001 From: "Ricardo B. Marliere" Date: Sat, 3 Feb 2024 11:58:03 -0300 Subject: [PATCH 024/117] s390/vfio-ap: make matrix_bus const Now that the driver core can properly handle constant struct bus_type, move the matrix_bus variable to be a constant structure as well, placing it into read-only memory which can not be modified at runtime. Cc: Greg Kroah-Hartman Suggested-by: Greg Kroah-Hartman Signed-off-by: "Ricardo B. Marliere" Acked-by: Halil Pasic Reviewed-by: Anthony Krowiak Reviewed-by: "Jason J. Herne" Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20240203-bus_cleanup-s390-v1-6-ac891afc7282@marliere.net Signed-off-by: Heiko Carstens --- drivers/s390/crypto/vfio_ap_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/crypto/vfio_ap_drv.c b/drivers/s390/crypto/vfio_ap_drv.c index a5ab03e42ff1..4aeb3e1213c7 100644 --- a/drivers/s390/crypto/vfio_ap_drv.c +++ b/drivers/s390/crypto/vfio_ap_drv.c @@ -60,7 +60,7 @@ static void vfio_ap_matrix_dev_release(struct device *dev) kfree(matrix_dev); } -static struct bus_type matrix_bus = { +static const struct bus_type matrix_bus = { .name = "matrix", }; From fd7eea27a3aed79b63b1726c00bde0d50cf207e2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 5 Feb 2024 16:48:43 +0100 Subject: [PATCH 025/117] Compiler Attributes: Add __uninitialized macro With INIT_STACK_ALL_PATTERN or INIT_STACK_ALL_ZERO enabled the kernel will be compiled with -ftrivial-auto-var-init=<...> which causes initialization of stack variables at function entry time. In order to avoid the performance impact that comes with this users can use the "uninitialized" attribute to prevent such initialization. Therefore provide the __uninitialized macro which can be used for cases where INIT_STACK_ALL_PATTERN or INIT_STACK_ALL_ZERO is enabled, but only selected variables should not be initialized. Acked-by: Kees Cook Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240205154844.3757121-2-hca@linux.ibm.com Signed-off-by: Heiko Carstens --- include/linux/compiler_attributes.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 28566624f008..f5859b8c68b4 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -333,6 +333,18 @@ */ #define __section(section) __attribute__((__section__(section))) +/* + * Optional: only supported since gcc >= 12 + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-uninitialized-variable-attribute + * clang: https://clang.llvm.org/docs/AttributeReference.html#uninitialized + */ +#if __has_attribute(__uninitialized__) +# define __uninitialized __attribute__((__uninitialized__)) +#else +# define __uninitialized +#endif + /* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-unused-function-attribute * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Type-Attributes.html#index-unused-type-attribute From 8d16ce148858669f05b8e117a0634010397e17d6 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 5 Feb 2024 16:48:44 +0100 Subject: [PATCH 026/117] s390/fpu: make use of __uninitialized macro Code sections in s390 specific kernel code which use floating point or vector registers all come with a 520 byte stack variable to save already in use registers, if required. With INIT_STACK_ALL_PATTERN or INIT_STACK_ALL_ZERO enabled this variable will always be initialized on function entry in addition to saving register contents, which contradicts the intention (performance improvement) of such code sections. Therefore provide a DECLARE_KERNEL_FPU_ONSTACK() macro which provides struct kernel_fpu variables with an __uninitialized attribute, and convert all existing code to use this. This way only this specific type of stack variable will not be initialized, regardless of config options. Reviewed-by: Nathan Chancellor Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20240205154844.3757121-3-hca@linux.ibm.com Signed-off-by: Heiko Carstens --- arch/s390/crypto/chacha-glue.c | 2 +- arch/s390/crypto/crc32-vx.c | 2 +- arch/s390/include/asm/fpu/types.h | 3 +++ arch/s390/kernel/sysinfo.c | 2 +- lib/raid6/s390vx.uc | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index ed9959e6f714..a823132fc563 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -22,7 +22,7 @@ static void chacha20_crypt_s390(u32 *state, u8 *dst, const u8 *src, unsigned int nbytes, const u32 *key, u32 *counter) { - struct kernel_fpu vxstate; + DECLARE_KERNEL_FPU_ONSTACK(vxstate); kernel_fpu_begin(&vxstate, KERNEL_VXR); chacha20_vx(dst, src, nbytes, key, counter); diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index 017143e9cef7..6ae3e3ff5b0a 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -49,8 +49,8 @@ u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); static u32 __pure ___fname(u32 crc, \ unsigned char const *data, size_t datalen) \ { \ - struct kernel_fpu vxstate; \ unsigned long prealign, aligned, remaining; \ + DECLARE_KERNEL_FPU_ONSTACK(vxstate); \ \ if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \ return ___crc32_sw(crc, data, datalen); \ diff --git a/arch/s390/include/asm/fpu/types.h b/arch/s390/include/asm/fpu/types.h index d889e9436865..b1afa13c07b7 100644 --- a/arch/s390/include/asm/fpu/types.h +++ b/arch/s390/include/asm/fpu/types.h @@ -35,4 +35,7 @@ struct kernel_fpu { }; }; +#define DECLARE_KERNEL_FPU_ONSTACK(name) \ + struct kernel_fpu name __uninitialized + #endif /* _ASM_S390_FPU_TYPES_H */ diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index f6f8f498c9be..b439f17516eb 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -426,9 +426,9 @@ subsys_initcall(create_proc_service_level); */ void s390_adjust_jiffies(void) { + DECLARE_KERNEL_FPU_ONSTACK(fpu); struct sysinfo_1_2_2 *info; unsigned long capability; - struct kernel_fpu fpu; info = (void *) get_zeroed_page(GFP_KERNEL); if (!info) diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc index cd0b9e95f499..7b0b355e1a26 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid6/s390vx.uc @@ -81,7 +81,7 @@ static inline void COPY_VEC(int x, int y) static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) { - struct kernel_fpu vxstate; + DECLARE_KERNEL_FPU_ONSTACK(vxstate); u8 **dptr, *p, *q; int d, z, z0; @@ -114,7 +114,7 @@ static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, size_t bytes, void **ptrs) { - struct kernel_fpu vxstate; + DECLARE_KERNEL_FPU_ONSTACK(vxstate); u8 **dptr, *p, *q; int d, z, z0; From f78bcb2e26dcac39a637e14c0ede9499822cb872 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 31 Jan 2024 09:30:36 +0100 Subject: [PATCH 027/117] s390/extmem: fix virtual vs physical address confusion Fix virtual vs physical address confusion. This does not fix a bug since virtual and physical address spaces are currently the same. Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/mm/extmem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index e41869f5cc95..282fefe107a2 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -136,7 +136,7 @@ dcss_diag(int *func, void *parameter, unsigned long rx, ry; int rc; - rx = (unsigned long) parameter; + rx = virt_to_phys(parameter); ry = (unsigned long) *func; diag_stat_inc(DIAG_STAT_X064); @@ -178,7 +178,7 @@ query_segment_type (struct dcss_segment *seg) /* initialize diag input parameters */ qin->qopcode = DCSS_FINDSEGA; - qin->qoutptr = (unsigned long) qout; + qin->qoutptr = virt_to_phys(qout); qin->qoutlen = sizeof(struct qout64); memcpy (qin->qname, seg->dcss_name, 8); From 304103736ba75be70991cd6336c28121d7bf14f5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 5 Feb 2024 21:32:12 +0100 Subject: [PATCH 028/117] s390/acrs: cleanup access register handling save_access_regs() and restore_access_regs() are only available by including switch_to.h. This is done by a couple of C files, which have nothing to do with switch_to(), but only need these functions. Move both functions to a new header file and improve the implementation: - Get rid of typedef - Add memory access instrumentation support - Use long displacement instructions lamy/stamy instead of lam/stam - all current users end up with better code because of this Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/access-regs.h | 38 +++++++++++++++++++++++++++++ arch/s390/include/asm/switch_to.h | 15 +----------- arch/s390/kernel/compat_signal.c | 2 +- arch/s390/kernel/early.c | 3 ++- arch/s390/kernel/ptrace.c | 1 + arch/s390/kernel/signal.c | 2 +- arch/s390/kernel/smp.c | 2 +- arch/s390/kvm/gaccess.c | 2 +- arch/s390/kvm/interrupt.c | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- 10 files changed, 48 insertions(+), 21 deletions(-) create mode 100644 arch/s390/include/asm/access-regs.h diff --git a/arch/s390/include/asm/access-regs.h b/arch/s390/include/asm/access-regs.h new file mode 100644 index 000000000000..1a6412d9f5ad --- /dev/null +++ b/arch/s390/include/asm/access-regs.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 1999, 2024 + */ + +#ifndef __ASM_S390_ACCESS_REGS_H +#define __ASM_S390_ACCESS_REGS_H + +#include +#include + +struct access_regs { + unsigned int regs[NUM_ACRS]; +}; + +static inline void save_access_regs(unsigned int *acrs) +{ + struct access_regs *regs = (struct access_regs *)acrs; + + instrument_write(regs, sizeof(*regs)); + asm volatile("stamy 0,15,%[regs]" + : [regs] "=QS" (*regs) + : + : "memory"); +} + +static inline void restore_access_regs(unsigned int *acrs) +{ + struct access_regs *regs = (struct access_regs *)acrs; + + instrument_read(regs, sizeof(*regs)); + asm volatile("lamy 0,15,%[regs]" + : + : [regs] "QS" (*regs) + : "memory"); +} + +#endif /* __ASM_S390_ACCESS_REGS_H */ diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h index c61b2cc1a8a8..8b0ca7d5f8b6 100644 --- a/arch/s390/include/asm/switch_to.h +++ b/arch/s390/include/asm/switch_to.h @@ -10,26 +10,13 @@ #include #include +#include #include #include extern struct task_struct *__switch_to(void *, void *); extern void update_cr_regs(struct task_struct *task); -static inline void save_access_regs(unsigned int *acrs) -{ - typedef struct { int _[NUM_ACRS]; } acrstype; - - asm volatile("stam 0,15,%0" : "=Q" (*(acrstype *)acrs)); -} - -static inline void restore_access_regs(unsigned int *acrs) -{ - typedef struct { int _[NUM_ACRS]; } acrstype; - - asm volatile("lam 0,15,%0" : : "Q" (*(acrstype *)acrs)); -} - #define switch_to(prev, next, last) do { \ /* save_fpu_regs() sets the CIF_FPU flag, which enforces \ * a restore of the floating point / vector registers as \ diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index f8fc6c25d051..13f41cb497be 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -24,10 +24,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include "compat_linux.h" diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 2345ea332b97..63d5b51cb45a 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -31,7 +33,6 @@ #include #include #include -#include #include "entry.h" #define decompressor_handled_param(param) \ diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index f1897a8bb221..d662ea9e54f5 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index 43e9661cd715..1517aa70678b 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -30,8 +30,8 @@ #include #include #include +#include #include -#include #include #include "entry.h" diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index c39d9f0d4b1c..f77a54e1e4bd 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -36,11 +36,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include #include diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 5bfcc50c1a68..d264aa81aa15 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -11,11 +11,11 @@ #include #include #include +#include #include #include #include "kvm-s390.h" #include "gaccess.h" -#include union asce { unsigned long val; diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index fc4007cc067a..6e435bf0c27e 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -19,13 +19,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ea63ac769889..6eab70b19abf 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -33,12 +33,12 @@ #include #include +#include #include #include #include #include #include -#include #include #include #include From 340750c13c3af2fc8cc4f993823a0b82b8a22845 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 5 Feb 2024 21:32:13 +0100 Subject: [PATCH 029/117] s390/switch_to: use generic header file Move the switch_to() implementation to process.c and use the generic switch_to.h header file instead, like some other architectures. This addresses also the oddity that the old switch_to() implementation assigns the return value of __switch_to() to 'prev' instead of 'last', like it should. Remove also all includes of switch_to.h from C files, except process.c. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ptrace.h | 4 ++++ arch/s390/include/asm/stacktrace.h | 1 - arch/s390/include/asm/switch_to.h | 36 ------------------------------ arch/s390/kernel/entry.S | 10 ++++----- arch/s390/kernel/entry.h | 1 + arch/s390/kernel/machine_kexec.c | 2 +- arch/s390/kernel/nmi.c | 1 - arch/s390/kernel/process.c | 23 ++++++++++++++++++- arch/s390/kernel/ptrace.c | 2 +- arch/s390/kernel/uprobes.c | 1 - drivers/s390/char/zcore.c | 1 - 11 files changed, 34 insertions(+), 48 deletions(-) delete mode 100644 arch/s390/include/asm/switch_to.h diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index d28bf8fb2799..788bc4467445 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -203,6 +203,10 @@ static inline int test_and_clear_pt_regs_flag(struct pt_regs *regs, int flag) return ret; } +struct task_struct; + +void update_cr_regs(struct task_struct *task); + /* * These are defined as per linux/ptrace.h, which see. */ diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index 31ec4f545e03..433fde85b14e 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -4,7 +4,6 @@ #include #include -#include struct stack_frame_user { unsigned long back_chain; diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h deleted file mode 100644 index 8b0ca7d5f8b6..000000000000 --- a/arch/s390/include/asm/switch_to.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright IBM Corp. 1999, 2009 - * - * Author(s): Martin Schwidefsky - */ - -#ifndef __ASM_SWITCH_TO_H -#define __ASM_SWITCH_TO_H - -#include -#include -#include -#include -#include - -extern struct task_struct *__switch_to(void *, void *); -extern void update_cr_regs(struct task_struct *task); - -#define switch_to(prev, next, last) do { \ - /* save_fpu_regs() sets the CIF_FPU flag, which enforces \ - * a restore of the floating point / vector registers as \ - * soon as the next task returns to user space \ - */ \ - save_fpu_regs(); \ - save_access_regs(&prev->thread.acrs[0]); \ - save_ri_cb(prev->thread.ri_cb); \ - save_gs_cb(prev->thread.gs_cb); \ - update_cr_regs(next); \ - restore_access_regs(&next->thread.acrs[0]); \ - restore_ri_cb(next->thread.ri_cb, prev->thread.ri_cb); \ - restore_gs_cb(next->thread.gs_cb); \ - prev = __switch_to(prev, next); \ -} while (0) - -#endif /* __ASM_SWITCH_TO_H */ diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 49a11f6dd7ae..f22cb6831f93 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -171,13 +171,13 @@ _LPP_OFFSET = __LC_LPP nop 0 /* - * Scheduler resume function, called by switch_to - * gpr2 = (task_struct *) prev - * gpr3 = (task_struct *) next + * Scheduler resume function, called by __switch_to + * gpr2 = (task_struct *)prev + * gpr3 = (task_struct *)next * Returns: * gpr2 = prev */ -SYM_FUNC_START(__switch_to) +SYM_FUNC_START(__switch_to_asm) stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task lghi %r4,__TASK_stack lghi %r1,__TASK_thread @@ -193,7 +193,7 @@ SYM_FUNC_START(__switch_to) lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40 BR_EX %r14 -SYM_FUNC_END(__switch_to) +SYM_FUNC_END(__switch_to_asm) #if IS_ENABLED(CONFIG_KVM) /* diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 9f41853f36b9..21969520f947 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -19,6 +19,7 @@ void mcck_int_handler(void); void restart_int_handler(void); void early_pgm_check_handler(void); +struct task_struct *__switch_to_asm(struct task_struct *prev, struct task_struct *next); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs); void __do_pgm_check(struct pt_regs *regs); void __do_syscall(struct pt_regs *regs, int per_trap); diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index aa22ffc16bcd..032a1124f04b 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -26,7 +27,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 9ad44c26d1a2..bbef8d954d01 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 4e3b366589fb..7227a73226f1 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -31,15 +31,18 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include #include #include #include #include -#include #include #include #include "entry.h" @@ -190,6 +193,24 @@ void execve_tail(void) asm volatile("sfpc %0" : : "d" (0)); } +struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) +{ + /* + * save_fpu_regs() sets the CIF_FPU flag, which enforces + * a restore of the floating point / vector registers as + * soon as the next task returns to user space. + */ + save_fpu_regs(); + save_access_regs(&prev->thread.acrs[0]); + save_ri_cb(prev->thread.ri_cb); + save_gs_cb(prev->thread.gs_cb); + update_cr_regs(next); + restore_access_regs(&next->thread.acrs[0]); + restore_ri_cb(next->thread.ri_cb, prev->thread.ri_cb); + restore_gs_cb(next->thread.gs_cb); + return __switch_to_asm(prev, next); +} + unsigned long __get_wchan(struct task_struct *p) { struct unwind_state state; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index d662ea9e54f5..7c5e99be2155 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -24,11 +24,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/uprobes.c b/arch/s390/kernel/uprobes.c index b88345ef8bd9..5b0633ea8d93 100644 --- a/arch/s390/kernel/uprobes.c +++ b/arch/s390/kernel/uprobes.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index bc3be0330f1d..0969fa01df58 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include "sclp.h" From 2151fd9a6d4ff8732a32f9ef2498fb7b95a0da29 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:53 -0700 Subject: [PATCH 030/117] s390/boot: add support for CONFIG_LD_ORPHAN_WARN arch/s390/boot/vmlinux uses a different linker script and build rules than the main vmlinux, so the '--orphan-handling' flag is not applied to it. Add support for '--orphan-handling' so that all sections are properly described in the linker script, which helps eliminate bugs between linker implementations having different orphan section heuristics. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-1-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index c7c81e5f9218..ace0bda1ad24 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -73,11 +73,12 @@ $(obj)/bzImage: $(obj)/vmlinux $(obj)/section_cmp.boot.data $(obj)/section_cmp.b $(obj)/section_cmp%: vmlinux $(obj)/vmlinux FORCE $(call if_changed,section_cmp) -LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup $(if $(CONFIG_VMLINUX_MAP),-Map=$(obj)/vmlinux.map) --build-id=sha1 -T +LDFLAGS_vmlinux-$(CONFIG_LD_ORPHAN_WARN) := --orphan-handling=$(CONFIG_LD_ORPHAN_WARN_LEVEL) +LDFLAGS_vmlinux := $(LDFLAGS_vmlinux-y) --oformat $(LD_BFD) -e startup $(if $(CONFIG_VMLINUX_MAP),-Map=$(obj)/vmlinux.map) --build-id=sha1 -T $(obj)/vmlinux: $(obj)/vmlinux.lds $(OBJECTS_ALL) FORCE $(call if_changed,ld) -LDFLAGS_vmlinux.syms := --oformat $(LD_BFD) -e startup -T +LDFLAGS_vmlinux.syms := $(LDFLAGS_vmlinux-y) --oformat $(LD_BFD) -e startup -T $(obj)/vmlinux.syms: $(obj)/vmlinux.lds $(OBJECTS) FORCE $(call if_changed,ld) From bdf2cd27a3293618c794c74bd2f4dee32bc6f477 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:54 -0700 Subject: [PATCH 031/117] s390: vmlinux.lds.S: handle '.data.rel' sections explicitly When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there are a lot of warnings around '.data.rel' sections: s390-linux-ld: warning: orphan section `.data.rel' from `kernel/sched/build_utility.o' being placed in section `.data.rel' s390-linux-ld: warning: orphan section `.data.rel.local' from `kernel/sched/build_utility.o' being placed in section `.data.rel.local' s390-linux-ld: warning: orphan section `.data.rel.ro' from `kernel/sched/build_utility.o' being placed in section `.data.rel.ro' s390-linux-ld: warning: orphan section `.data.rel.ro.local' from `kernel/sched/build_utility.o' being placed in section `.data.rel.ro.local' Describe these in vmlinux.lds.S so there is no more warning and the sections are placed consistently between linkers. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-2-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/kernel/vmlinux.lds.S | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index e32ef446f451..d231a3faf981 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -59,6 +59,9 @@ SECTIONS } :text = 0x0700 RO_DATA(PAGE_SIZE) + .data.rel.ro : { + *(.data.rel.ro .data.rel.ro.*) + } . = ALIGN(PAGE_SIZE); _sdata = .; /* Start of data section */ @@ -73,6 +76,9 @@ SECTIONS __end_ro_after_init = .; RW_DATA(0x100, PAGE_SIZE, THREAD_SIZE) + .data.rel : { + *(.data.rel*) + } BOOT_DATA_PRESERVED . = ALIGN(8); From 30226853d6ecd274ec35b9d7b8dcdfc1e0981f05 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:55 -0700 Subject: [PATCH 032/117] s390: vmlinux.lds.S: explicitly handle '.got' and '.plt' sections When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there are a lot of warnings around the GOT and PLT sections: s390-linux-ld: warning: orphan section `.plt' from `arch/s390/kernel/head64.o' being placed in section `.plt' s390-linux-ld: warning: orphan section `.got' from `arch/s390/kernel/head64.o' being placed in section `.got' s390-linux-ld: warning: orphan section `.got.plt' from `arch/s390/kernel/head64.o' being placed in section `.got.plt' s390-linux-ld: warning: orphan section `.iplt' from `arch/s390/kernel/head64.o' being placed in section `.iplt' s390-linux-ld: warning: orphan section `.igot.plt' from `arch/s390/kernel/head64.o' being placed in section `.igot.plt' s390-linux-ld: warning: orphan section `.iplt' from `arch/s390/boot/head.o' being placed in section `.iplt' s390-linux-ld: warning: orphan section `.igot.plt' from `arch/s390/boot/head.o' being placed in section `.igot.plt' s390-linux-ld: warning: orphan section `.got' from `arch/s390/boot/head.o' being placed in section `.got' Currently, only the '.got' section is actually emitted in the final binary. In a manner similar to other architectures, put the '.got' section near the '.data' section and coalesce the PLT sections, checking that the final section is zero sized, which is a safe/tested approach versus full discard. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-3-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 16 ++++++++++++++++ arch/s390/kernel/vmlinux.lds.S | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 389df0e0d9e5..cb4e95c691ae 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -39,6 +39,9 @@ SECTIONS *(.rodata.*) _erodata = . ; } + .got : { + *(.got) + } NOTES .data : { _data = . ; @@ -118,6 +121,19 @@ SECTIONS } _end = .; + /* + * Sections that should stay zero sized, which is safer to + * explicitly check instead of blindly discarding. + */ + .got.plt : { + *(.got.plt) + } + ASSERT(SIZEOF(.got.plt) == 0, "Unexpected GOT/PLT entries detected!") + .plt : { + *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) + } + ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!") + /* Sections to be discarded */ /DISCARD/ : { *(.eh_frame) diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index d231a3faf981..725e3122c214 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -62,6 +62,9 @@ SECTIONS .data.rel.ro : { *(.data.rel.ro .data.rel.ro.*) } + .got : { + *(.got) + } . = ALIGN(PAGE_SIZE); _sdata = .; /* Start of data section */ @@ -241,6 +244,19 @@ SECTIONS DWARF_DEBUG ELF_DETAILS + /* + * Sections that should stay zero sized, which is safer to + * explicitly check instead of blindly discarding. + */ + .got.plt : { + *(.got.plt) + } + ASSERT(SIZEOF(.got.plt) == 0, "Unexpected GOT/PLT entries detected!") + .plt : { + *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) + } + ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!") + /* Sections to be discarded */ DISCARDS /DISCARD/ : { From a691c8a6efe0220f244f7bb59a3af39383b57d2c Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:56 -0700 Subject: [PATCH 033/117] s390: vmlinux.lds.S: explicitly keep various sections When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there are some warnings around certain ELF sections: s390-linux-ld: warning: orphan section `.dynstr' from `arch/s390/kernel/head64.o' being placed in section `.dynstr' s390-linux-ld: warning: orphan section `.dynamic' from `arch/s390/kernel/head64.o' being placed in section `.dynamic' s390-linux-ld: warning: orphan section `.hash' from `arch/s390/kernel/head64.o' being placed in section `.hash' s390-linux-ld: warning: orphan section `.gnu.hash' from `arch/s390/kernel/head64.o' being placed in section `.gnu.hash' Explicitly keep those sections like other architectures when CONFIG_RELOCATABLE is enabled, which is always true for s390. [hca@linux.ibm.com: keep sections instead of discarding] Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-4-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/kernel/vmlinux.lds.S | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 725e3122c214..63cb403b3344 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -200,6 +200,18 @@ SECTIONS *(.rela*) __rela_dyn_end = .; } + .dynamic ALIGN(8) : { + *(.dynamic) + } + .dynstr ALIGN(8) : { + *(.dynstr) + } + .hash ALIGN(8) : { + *(.hash) + } + .gnu.hash ALIGN(8) : { + *(.gnu.hash) + } . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */ From b23ab303dd95d940a3cb718ef15323fb428cba63 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:57 -0700 Subject: [PATCH 034/117] s390/boot: vmlinux.lds.S: handle '.init.text' When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there is a warning about the presence of an '.init.text' section in arch/s390/boot: s390-linux-ld: warning: orphan section `.init.text' from `arch/s390/boot/sclp_early_core.o' being placed in section `.init.text' arch/s390/boot/sclp_early_core.c includes a file from the main kernel build, which picks up a usage of '__init' somewhere. For the decompressed image, this section can just be coalesced into '.text'. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-5-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index cb4e95c691ae..5692a72e9cd6 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -31,6 +31,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + INIT_TEXT _etext = . ; } .rodata : { From 64d590a24f7a570fd8d25d05edd4f433e32900a8 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:58 -0700 Subject: [PATCH 035/117] s390/boot: vmlinux.lds.S: handle '.rela' sections When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there are several warnings from arch/s390/boot/head.o due to the unhandled presence of '.rela' sections: s390-linux-ld: warning: orphan section `.rela.iplt' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.head.text' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.got' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.data' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.data.rel.ro' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.iplt' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.head.text' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.got' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.data' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' s390-linux-ld: warning: orphan section `.rela.data.rel.ro' from `arch/s390/boot/head.o' being placed in section `.rela.dyn' These sections are unneeded for the decompressor and they are not emitted in the binary currently. In a manner similar to other architectures, coalesce the sections into '.rela.dyn' and ensure it is zero sized, which is a safe/tested approach versus full discard. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-6-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 5692a72e9cd6..ba6520a9541b 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -134,6 +134,10 @@ SECTIONS *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) } ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!") + .rela.dyn : { + *(.rela.*) *(.rela_*) + } + ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") /* Sections to be discarded */ /DISCARD/ : { From ba6c26af1ee72f1812cbdac899416e7e1afa1fdb Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:14:59 -0700 Subject: [PATCH 036/117] s390/boot: vmlinux.lds.S: handle DWARF debug sections When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there are several series of warnings for each file in arch/s390/boot due to the boot linker script not handling the DWARF debug sections: s390-linux-ld: warning: orphan section `.debug_line' from `arch/s390/boot/head.o' being placed in section `.debug_line' s390-linux-ld: warning: orphan section `.debug_info' from `arch/s390/boot/head.o' being placed in section `.debug_info' s390-linux-ld: warning: orphan section `.debug_abbrev' from `arch/s390/boot/head.o' being placed in section `.debug_abbrev' s390-linux-ld: warning: orphan section `.debug_aranges' from `arch/s390/boot/head.o' being placed in section `.debug_aranges' s390-linux-ld: warning: orphan section `.debug_str' from `arch/s390/boot/head.o' being placed in section `.debug_str' include/asm-generic/vmlinux.lds.h has a macro for DWARF debug sections named DWARF_DEBUG, use it to clear up the warnings. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Reviewed-by: Fangrui Song Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-7-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index ba6520a9541b..722025dec0ee 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -122,6 +122,8 @@ SECTIONS } _end = .; + DWARF_DEBUG + /* * Sections that should stay zero sized, which is safer to * explicitly check instead of blindly discarding. From 6a4d37c886c1324c10452c06192bf6d7ebb7c8af Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:15:00 -0700 Subject: [PATCH 037/117] s390/boot: vmlinux.lds.S: handle ELF required sections When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there is a warning around the '.comment' section for each file in arch/s390/boot s390-linux-ld: warning: orphan section `.comment' from `arch/s390/boot/als.o' being placed in section `.comment' s390-linux-ld: warning: orphan section `.comment' from `arch/s390/boot/startup.o' being placed in section `.comment' s390-linux-ld: warning: orphan section `.comment' from `arch/s390/boot/physmem_info.o' being placed in section `.comment' include/asm-generic/vmlinux.lds.h has a macro for required ELF sections not related to debugging named ELF_DETAILS, use it to clear up the warnings. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-8-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 722025dec0ee..fb599e13dfe9 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -123,6 +123,7 @@ SECTIONS _end = .; DWARF_DEBUG + ELF_DETAILS /* * Sections that should stay zero sized, which is safer to From c0f98ea0e7eafb3ca5ea4a178de6d0ce36333ae7 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:15:01 -0700 Subject: [PATCH 038/117] s390/boot: vmlinux.lds.S: handle commonly discarded sections When building with CONFIG_LD_ORPHAN_WARN after selecting CONFIG_ARCH_HAS_LD_ORPHAN_WARN, there are several series of warnings from the various discardable sections that the kernel adds for build purposes that are not needed at runtime: s390-linux-ld: warning: orphan section `.export_symbol' from `arch/s390/boot/decompressor.o' being placed in section `.export_symbol' s390-linux-ld: warning: orphan section `.discard.addressable' from `arch/s390/boot/decompressor.o' being placed in section `.discard.addressable' s390-linux-ld: warning: orphan section `.modinfo' from `arch/s390/boot/decompressor.o' being placed in section `.modinfo' include/asm-generic/vmlinux.lds.h has a macro for easily discarding these sections across the kernel named COMMON_DISCARDS, use it to clear up the warnings. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-9-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index fb599e13dfe9..4e3fb2b7abb6 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -144,6 +144,7 @@ SECTIONS /* Sections to be discarded */ /DISCARD/ : { + COMMON_DISCARDS *(.eh_frame) *(__ex_table) *(*__ksymtab*) From acb7c202baa76235ed478043809366e7de1741c2 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:15:02 -0700 Subject: [PATCH 039/117] s390: select CONFIG_ARCH_WANT_LD_ORPHAN_WARN Now that all sections have been properly accounted for in the s390 linker scripts, select CONFIG_ARCH_WANT_LD_ORPHAN_WARN so that '--orphan-handling' is added to LDFLAGS to catch any future sections that are added without being described in linker scripts. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-10-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index fe565f3a3a91..771235aee6bf 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -127,6 +127,7 @@ config S390 select ARCH_WANT_DEFAULT_BPF_JIT select ARCH_WANT_IPC_PARSE_VERSION select ARCH_WANT_KERNEL_PMD_MKWRITE + select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 From 55cce52f1cec914870a12e8e9f82659037648cb8 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 7 Feb 2024 17:15:03 -0700 Subject: [PATCH 040/117] s390: link vmlinux with '-z notext' ld.bfd defaults to '-z notext' (although it is customizable with the '--enable-textrel-check' configure option) but ld.lld defaults to '-z text', which causes issues with building the kernel due to the presence of dynamic relocations in sections that are not writable. ld.lld: error: relocation R_390_64 cannot be used against local symbol; recompile with -fPIC Add '-z notext' to avoid these errors, as this is expected, which matches other architectures. Signed-off-by: Nathan Chancellor Tested-by: Justin Stitt Reviewed-by: Fangrui Song Link: https://lore.kernel.org/r/20240207-s390-lld-and-orphan-warn-v1-11-8a665b3346ab@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 73873e451686..994f9b3d575f 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -15,7 +15,7 @@ KBUILD_CFLAGS_MODULE += -fPIC KBUILD_AFLAGS += -m64 KBUILD_CFLAGS += -m64 KBUILD_CFLAGS += -fPIE -LDFLAGS_vmlinux := -pie +LDFLAGS_vmlinux := -pie -z notext aflags_dwarf := -Wa,-gdwarf-2 KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ ifndef CONFIG_AS_IS_LLVM From 3938490e78f443fb0b734a15a50abba020638ff8 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 9 Feb 2024 11:48:25 +0100 Subject: [PATCH 041/117] s390/bug: remove entry size from __bug_table section Commit e21f8baf8d9a ("s390/bug: add entry size to the __bug_table section") changed the __EMIT_BUG() inline assembly to emit mergeable __bug_table entries. This is at least currently not needed, but causes problems with the upcoming s390 ld.lld support: ld.lld: error: drivers/nvme/host/fc.o:(__bug_table): writable SHF_MERGE section is not supported Therefore revert the change for now. Reported-by: Nathan Chancellor Closes: https://lore.kernel.org/all/20240207-s390-lld-and-orphan-warn-v1-0-8a665b3346ab@kernel.org/ Suggested-by: Nathan Chancellor Reviewed-by: Nathan Chancellor Signed-off-by: Heiko Carstens --- arch/s390/include/asm/bug.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index aebe1e22c7be..c500d45fb465 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -14,7 +14,7 @@ ".section .rodata.str,\"aMS\",@progbits,1\n" \ "1: .asciz \""__FILE__"\"\n" \ ".previous\n" \ - ".section __bug_table,\"awM\",@progbits,%2\n" \ + ".section __bug_table,\"aw\"\n" \ "2: .long 0b-.\n" \ " .long 1b-.\n" \ " .short %0,%1\n" \ @@ -30,7 +30,7 @@ #define __EMIT_BUG(x) do { \ asm_inline volatile( \ "0: mc 0,0\n" \ - ".section __bug_table,\"awM\",@progbits,%1\n" \ + ".section __bug_table,\"aw\"\n" \ "1: .long 0b-.\n" \ " .short %0\n" \ " .org 1b+%1\n" \ From 616c4ea9bce426aa6efd6dc333bdd479ce352df0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 9 Feb 2024 11:54:01 +0100 Subject: [PATCH 042/117] s390/vdso: remove unused ENTRY in linker scripts When linking vdso64.so.dbg with ld.lld, there is a warning about not finding _start for the starting address: ld.lld: warning: cannot find entry symbol _start; not setting start address Fix this by removing the unused ENTRY in both vdso linker scripts. See commit e247172854a5 ("powerpc/vdso: Remove unused ENTRY in linker scripts"), which solved the same problem for powerpc, for further details. Reviewed-by: Nathan Chancellor Signed-off-by: Heiko Carstens --- arch/s390/kernel/vdso32/vdso32.lds.S | 1 - arch/s390/kernel/vdso64/vdso64.lds.S | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/s390/kernel/vdso32/vdso32.lds.S b/arch/s390/kernel/vdso32/vdso32.lds.S index edf5ff1debe1..65b9513a5a0e 100644 --- a/arch/s390/kernel/vdso32/vdso32.lds.S +++ b/arch/s390/kernel/vdso32/vdso32.lds.S @@ -9,7 +9,6 @@ OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390") OUTPUT_ARCH(s390:31-bit) -ENTRY(_start) SECTIONS { diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S index 4461ea151e49..37e2a505e81d 100644 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ b/arch/s390/kernel/vdso64/vdso64.lds.S @@ -9,7 +9,6 @@ OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") OUTPUT_ARCH(s390:64-bit) -ENTRY(_start) SECTIONS { From b20ea29a709e46e929b33713f4bfdf51db3d99be Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 14 Feb 2024 15:53:48 -0700 Subject: [PATCH 043/117] s390: don't allow CONFIG_COMPAT with LD=ld.lld When building 'ARCH=s390 defconfig compat.config' with GCC and LD=ld.lld, there is an error when attempting to link the compat vDSO: ld.lld: error: unknown emulation: elf_s390 make[4]: *** [arch/s390/kernel/vdso32/Makefile:48: arch/s390/kernel/vdso32/vdso32.so.dbg] Error 1 Much like clang, ld.lld only supports the 64-bit s390 emulation. Add a dependency on not using LLD to CONFIG_COMPAT to avoid breaking the build with this toolchain combination. Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240214-s390-compat-lld-dep-v1-1-abf1f4b5e514@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 771235aee6bf..39e937309fc2 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -449,7 +449,7 @@ config COMPAT select COMPAT_OLD_SIGACTION select HAVE_UID16 depends on MULTIUSER - depends on !CC_IS_CLANG + depends on !CC_IS_CLANG && !LD_IS_LLD help Select this option if you want to enable your system kernel to handle system-calls from ELF binaries for 31 bit ESA. This option From e8054eaeb5a57d1aa930962185e74060db0e9308 Mon Sep 17 00:00:00 2001 From: Gerald Schaefer Date: Wed, 14 Feb 2024 16:17:08 +0100 Subject: [PATCH 044/117] s390/setup: fix virtual vs physical address confusion Fix virtual vs physical address confusion. This does not fix a bug since virtual and physical address spaces are currently the same. /proc/iomem should report the physical address ranges, so use __pa_symbol() for resource registration, similar to other architectures. Signed-off-by: Gerald Schaefer Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/kernel/setup.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d1f3b56e7afc..24ed33f044ec 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -504,12 +504,12 @@ static void __init setup_resources(void) int j; u64 i; - code_resource.start = (unsigned long) _text; - code_resource.end = (unsigned long) _etext - 1; - data_resource.start = (unsigned long) _etext; - data_resource.end = (unsigned long) _edata - 1; - bss_resource.start = (unsigned long) __bss_start; - bss_resource.end = (unsigned long) __bss_stop - 1; + code_resource.start = __pa_symbol(_text); + code_resource.end = __pa_symbol(_etext) - 1; + data_resource.start = __pa_symbol(_etext); + data_resource.end = __pa_symbol(_edata) - 1; + bss_resource.start = __pa_symbol(__bss_start); + bss_resource.end = __pa_symbol(__bss_stop) - 1; for_each_mem_range(i, &start, &end) { res = memblock_alloc(sizeof(*res), 8); From 88e4c0da9b08fb8dd52840922837589157455449 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 29 Jan 2024 16:53:21 +0100 Subject: [PATCH 045/117] s390/zcrypt: harmonize debug feature calls and defines This patch harmonizes the calls and defines around the s390 debug feature as it is used in the AP bus and zcrypt device driver code. More or less cleanup and renaming, no functional changes. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 2 +- drivers/s390/crypto/ap_debug.h | 2 +- drivers/s390/crypto/zcrypt_api.c | 2 +- drivers/s390/crypto/zcrypt_ccamisc.c | 214 ++++++++++++-------------- drivers/s390/crypto/zcrypt_debug.h | 2 +- drivers/s390/crypto/zcrypt_ep11misc.c | 127 +++++++-------- 6 files changed, 156 insertions(+), 193 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 2ecf4d36e78b..732cd8200d0e 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -2190,7 +2190,7 @@ static void ap_config_timeout(struct timer_list *unused) static int __init ap_debug_init(void) { ap_dbf_info = debug_register("ap", 2, 1, - DBF_MAX_SPRINTF_ARGS * sizeof(long)); + AP_DBF_MAX_SPRINTF_ARGS * sizeof(long)); debug_register_view(ap_dbf_info, &debug_sprintf_view); debug_set_level(ap_dbf_info, DBF_ERR); diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h index c083ce88a9a6..9c53b64fb738 100644 --- a/drivers/s390/crypto/ap_debug.h +++ b/drivers/s390/crypto/ap_debug.h @@ -16,7 +16,7 @@ #define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO) #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO) -#define DBF_MAX_SPRINTF_ARGS 6 +#define AP_DBF_MAX_SPRINTF_ARGS 6 #define AP_DBF(...) \ debug_sprintf_event(ap_dbf_info, ##__VA_ARGS__) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 74200f54dfff..55c163610475 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -2038,7 +2038,7 @@ EXPORT_SYMBOL(zcrypt_wait_api_operational); int __init zcrypt_debug_init(void) { zcrypt_dbf_info = debug_register("zcrypt", 2, 1, - DBF_MAX_SPRINTF_ARGS * sizeof(long)); + ZCRYPT_DBF_MAX_SPRINTF_ARGS * sizeof(long)); debug_register_view(zcrypt_dbf_info, &debug_sprintf_view); debug_set_level(zcrypt_dbf_info, DBF_ERR); diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 263fe182648b..0a3a678ffc7e 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -23,11 +23,6 @@ #include "zcrypt_msgtype6.h" #include "zcrypt_ccamisc.h" -#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) -#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) -#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) -#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) - /* Size of parameter block used for all cca requests/replies */ #define PARMBSIZE 512 @@ -367,8 +362,8 @@ int cca_genseckey(u16 cardnr, u16 domain, memcpy(preqparm->lv1.key_length, "KEYLN32 ", 8); break; default: - DEBUG_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -386,15 +381,15 @@ int cca_genseckey(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, errno %d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s secure key generate failure, card response %d/%d\n", - __func__, + ZCRYPT_DBF_ERR("%s secure key generate failure, card response %d/%d\n", + __func__, (int)prepcblk->ccp_rtcode, (int)prepcblk->ccp_rscode); rc = -EIO; @@ -411,8 +406,8 @@ int cca_genseckey(u16 cardnr, u16 domain, - sizeof(prepparm->lv3.keyblock.toklen) - sizeof(prepparm->lv3.keyblock.tokattr); if (seckeysize != SECKEYBLOBSIZE) { - DEBUG_ERR("%s secure token size mismatch %d != %d bytes\n", - __func__, seckeysize, SECKEYBLOBSIZE); + ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n", + __func__, seckeysize, SECKEYBLOBSIZE); rc = -EIO; goto out; } @@ -505,8 +500,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, keysize = 32; break; default: - DEBUG_ERR("%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -524,17 +519,17 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s clear key import failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s clear key import failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -549,8 +544,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, - sizeof(prepparm->lv3.keyblock.toklen) - sizeof(prepparm->lv3.keyblock.tokattr); if (seckeysize != SECKEYBLOBSIZE) { - DEBUG_ERR("%s secure token size mismatch %d != %d bytes\n", - __func__, seckeysize, SECKEYBLOBSIZE); + ZCRYPT_DBF_ERR("%s secure token size mismatch %d != %d bytes\n", + __func__, seckeysize, SECKEYBLOBSIZE); rc = -EIO; goto out; } @@ -651,17 +646,17 @@ int cca_sec2protkey(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) rc = -EAGAIN; else @@ -669,10 +664,10 @@ int cca_sec2protkey(u16 cardnr, u16 domain, goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN("%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -683,8 +678,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, /* check the returned keyblock */ if (prepparm->lv3.ckb.version != 0x01 && prepparm->lv3.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x\n", - __func__, (int)prepparm->lv3.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n", + __func__, (int)prepparm->lv3.ckb.version); rc = -EIO; goto out; } @@ -707,8 +702,8 @@ int cca_sec2protkey(u16 cardnr, u16 domain, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keylen %d\n", - __func__, prepparm->lv3.ckb.len); + ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n", + __func__, prepparm->lv3.ckb.len); rc = -EIO; goto out; } @@ -840,9 +835,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, case 256: break; default: - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -880,19 +874,17 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s cipher key generate failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s cipher key generate failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -905,8 +897,8 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, /* do some plausibility checks on the key block */ if (prepparm->kb.len < 120 + 5 * sizeof(uint16_t) || prepparm->kb.len > 136 + 5 * sizeof(uint16_t)) { - DEBUG_ERR("%s reply with invalid or unknown key block\n", - __func__); + ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n", + __func__); rc = -EIO; goto out; } @@ -1048,19 +1040,17 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s CSNBKPI2 failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s CSNBKPI2 failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } @@ -1073,8 +1063,8 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, /* do some plausibility checks on the key block */ if (prepparm->kb.len < 120 + 3 * sizeof(uint16_t) || prepparm->kb.len > 136 + 3 * sizeof(uint16_t)) { - DEBUG_ERR("%s reply with invalid or unknown key block\n", - __func__); + ZCRYPT_DBF_ERR("%s reply with invalid or unknown key block\n", + __func__); rc = -EIO; goto out; } @@ -1132,33 +1122,29 @@ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, rc = _ip_cprb_helper(card, dom, "AES ", "FIRST ", "MIN3PART", exorbuf, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, clrkey, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, exorbuf, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "COMPLETE", NULL, NULL, keybitsize, token, &tokensize); if (rc) { - DEBUG_ERR( - "%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", + __func__, rc); goto out; } @@ -1265,19 +1251,17 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) rc = -EAGAIN; else @@ -1285,11 +1269,10 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN( - "%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -1300,15 +1283,14 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, /* check the returned keyblock */ if (prepparm->vud.ckb.version != 0x01 && prepparm->vud.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x\n", - __func__, (int)prepparm->vud.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x\n", + __func__, (int)prepparm->vud.ckb.version); rc = -EIO; goto out; } if (prepparm->vud.ckb.algo != 0x02) { - DEBUG_ERR( - "%s reply param keyblock algo mismatch 0x%02x != 0x02\n", - __func__, (int)prepparm->vud.ckb.algo); + ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x02\n", + __func__, (int)prepparm->vud.ckb.algo); rc = -EIO; goto out; } @@ -1331,8 +1313,8 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keylen %d\n", - __func__, prepparm->vud.ckb.keylen); + ZCRYPT_DBF_ERR("%s unknown/unsupported keylen %d\n", + __func__, prepparm->vud.ckb.keylen); rc = -EIO; goto out; } @@ -1432,19 +1414,17 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR( - "%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); if (prepcblk->ccp_rtcode == 8 && prepcblk->ccp_rscode == 2290) rc = -EAGAIN; else @@ -1452,11 +1432,10 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, goto out; } if (prepcblk->ccp_rscode != 0) { - DEBUG_WARN( - "%s unwrap secure key warning, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_WARN("%s unwrap secure key warning, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); } /* process response cprb param block */ @@ -1466,23 +1445,22 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, /* check the returned keyblock */ if (prepparm->vud.ckb.version != 0x02) { - DEBUG_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", - __func__, (int)prepparm->vud.ckb.version); + ZCRYPT_DBF_ERR("%s reply param keyblock version mismatch 0x%02x != 0x02\n", + __func__, (int)prepparm->vud.ckb.version); rc = -EIO; goto out; } if (prepparm->vud.ckb.algo != 0x81) { - DEBUG_ERR( - "%s reply param keyblock algo mismatch 0x%02x != 0x81\n", - __func__, (int)prepparm->vud.ckb.algo); + ZCRYPT_DBF_ERR("%s reply param keyblock algo mismatch 0x%02x != 0x81\n", + __func__, (int)prepparm->vud.ckb.algo); rc = -EIO; goto out; } /* copy the translated protected key */ if (prepparm->vud.ckb.keylen > *protkeylen) { - DEBUG_ERR("%s prot keylen mismatch %d > buffersize %u\n", - __func__, prepparm->vud.ckb.keylen, *protkeylen); + ZCRYPT_DBF_ERR("%s prot keylen mismatch %d > buffersize %u\n", + __func__, prepparm->vud.ckb.keylen, *protkeylen); rc = -EIO; goto out; } @@ -1550,17 +1528,17 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, /* forward xcrb with request CPRB and reply CPRB to zcrypt dd */ rc = zcrypt_send_cprb(&xcrb); if (rc) { - DEBUG_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_cprb (cardnr=%d domain=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } /* check response returncode and reasoncode */ if (prepcblk->ccp_rtcode != 0) { - DEBUG_ERR("%s unwrap secure key failure, card response %d/%d\n", - __func__, - (int)prepcblk->ccp_rtcode, - (int)prepcblk->ccp_rscode); + ZCRYPT_DBF_ERR("%s unwrap secure key failure, card response %d/%d\n", + __func__, + (int)prepcblk->ccp_rtcode, + (int)prepcblk->ccp_rscode); rc = -EIO; goto out; } diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h index 5cf88aabd64b..8e947a23a359 100644 --- a/drivers/s390/crypto/zcrypt_debug.h +++ b/drivers/s390/crypto/zcrypt_debug.h @@ -17,7 +17,7 @@ #define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO) #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO) -#define DBF_MAX_SPRINTF_ARGS 6 +#define ZCRYPT_DBF_MAX_SPRINTF_ARGS 6 #define ZCRYPT_DBF(...) \ debug_sprintf_event(zcrypt_dbf_info, ##__VA_ARGS__) diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index 0a877f9792c2..eb7f5489ccf9 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -24,11 +24,6 @@ #include "zcrypt_ep11misc.h" #include "zcrypt_ccamisc.h" -#define DEBUG_DBG(...) ZCRYPT_DBF(DBF_DEBUG, ##__VA_ARGS__) -#define DEBUG_INFO(...) ZCRYPT_DBF(DBF_INFO, ##__VA_ARGS__) -#define DEBUG_WARN(...) ZCRYPT_DBF(DBF_WARN, ##__VA_ARGS__) -#define DEBUG_ERR(...) ZCRYPT_DBF(DBF_ERR, ##__VA_ARGS__) - #define EP11_PINBLOB_V1_BYTES 56 /* default iv used here */ @@ -510,7 +505,7 @@ static int check_reply_pl(const u8 *pl, const char *func) /* start tag */ if (*pl++ != 0x30) { - DEBUG_ERR("%s reply start tag mismatch\n", func); + ZCRYPT_DBF_ERR("%s reply start tag mismatch\n", func); return -EIO; } @@ -527,40 +522,41 @@ static int check_reply_pl(const u8 *pl, const char *func) len = *((u16 *)pl); pl += 2; } else { - DEBUG_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", - func, *pl); + ZCRYPT_DBF_ERR("%s reply start tag lenfmt mismatch 0x%02hhx\n", + func, *pl); return -EIO; } /* len should cover at least 3 fields with 32 bit value each */ if (len < 3 * 6) { - DEBUG_ERR("%s reply length %d too small\n", func, len); + ZCRYPT_DBF_ERR("%s reply length %d too small\n", func, len); return -EIO; } /* function tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s function tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s function tag or length mismatch\n", func); return -EIO; } pl += 6; /* dom tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s dom tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s dom tag or length mismatch\n", func); return -EIO; } pl += 6; /* return value tag, length and value */ if (pl[0] != 0x04 || pl[1] != 0x04) { - DEBUG_ERR("%s return value tag or length mismatch\n", func); + ZCRYPT_DBF_ERR("%s return value tag or length mismatch\n", + func); return -EIO; } pl += 2; ret = *((u32 *)pl); if (ret != 0) { - DEBUG_ERR("%s return value 0x%04x != 0\n", func, ret); + ZCRYPT_DBF_ERR("%s return value 0x%04x != 0\n", func, ret); return -EIO; } @@ -626,9 +622,8 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)cardnr, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)cardnr, (int)domain, rc); goto out; } @@ -636,13 +631,13 @@ static int ep11_query_info(u16 cardnr, u16 domain, u32 query_type, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > buflen) { - DEBUG_ERR("%s mismatch between reply data len and buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch between reply data len and buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -816,9 +811,8 @@ static int _ep11_genaeskey(u16 card, u16 domain, case 256: break; default: - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); rc = -EINVAL; goto out; } @@ -878,9 +872,8 @@ static int _ep11_genaeskey(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -888,13 +881,13 @@ static int _ep11_genaeskey(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *keybufsize) { - DEBUG_ERR("%s mismatch reply data len / key buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1030,9 +1023,8 @@ static int ep11_cryptsingle(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -1040,7 +1032,7 @@ static int ep11_cryptsingle(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } @@ -1053,14 +1045,14 @@ static int ep11_cryptsingle(u16 card, u16 domain, n = *((u16 *)p); p += 2; } else { - DEBUG_ERR("%s unknown reply data length format 0x%02hhx\n", - __func__, rep_pl->data_lenfmt); + ZCRYPT_DBF_ERR("%s unknown reply data length format 0x%02hhx\n", + __func__, rep_pl->data_lenfmt); rc = -EIO; goto out; } if (n > *outbufsize) { - DEBUG_ERR("%s mismatch reply data len %d / output buffer %zu\n", - __func__, n, *outbufsize); + ZCRYPT_DBF_ERR("%s mismatch reply data len %d / output buffer %zu\n", + __func__, n, *outbufsize); rc = -ENOSPC; goto out; } @@ -1188,9 +1180,8 @@ static int _ep11_unwrapkey(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -1198,13 +1189,13 @@ static int _ep11_unwrapkey(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *keybufsize) { - DEBUG_ERR("%s mismatch reply data len / key buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / key buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1343,9 +1334,8 @@ static int _ep11_wrapkey(u16 card, u16 domain, rc = zcrypt_send_ep11_cprb(urb); if (rc) { - DEBUG_ERR( - "%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", - __func__, (int)card, (int)domain, rc); + ZCRYPT_DBF_ERR("%s zcrypt_send_ep11_cprb(card=%d dom=%d) failed, rc=%d\n", + __func__, (int)card, (int)domain, rc); goto out; } @@ -1353,13 +1343,13 @@ static int _ep11_wrapkey(u16 card, u16 domain, if (rc) goto out; if (rep_pl->data_tag != 0x04 || rep_pl->data_lenfmt != 0x82) { - DEBUG_ERR("%s unknown reply data format\n", __func__); + ZCRYPT_DBF_ERR("%s unknown reply data format\n", __func__); rc = -EIO; goto out; } if (rep_pl->data_len > *datasize) { - DEBUG_ERR("%s mismatch reply data len / data buffer len\n", - __func__); + ZCRYPT_DBF_ERR("%s mismatch reply data len / data buffer len\n", + __func__); rc = -ENOSPC; goto out; } @@ -1386,9 +1376,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, if (keybitsize == 128 || keybitsize == 192 || keybitsize == 256) { clrkeylen = keybitsize / 8; } else { - DEBUG_ERR( - "%s unknown/unsupported keybitsize %d\n", - __func__, keybitsize); + ZCRYPT_DBF_ERR("%s unknown/unsupported keybitsize %d\n", + __func__, keybitsize); return -EINVAL; } @@ -1405,9 +1394,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, 0x00006c00, /* EN/DECRYPT, WRAP/UNWRAP */ kek, &keklen); if (rc) { - DEBUG_ERR( - "%s generate kek key failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s generate kek key failed, rc=%d\n", + __func__, rc); goto out; } @@ -1415,9 +1403,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, rc = ep11_cryptsingle(card, domain, 0, 0, def_iv, kek, keklen, clrkey, clrkeylen, encbuf, &encbuflen); if (rc) { - DEBUG_ERR( - "%s encrypting key value with kek key failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s encrypting key value with kek key failed, rc=%d\n", + __func__, rc); goto out; } @@ -1426,9 +1413,8 @@ int ep11_clr2keyblob(u16 card, u16 domain, u32 keybitsize, u32 keygenflags, encbuf, encbuflen, 0, def_iv, keybitsize, 0, keybuf, keybufsize, keytype); if (rc) { - DEBUG_ERR( - "%s importing key value as new key failed,, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s importing key value as new key failed,, rc=%d\n", + __func__, rc); goto out; } @@ -1476,17 +1462,16 @@ int ep11_kblob2protkey(u16 card, u16 dom, rc = _ep11_wrapkey(card, dom, (u8 *)key, keylen, 0, def_iv, wkbuf, &wkbuflen); if (rc) { - DEBUG_ERR( - "%s rewrapping ep11 key to pkey failed, rc=%d\n", - __func__, rc); + ZCRYPT_DBF_ERR("%s rewrapping ep11 key to pkey failed, rc=%d\n", + __func__, rc); goto out; } wki = (struct wk_info *)wkbuf; /* check struct version and pkey type */ if (wki->version != 1 || wki->pkeytype < 1 || wki->pkeytype > 5) { - DEBUG_ERR("%s wk info version %d or pkeytype %d mismatch.\n", - __func__, (int)wki->version, (int)wki->pkeytype); + ZCRYPT_DBF_ERR("%s wk info version %d or pkeytype %d mismatch.\n", + __func__, (int)wki->version, (int)wki->pkeytype); rc = -EIO; goto out; } @@ -1511,8 +1496,8 @@ int ep11_kblob2protkey(u16 card, u16 dom, *protkeytype = PKEY_KEYTYPE_AES_256; break; default: - DEBUG_ERR("%s unknown/unsupported AES pkeysize %d\n", - __func__, (int)wki->pkeysize); + ZCRYPT_DBF_ERR("%s unknown/unsupported AES pkeysize %d\n", + __func__, (int)wki->pkeysize); rc = -EIO; goto out; } @@ -1525,16 +1510,16 @@ int ep11_kblob2protkey(u16 card, u16 dom, break; case 2: /* TDES */ default: - DEBUG_ERR("%s unknown/unsupported key type %d\n", - __func__, (int)wki->pkeytype); + ZCRYPT_DBF_ERR("%s unknown/unsupported key type %d\n", + __func__, (int)wki->pkeytype); rc = -EIO; goto out; } /* copy the translated protected key */ if (wki->pkeysize > *protkeylen) { - DEBUG_ERR("%s wk info pkeysize %llu > protkeysize %u\n", - __func__, wki->pkeysize, *protkeylen); + ZCRYPT_DBF_ERR("%s wk info pkeysize %llu > protkeysize %u\n", + __func__, wki->pkeysize, *protkeylen); rc = -EINVAL; goto out; } From 08b2c3706de21d77cfe88017536f790a86bed397 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 10:07:28 +0100 Subject: [PATCH 046/117] s390/zcrypt: introduce dynamic debugging for AP and zcrypt code This patch replaces all the s390 debug feature calls with debug level by dynamic debug calls pr_debug. These calls are much more flexible and each single invocation can get enabled/disabled at runtime wheres the s390 debug feature debug calls have only one knob - enable or disable all in one bunch. The benefit is especially significant with high frequency called functions like the AP bus scan. In most debugging scenarios you don't want and need them, but sometimes it is crucial to know exactly when and how long the AP bus scan took. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 38 +++++++++++---------- drivers/s390/crypto/ap_debug.h | 2 -- drivers/s390/crypto/ap_queue.c | 27 ++++++++------- drivers/s390/crypto/zcrypt_api.c | 46 +++++++++++++------------- drivers/s390/crypto/zcrypt_debug.h | 2 -- drivers/s390/crypto/zcrypt_msgtype50.c | 14 ++++---- drivers/s390/crypto/zcrypt_msgtype6.c | 45 +++++++++++++------------ 7 files changed, 89 insertions(+), 85 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 732cd8200d0e..fdc0f1036544 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -826,8 +826,8 @@ static int __ap_revise_reserved(struct device *dev, void *dummy) drvres = to_ap_drv(dev->driver)->flags & AP_DRIVER_FLAG_DEFAULT; if (!!devres != !!drvres) { - AP_DBF_DBG("%s reprobing queue=%02x.%04x\n", - __func__, card, queue); + pr_debug("%s reprobing queue=%02x.%04x\n", + __func__, card, queue); rc = device_reprobe(dev); if (rc) AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", @@ -1030,7 +1030,7 @@ EXPORT_SYMBOL(ap_bus_force_rescan); */ void ap_bus_cfg_chg(void) { - AP_DBF_DBG("%s config change, forcing bus rescan\n", __func__); + pr_debug("%s config change, forcing bus rescan\n", __func__); ap_bus_force_rescan(); } @@ -1888,8 +1888,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED; } spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev checkstop on\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev checkstop on\n", + __func__, ac->id, dom); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); goto put_dev_and_continue; @@ -1899,8 +1899,8 @@ static inline void ap_scan_domains(struct ap_card *ac) if (aq->dev_state > AP_DEV_STATE_UNINITIATED) _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev checkstop off\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev checkstop off\n", + __func__, ac->id, dom); goto put_dev_and_continue; } /* config state change */ @@ -1912,8 +1912,8 @@ static inline void ap_scan_domains(struct ap_card *ac) aq->last_err_rc = AP_RESPONSE_DECONFIGURED; } spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev config off\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev config off\n", + __func__, ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); /* 'receive' pending messages with -EAGAIN */ ap_flush_queue(aq); @@ -1924,8 +1924,8 @@ static inline void ap_scan_domains(struct ap_card *ac) if (aq->dev_state > AP_DEV_STATE_UNINITIATED) _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); - AP_DBF_DBG("%s(%d,%d) queue dev config on\n", - __func__, ac->id, dom); + pr_debug("%s(%d,%d) queue dev config on\n", + __func__, ac->id, dom); ap_send_config_uevent(&aq->ap_dev, aq->config); goto put_dev_and_continue; } @@ -1997,8 +1997,8 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - AP_DBF_DBG("%s(%d) no type info (no APQN found), ignored\n", - __func__, ap); + pr_debug("%s(%d) no type info (no APQN found), ignored\n", + __func__, ap); } return; } @@ -2010,8 +2010,8 @@ static inline void ap_scan_adapter(int ap) ap_scan_rm_card_dev_and_queue_devs(ac); put_device(dev); } else { - AP_DBF_DBG("%s(%d) no valid type (0) info, ignored\n", - __func__, ap); + pr_debug("%s(%d) no valid type (0) info, ignored\n", + __func__, ap); } return; } @@ -2144,14 +2144,14 @@ static void ap_scan_bus(struct work_struct *unused) { int ap, config_changed = 0; + pr_debug(">%s\n", __func__); + /* config change notify */ config_changed = ap_get_configuration(); if (config_changed) notify_config_changed(); ap_select_domain(); - AP_DBF_DBG("%s running\n", __func__); - /* loop over all possible adapters */ for (ap = 0; ap <= ap_max_adapter_id; ap++) ap_scan_adapter(ap); @@ -2174,12 +2174,14 @@ static void ap_scan_bus(struct work_struct *unused) } if (atomic64_inc_return(&ap_scan_bus_count) == 1) { - AP_DBF_DBG("%s init scan complete\n", __func__); + pr_debug("%s init scan complete\n", __func__); ap_send_init_scan_done_uevent(); ap_check_bindings_complete(); } mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + + pr_debug("<%s\n", __func__); } static void ap_config_timeout(struct timer_list *unused) diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h index 9c53b64fb738..2f66271b8564 100644 --- a/drivers/s390/crypto/ap_debug.h +++ b/drivers/s390/crypto/ap_debug.h @@ -26,8 +26,6 @@ debug_sprintf_event(ap_dbf_info, DBF_WARN, ##__VA_ARGS__) #define AP_DBF_INFO(...) \ debug_sprintf_event(ap_dbf_info, DBF_INFO, ##__VA_ARGS__) -#define AP_DBF_DBG(...) \ - debug_sprintf_event(ap_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *ap_dbf_info; diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 682595443145..d8268fee3bd5 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -169,6 +169,9 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) aq->queue_count = 0; list_splice_init(&aq->pendingq, &aq->requestq); aq->requestq_count += aq->pendingq_count; + pr_debug("%s queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n", + __func__, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), + aq->pendingq_count, aq->requestq_count); aq->pendingq_count = 0; break; default: @@ -446,9 +449,9 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq) case AP_BS_Q_USABLE: /* association is through */ aq->sm_state = AP_SM_STATE_IDLE; - AP_DBF_DBG("%s queue 0x%02x.%04x associated with %u\n", - __func__, AP_QID_CARD(aq->qid), - AP_QID_QUEUE(aq->qid), aq->assoc_idx); + pr_debug("%s queue 0x%02x.%04x associated with %u\n", + __func__, AP_QID_CARD(aq->qid), + AP_QID_QUEUE(aq->qid), aq->assoc_idx); return AP_SM_WAIT_NONE; case AP_BS_Q_USABLE_NO_SECURE_KEY: /* association still pending */ @@ -690,9 +693,9 @@ static ssize_t ap_functions_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } @@ -846,9 +849,9 @@ static ssize_t se_bind_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } @@ -974,9 +977,9 @@ static ssize_t se_associate_show(struct device *dev, status = ap_test_queue(aq->qid, 1, &hwinfo); if (status.response_code > AP_RESPONSE_BUSY) { - AP_DBF_DBG("%s RC 0x%02x on tapq(0x%02x.%04x)\n", - __func__, status.response_code, - AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); + pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n", + __func__, status.response_code, + AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid)); return -EIO; } diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 55c163610475..00e7e05a289c 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -12,6 +12,9 @@ * Multiple device nodes: Harald Freudenberger */ +#define KMSG_COMPONENT "zcrypt" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + #include #include #include @@ -715,8 +718,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("%s no matching queue found => ENODEV\n", __func__); rc = -ENODEV; goto out; } @@ -820,8 +822,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("%s no matching queue found => ENODEV\n", __func__); rc = -ENODEV; goto out; } @@ -940,8 +941,8 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no match for address %02x.%04x => ENODEV\n", - __func__, xcrb->user_defined, *domain); + pr_debug("%s no match for address %02x.%04x => ENODEV\n", + __func__, xcrb->user_defined, *domain); rc = -ENODEV; goto out; } @@ -1113,15 +1114,15 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, if (!pref_zq) { if (targets && target_num == 1) { - ZCRYPT_DBF_DBG("%s no match for address %02x.%04x => ENODEV\n", - __func__, (int)targets->ap_id, - (int)targets->dom_id); + pr_debug("%s no match for address %02x.%04x => ENODEV\n", + __func__, (int)targets->ap_id, + (int)targets->dom_id); } else if (targets) { - ZCRYPT_DBF_DBG("%s no match for %d target addrs => ENODEV\n", - __func__, (int)target_num); + pr_debug("%s no match for %d target addrs => ENODEV\n", + __func__, (int)target_num); } else { - ZCRYPT_DBF_DBG("%s no match for address ff.ffff => ENODEV\n", - __func__); + pr_debug("%s no match for address ff.ffff => ENODEV\n", + __func__); } rc = -ENODEV; goto out_free; @@ -1199,8 +1200,7 @@ static long zcrypt_rng(char *buffer) spin_unlock(&zcrypt_list_lock); if (!pref_zq) { - ZCRYPT_DBF_DBG("%s no matching queue found => ENODEV\n", - __func__); + pr_debug("%s no matching queue found => ENODEV\n", __func__); rc = -ENODEV; goto out; } @@ -1444,7 +1444,7 @@ static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg) if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { - ZCRYPT_DBF_DBG("ioctl ICARSAMODEXPO rc=%d\n", rc); + pr_debug("ioctl ICARSAMODEXPO rc=%d\n", rc); return rc; } return put_user(mex.outputdatalength, &umex->outputdatalength); @@ -1476,7 +1476,7 @@ static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg) if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { - ZCRYPT_DBF_DBG("ioctl ICARSACRT rc=%d\n", rc); + pr_debug("ioctl ICARSACRT rc=%d\n", rc); return rc; } return put_user(crt.outputdatalength, &ucrt->outputdatalength); @@ -1508,8 +1508,8 @@ static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg) if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - ZCRYPT_DBF_DBG("ioctl ZSENDCPRB rc=%d status=0x%x\n", - rc, xcrb.status); + pr_debug("ioctl ZSENDCPRB rc=%d status=0x%x\n", + rc, xcrb.status); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) return -EFAULT; return rc; @@ -1541,7 +1541,7 @@ static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg) if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) - ZCRYPT_DBF_DBG("ioctl ZSENDEP11CPRB rc=%d\n", rc); + pr_debug("ioctl ZSENDEP11CPRB rc=%d\n", rc); if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb))) return -EFAULT; return rc; @@ -1670,7 +1670,7 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd, } /* unknown ioctl number */ default: - ZCRYPT_DBF_DBG("unknown ioctl 0x%08x\n", cmd); + pr_debug("unknown ioctl 0x%08x\n", cmd); return -ENOIOCTLCMD; } } @@ -2014,8 +2014,8 @@ int zcrypt_wait_api_operational(void) break; default: /* other failure */ - ZCRYPT_DBF_DBG("%s ap_wait_init_apqn_bindings_complete()=%d\n", - __func__, rc); + pr_debug("%s ap_wait_init_apqn_bindings_complete()=%d\n", + __func__, rc); break; } break; diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h index 8e947a23a359..9a208dc4c200 100644 --- a/drivers/s390/crypto/zcrypt_debug.h +++ b/drivers/s390/crypto/zcrypt_debug.h @@ -27,8 +27,6 @@ debug_sprintf_event(zcrypt_dbf_info, DBF_WARN, ##__VA_ARGS__) #define ZCRYPT_DBF_INFO(...) \ debug_sprintf_event(zcrypt_dbf_info, DBF_INFO, ##__VA_ARGS__) -#define ZCRYPT_DBF_DBG(...) \ - debug_sprintf_event(zcrypt_dbf_info, DBF_DEBUG, ##__VA_ARGS__) extern debug_info_t *zcrypt_dbf_info; diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index 2e155de8abe5..3b39cb8f926d 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -427,7 +427,7 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, len = t80h->len; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", __func__); msg->rc = -EMSGSIZE; goto out; } @@ -487,9 +487,9 @@ static long zcrypt_msgtype50_modexpo(struct zcrypt_queue *zq, out: ap_msg->private = NULL; if (rc) - ZCRYPT_DBF_DBG("%s send me cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send me cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -537,9 +537,9 @@ static long zcrypt_msgtype50_modexpo_crt(struct zcrypt_queue *zq, out: ap_msg->private = NULL; if (rc) - ZCRYPT_DBF_DBG("%s send crt cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send crt cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 3c53abbdc342..215f257d2360 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -437,9 +437,9 @@ static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg, ap_msg->flags |= AP_MSG_FLAG_ADMIN; break; default: - ZCRYPT_DBF_DBG("%s unknown CPRB minor version '%c%c'\n", - __func__, msg->cprbx.func_id[0], - msg->cprbx.func_id[1]); + pr_debug("%s unknown CPRB minor version '%c%c'\n", + __func__, msg->cprbx.func_id[0], + msg->cprbx.func_id[1]); } /* copy data block */ @@ -629,9 +629,9 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy CPRB to user */ if (xcrb->reply_control_blk_length < msg->fmt2.count1) { - ZCRYPT_DBF_DBG("%s reply_control_blk_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_control_blk_length, - msg->fmt2.count1); + pr_debug("%s reply_control_blk_length %u < required %u => EMSGSIZE\n", + __func__, xcrb->reply_control_blk_length, + msg->fmt2.count1); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_control_blk_addr, @@ -642,9 +642,9 @@ static int convert_type86_xcrb(bool userspace, struct zcrypt_queue *zq, /* Copy data buffer to user */ if (msg->fmt2.count2) { if (xcrb->reply_data_length < msg->fmt2.count2) { - ZCRYPT_DBF_DBG("%s reply_data_length %u < required %u => EMSGSIZE\n", - __func__, xcrb->reply_data_length, - msg->fmt2.count2); + pr_debug("%s reply_data_length %u < required %u => EMSGSIZE\n", + __func__, xcrb->reply_data_length, + msg->fmt2.count2); return -EMSGSIZE; } if (z_copy_to_user(userspace, xcrb->reply_data_addr, @@ -673,9 +673,9 @@ static int convert_type86_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, char *data = reply->msg; if (xcrb->resp_len < msg->fmt2.count1) { - ZCRYPT_DBF_DBG("%s resp_len %u < required %u => EMSGSIZE\n", - __func__, (unsigned int)xcrb->resp_len, - msg->fmt2.count1); + pr_debug("%s resp_len %u < required %u => EMSGSIZE\n", + __func__, (unsigned int)xcrb->resp_len, + msg->fmt2.count1); return -EMSGSIZE; } @@ -875,7 +875,8 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = sizeof(struct type86x_reply) + t86r->length; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", + __func__); msg->rc = -EMSGSIZE; goto out; } @@ -889,7 +890,8 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", + __func__); msg->rc = -EMSGSIZE; goto out; } @@ -939,7 +941,8 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, len = t86r->fmt2.offset1 + t86r->fmt2.count1; if (len > reply->bufsize || len > msg->bufsize || len != reply->len) { - ZCRYPT_DBF_DBG("%s len mismatch => EMSGSIZE\n", __func__); + pr_debug("%s len mismatch => EMSGSIZE\n", + __func__); msg->rc = -EMSGSIZE; goto out; } @@ -1151,9 +1154,9 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq, out: if (rc) - ZCRYPT_DBF_DBG("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } @@ -1274,9 +1277,9 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue * out: if (rc) - ZCRYPT_DBF_DBG("%s send cprb at dev=%02x.%04x rc=%d\n", - __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), rc); + pr_debug("%s send cprb at dev=%02x.%04x rc=%d\n", + __func__, AP_QID_CARD(zq->queue->qid), + AP_QID_QUEUE(zq->queue->qid), rc); return rc; } From 0ccac45295403a7730d7bdd8b047a824b7a23a2e Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 29 Jan 2024 16:56:42 +0100 Subject: [PATCH 047/117] s390/pkey: harmonize pkey s390 debug feature calls Cleanup and harmonize the s390 debug feature calls and defines for the pkey module to be similar to the debug feature as it is used in the zcrypt device driver and AP bus. More or less only renaming but no functional changes. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/pkey_api.c | 188 +++++++++++++++++---------------- 1 file changed, 97 insertions(+), 91 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 6cfb6b2340c9..050cecac7191 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -42,24 +42,24 @@ MODULE_DESCRIPTION("s390 protected key interface"); * debug feature data and functions */ -static debug_info_t *debug_info; +static debug_info_t *pkey_dbf_info; -#define DEBUG_DBG(...) debug_sprintf_event(debug_info, 6, ##__VA_ARGS__) -#define DEBUG_INFO(...) debug_sprintf_event(debug_info, 5, ##__VA_ARGS__) -#define DEBUG_WARN(...) debug_sprintf_event(debug_info, 4, ##__VA_ARGS__) -#define DEBUG_ERR(...) debug_sprintf_event(debug_info, 3, ##__VA_ARGS__) +#define PKEY_DBF_DBG(...) debug_sprintf_event(pkey_dbf_info, 6, ##__VA_ARGS__) +#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__) +#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__) +#define PKEY_DBF_ERR(...) debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__) static void __init pkey_debug_init(void) { /* 5 arguments per dbf entry (including the format string ptr) */ - debug_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); - debug_register_view(debug_info, &debug_sprintf_view); - debug_set_level(debug_info, 3); + pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long)); + debug_register_view(pkey_dbf_info, &debug_sprintf_view); + debug_set_level(pkey_dbf_info, 3); } static void __exit pkey_debug_exit(void) { - debug_unregister(debug_info); + debug_unregister(pkey_dbf_info); } /* inside view of a protected key token (only type 0x00 version 0x01) */ @@ -163,14 +163,14 @@ static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, fc = CPACF_PCKMO_ENC_ECC_ED448_KEY; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, keytype); return -EINVAL; } if (*protkeylen < keysize + AES_WK_VP_SIZE) { - DEBUG_ERR("%s prot key buffer size too small: %u < %d\n", - __func__, *protkeylen, keysize + AES_WK_VP_SIZE); + PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n", + __func__, *protkeylen, keysize + AES_WK_VP_SIZE); return -EINVAL; } @@ -182,7 +182,7 @@ static int pkey_clr2protkey(u32 keytype, const u8 *clrkey, } /* check for the pckmo subfunction we need now */ if (!cpacf_test_func(&pckmo_functions, fc)) { - DEBUG_ERR("%s pckmo functions not available\n", __func__); + PKEY_DBF_ERR("%s pckmo functions not available\n", __func__); return -ENODEV; } @@ -244,7 +244,7 @@ static int pkey_skey2pkey(const u8 *key, u8 *protkey, } if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + PKEY_DBF_DBG("%s failed rc=%d\n", __func__, rc); return rc; } @@ -283,7 +283,7 @@ static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, out: kfree(apqns); if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + PKEY_DBF_DBG("%s failed rc=%d\n", __func__, rc); return rc; } @@ -320,7 +320,7 @@ static int pkey_ep11key2pkey(const u8 *key, size_t keylen, out: kfree(apqns); if (rc) - DEBUG_DBG("%s failed rc=%d\n", __func__, rc); + PKEY_DBF_DBG("%s failed rc=%d\n", __func__, rc); return rc; } @@ -336,7 +336,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, int rc; /* check the secure key for valid AES secure key */ - rc = cca_check_secaeskeytoken(debug_info, 3, (u8 *)seckey, 0); + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, (u8 *)seckey, 0); if (rc) goto out; if (pattributes) @@ -351,7 +351,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, if (rc > 0) { /* key mkvp matches to old master key mkvp */ - DEBUG_DBG("%s secure key has old mkvp\n", __func__); + PKEY_DBF_DBG("%s secure key has old mkvp\n", __func__); if (pattributes) *pattributes |= PKEY_VERIFY_ATTR_OLD_MKVP; rc = 0; @@ -363,7 +363,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, *pdomain = domain; out: - DEBUG_DBG("%s rc=%d\n", __func__, rc); + PKEY_DBF_DBG("%s rc=%d\n", __func__, rc); return rc; } @@ -379,8 +379,8 @@ static int pkey_genprotkey(u32 keytype, u8 *protkey, keysize = pkey_keytype_aes_to_size(keytype); if (!keysize) { - DEBUG_ERR("%s unknown/unsupported keytype %d\n", __func__, - keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__, + keytype); return -EINVAL; } @@ -428,13 +428,13 @@ static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, fc = CPACF_KMC_PAES_256; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", __func__, - protkeytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", __func__, + protkeytype); return -EINVAL; } if (protkeylen != pkeylen) { - DEBUG_ERR("%s invalid protected key size %u for keytype %u\n", - __func__, protkeylen, protkeytype); + PKEY_DBF_ERR("%s invalid protected key size %u for keytype %u\n", + __func__, protkeylen, protkeytype); return -EINVAL; } @@ -446,7 +446,7 @@ static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen, k = cpacf_kmc(fc | CPACF_ENCRYPT, ¶m, null_msg, dest_buf, sizeof(null_msg)); if (k != sizeof(null_msg)) { - DEBUG_ERR("%s protected key is not valid\n", __func__); + PKEY_DBF_ERR("%s protected key is not valid\n", __func__); return -EKEYREJECTED; } @@ -464,13 +464,13 @@ static int nonccatokaes2pkey(const struct clearkeytoken *t, keysize = pkey_keytype_aes_to_size(t->keytype); if (!keysize) { - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, t->keytype); return -EINVAL; } if (t->len != keysize) { - DEBUG_ERR("%s non clear key aes token: invalid key len %u\n", - __func__, t->len); + PKEY_DBF_ERR("%s non clear key aes token: invalid key len %u\n", + __func__, t->len); return -EINVAL; } @@ -505,7 +505,7 @@ static int nonccatokaes2pkey(const struct clearkeytoken *t, goto out; failure: - DEBUG_ERR("%s unable to build protected key from clear", __func__); + PKEY_DBF_ERR("%s unable to build protected key from clear", __func__); out: kfree(tmpbuf); @@ -536,14 +536,14 @@ static int nonccatokecc2pkey(const struct clearkeytoken *t, keylen = 64; break; default: - DEBUG_ERR("%s unknown/unsupported keytype %u\n", - __func__, t->keytype); + PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n", + __func__, t->keytype); return -EINVAL; } if (t->len != keylen) { - DEBUG_ERR("%s non clear key ecc token: invalid key len %u\n", - __func__, t->len); + PKEY_DBF_ERR("%s non clear key ecc token: invalid key len %u\n", + __func__, t->len); return -EINVAL; } @@ -551,8 +551,8 @@ static int nonccatokecc2pkey(const struct clearkeytoken *t, rc = pkey_clr2protkey(t->keytype, t->clearkey, protkey, protkeylen, protkeytype); if (rc) { - DEBUG_ERR("%s unable to build protected key from clear", - __func__); + PKEY_DBF_ERR("%s unable to build protected key from clear", + __func__); } return rc; @@ -604,15 +604,15 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, protkeylen, protkeytype); break; default: - DEBUG_ERR("%s unknown/unsupported non cca clear key type %u\n", - __func__, t->keytype); + PKEY_DBF_ERR("%s unknown/unsupported non cca clear key type %u\n", + __func__, t->keytype); return -EINVAL; } break; } case TOKVER_EP11_AES: { /* check ep11 key for exportable as protected key */ - rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); if (rc) goto out; rc = pkey_ep11key2pkey(key, keylen, @@ -621,15 +621,16 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, } case TOKVER_EP11_AES_WITH_HEADER: /* check ep11 key with header for exportable as protected key */ - rc = ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1); + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); if (rc) goto out; rc = pkey_ep11key2pkey(key, keylen, protkey, protkeylen, protkeytype); break; default: - DEBUG_ERR("%s unknown/unsupported non-CCA token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown/unsupported non-CCA token version %d\n", + __func__, hdr->version); } out: @@ -654,8 +655,8 @@ static int pkey_ccainttok2pkey(const u8 *key, u32 keylen, return -EINVAL; break; default: - DEBUG_ERR("%s unknown/unsupported CCA internal token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown/unsupported CCA internal token version %d\n", + __func__, hdr->version); return -EINVAL; } @@ -672,7 +673,7 @@ int pkey_keyblob2pkey(const u8 *key, u32 keylen, int rc; if (keylen < sizeof(struct keytoken_header)) { - DEBUG_ERR("%s invalid keylen %d\n", __func__, keylen); + PKEY_DBF_ERR("%s invalid keylen %d\n", __func__, keylen); return -EINVAL; } @@ -686,12 +687,12 @@ int pkey_keyblob2pkey(const u8 *key, u32 keylen, protkey, protkeylen, protkeytype); break; default: - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); + PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); return -EINVAL; } - DEBUG_DBG("%s rc=%d\n", __func__, rc); + PKEY_DBF_DBG("%s rc=%d\n", __func__, rc); return rc; } EXPORT_SYMBOL(pkey_keyblob2pkey); @@ -839,7 +840,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, hdr->version == TOKVER_CCA_AES) { struct secaeskeytoken *t = (struct secaeskeytoken *)key; - rc = cca_check_secaeskeytoken(debug_info, 3, key, 0); + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); if (rc) goto out; if (ktype) @@ -869,7 +870,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, hdr->version == TOKVER_CCA_VLSC) { struct cipherkeytoken *t = (struct cipherkeytoken *)key; - rc = cca_check_secaescipherkey(debug_info, 3, key, 0, 1); + rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); if (rc) goto out; if (ktype) @@ -907,7 +908,7 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, struct ep11keyblob *kb = (struct ep11keyblob *)key; int api; - rc = ep11_check_aes_key(debug_info, 3, key, keylen, 1); + rc = ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1); if (rc) goto out; if (ktype) @@ -933,8 +934,8 @@ static int pkey_verifykey2(const u8 *key, size_t keylen, struct ep11kblob_header *kh = (struct ep11kblob_header *)key; int api; - rc = ep11_check_aes_key_with_hdr(debug_info, 3, - key, keylen, 1); + rc = ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1); if (rc) goto out; if (ktype) @@ -981,25 +982,27 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, if (hdr->version == TOKVER_CCA_AES) { if (keylen != sizeof(struct secaeskeytoken)) return -EINVAL; - if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) return -EINVAL; } else if (hdr->version == TOKVER_CCA_VLSC) { if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) return -EINVAL; - if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) return -EINVAL; } else { - DEBUG_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown CCA internal token version %d\n", + __func__, hdr->version); return -EINVAL; } } else if (hdr->type == TOKTYPE_NON_CCA) { if (hdr->version == TOKVER_EP11_AES) { - if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) + if (ep11_check_aes_key(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else if (hdr->version == TOKVER_EP11_AES_WITH_HEADER) { - if (ep11_check_aes_key_with_hdr(debug_info, 3, - key, keylen, 1)) + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else { return pkey_nonccatok2pkey(key, keylen, @@ -1007,8 +1010,8 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, protkeytype); } } else { - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); + PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); return -EINVAL; } @@ -1234,50 +1237,53 @@ static int pkey_keyblob2pkey3(const struct pkey_apqn *apqns, size_t nr_apqns, hdr->version == TOKVER_EP11_AES_WITH_HEADER && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { /* EP11 AES key blob with header */ - if (ep11_check_aes_key_with_hdr(debug_info, 3, key, keylen, 1)) + if (ep11_check_aes_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_ECC_WITH_HEADER && is_ep11_keyblob(key + sizeof(struct ep11kblob_header))) { /* EP11 ECC key blob with header */ - if (ep11_check_ecc_key_with_hdr(debug_info, 3, key, keylen, 1)) + if (ep11_check_ecc_key_with_hdr(pkey_dbf_info, + 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA && hdr->version == TOKVER_EP11_AES && is_ep11_keyblob(key)) { /* EP11 AES key blob with header in session field */ - if (ep11_check_aes_key(debug_info, 3, key, keylen, 1)) + if (ep11_check_aes_key(pkey_dbf_info, 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_CCA_INTERNAL) { if (hdr->version == TOKVER_CCA_AES) { /* CCA AES data key */ if (keylen != sizeof(struct secaeskeytoken)) return -EINVAL; - if (cca_check_secaeskeytoken(debug_info, 3, key, 0)) + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) return -EINVAL; } else if (hdr->version == TOKVER_CCA_VLSC) { /* CCA AES cipher key */ if (keylen < hdr->len || keylen > MAXCCAVLSCTOKENSIZE) return -EINVAL; - if (cca_check_secaescipherkey(debug_info, 3, key, 0, 1)) + if (cca_check_secaescipherkey(pkey_dbf_info, + 3, key, 0, 1)) return -EINVAL; } else { - DEBUG_ERR("%s unknown CCA internal token version %d\n", - __func__, hdr->version); + PKEY_DBF_ERR("%s unknown CCA internal token version %d\n", + __func__, hdr->version); return -EINVAL; } } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { /* CCA ECC (private) key */ if (keylen < sizeof(struct eccprivkeytoken)) return -EINVAL; - if (cca_check_sececckeytoken(debug_info, 3, key, keylen, 1)) + if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_NON_CCA) { return pkey_nonccatok2pkey(key, keylen, protkey, protkeylen, protkeytype); } else { - DEBUG_ERR("%s unknown/unsupported blob type %d\n", - __func__, hdr->type); + PKEY_DBF_ERR("%s unknown/unsupported blob type %d\n", + __func__, hdr->type); return -EINVAL; } @@ -1350,7 +1356,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_genseckey(kgs.cardnr, kgs.domain, kgs.keytype, kgs.seckey.seckey); - DEBUG_DBG("%s cca_genseckey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s cca_genseckey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ugs, &kgs, sizeof(kgs))) @@ -1365,7 +1371,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, kcs.clrkey.clrkey, kcs.seckey.seckey); - DEBUG_DBG("%s cca_clr2seckey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s cca_clr2seckey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ucs, &kcs, sizeof(kcs))) @@ -1383,7 +1389,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = cca_sec2protkey(ksp.cardnr, ksp.domain, ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); - DEBUG_DBG("%s cca_sec2protkey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s cca_sec2protkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(usp, &ksp, sizeof(ksp))) @@ -1400,7 +1406,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, kcp.protkey.protkey, &kcp.protkey.len, &kcp.protkey.type); - DEBUG_DBG("%s pkey_clr2protkey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_clr2protkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ucp, &kcp, sizeof(kcp))) @@ -1416,7 +1422,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_findcard(kfc.seckey.seckey, &kfc.cardnr, &kfc.domain, 1); - DEBUG_DBG("%s cca_findcard()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s cca_findcard()=%d\n", __func__, rc); if (rc < 0) break; if (copy_to_user(ufc, &kfc, sizeof(kfc))) @@ -1432,7 +1438,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, ksp.protkey.len = sizeof(ksp.protkey.protkey); rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); - DEBUG_DBG("%s pkey_skey2pkey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_skey2pkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(usp, &ksp, sizeof(ksp))) @@ -1447,7 +1453,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, &kvk.keysize, &kvk.attributes); - DEBUG_DBG("%s pkey_verifykey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_verifykey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(uvk, &kvk, sizeof(kvk))) @@ -1463,7 +1469,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kgp.protkey.len = sizeof(kgp.protkey.protkey); rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, &kgp.protkey.len, &kgp.protkey.type); - DEBUG_DBG("%s pkey_genprotkey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_genprotkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ugp, &kgp, sizeof(kgp))) @@ -1478,7 +1484,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = pkey_verifyprotkey(kvp.protkey.protkey, kvp.protkey.len, kvp.protkey.type); - DEBUG_DBG("%s pkey_verifyprotkey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_verifyprotkey()=%d\n", __func__, rc); break; } case PKEY_KBLOB2PROTK: { @@ -1494,7 +1500,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, ktp.protkey.len = sizeof(ktp.protkey.protkey); rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, &ktp.protkey.len, &ktp.protkey.type); - DEBUG_DBG("%s pkey_keyblob2pkey()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_keyblob2pkey()=%d\n", __func__, rc); memzero_explicit(kkey, ktp.keylen); kfree(kkey); if (rc) @@ -1523,7 +1529,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_genseckey2(apqns, kgs.apqn_entries, kgs.type, kgs.size, kgs.keygenflags, kkey, &klen); - DEBUG_DBG("%s pkey_genseckey2()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_genseckey2()=%d\n", __func__, rc); kfree(apqns); if (rc) { kfree(kkey); @@ -1565,7 +1571,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, kcs.type, kcs.size, kcs.keygenflags, kcs.clrkey.clrkey, kkey, &klen); - DEBUG_DBG("%s pkey_clr2seckey2()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_clr2seckey2()=%d\n", __func__, rc); kfree(apqns); if (rc) { kfree(kkey); @@ -1601,7 +1607,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_verifykey2(kkey, kvk.keylen, &kvk.cardnr, &kvk.domain, &kvk.type, &kvk.size, &kvk.flags); - DEBUG_DBG("%s pkey_verifykey2()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_verifykey2()=%d\n", __func__, rc); kfree(kkey); if (rc) break; @@ -1630,7 +1636,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kkey, ktp.keylen, ktp.protkey.protkey, &ktp.protkey.len, &ktp.protkey.type); - DEBUG_DBG("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); kfree(kkey); @@ -1664,7 +1670,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, apqns, &nr_apqns); - DEBUG_DBG("%s pkey_apqns4key()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_apqns4key()=%d\n", __func__, rc); kfree(kkey); if (rc && rc != -ENOSPC) { kfree(apqns); @@ -1707,7 +1713,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, kat.flags, apqns, &nr_apqns); - DEBUG_DBG("%s pkey_apqns4keytype()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_apqns4keytype()=%d\n", __func__, rc); if (rc && rc != -ENOSPC) { kfree(apqns); break; @@ -1757,7 +1763,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, kkey, ktp.keylen, protkey, &protkeylen, &ktp.pkeytype); - DEBUG_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); + PKEY_DBF_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); kfree(kkey); From 6d749b4e02087fa4a68092eef260d31a345603c6 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 10:14:47 +0100 Subject: [PATCH 048/117] s390/pkey: introduce dynamic debugging for pkey This patch replaces all the s390 debug feature calls with debug level by dynamic debug calls pr_debug. These calls are much more flexible and each single invocation can get enabled/disabled at runtime wheres the s390 debug feature debug calls have only one knob - enable or disable all in one bunch. This patch follows a similar change for the AP bus and zcrypt device driver code. All this code uses dynamic debugging with pr_debug and friends for emitting debug traces now. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/pkey_api.c | 47 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 050cecac7191..4c60b5acbae1 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -44,7 +44,6 @@ MODULE_DESCRIPTION("s390 protected key interface"); static debug_info_t *pkey_dbf_info; -#define PKEY_DBF_DBG(...) debug_sprintf_event(pkey_dbf_info, 6, ##__VA_ARGS__) #define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__) #define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__) #define PKEY_DBF_ERR(...) debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__) @@ -244,7 +243,7 @@ static int pkey_skey2pkey(const u8 *key, u8 *protkey, } if (rc) - PKEY_DBF_DBG("%s failed rc=%d\n", __func__, rc); + pr_debug("%s failed rc=%d\n", __func__, rc); return rc; } @@ -283,7 +282,7 @@ static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, out: kfree(apqns); if (rc) - PKEY_DBF_DBG("%s failed rc=%d\n", __func__, rc); + pr_debug("%s failed rc=%d\n", __func__, rc); return rc; } @@ -320,7 +319,7 @@ static int pkey_ep11key2pkey(const u8 *key, size_t keylen, out: kfree(apqns); if (rc) - PKEY_DBF_DBG("%s failed rc=%d\n", __func__, rc); + pr_debug("%s failed rc=%d\n", __func__, rc); return rc; } @@ -351,7 +350,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, if (rc > 0) { /* key mkvp matches to old master key mkvp */ - PKEY_DBF_DBG("%s secure key has old mkvp\n", __func__); + pr_debug("%s secure key has old mkvp\n", __func__); if (pattributes) *pattributes |= PKEY_VERIFY_ATTR_OLD_MKVP; rc = 0; @@ -363,7 +362,7 @@ static int pkey_verifykey(const struct pkey_seckey *seckey, *pdomain = domain; out: - PKEY_DBF_DBG("%s rc=%d\n", __func__, rc); + pr_debug("%s rc=%d\n", __func__, rc); return rc; } @@ -692,7 +691,7 @@ int pkey_keyblob2pkey(const u8 *key, u32 keylen, return -EINVAL; } - PKEY_DBF_DBG("%s rc=%d\n", __func__, rc); + pr_debug("%s rc=%d\n", __func__, rc); return rc; } EXPORT_SYMBOL(pkey_keyblob2pkey); @@ -1356,7 +1355,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_genseckey(kgs.cardnr, kgs.domain, kgs.keytype, kgs.seckey.seckey); - PKEY_DBF_DBG("%s cca_genseckey()=%d\n", __func__, rc); + pr_debug("%s cca_genseckey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ugs, &kgs, sizeof(kgs))) @@ -1371,7 +1370,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_clr2seckey(kcs.cardnr, kcs.domain, kcs.keytype, kcs.clrkey.clrkey, kcs.seckey.seckey); - PKEY_DBF_DBG("%s cca_clr2seckey()=%d\n", __func__, rc); + pr_debug("%s cca_clr2seckey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ucs, &kcs, sizeof(kcs))) @@ -1389,7 +1388,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = cca_sec2protkey(ksp.cardnr, ksp.domain, ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); - PKEY_DBF_DBG("%s cca_sec2protkey()=%d\n", __func__, rc); + pr_debug("%s cca_sec2protkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(usp, &ksp, sizeof(ksp))) @@ -1406,7 +1405,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_clr2protkey(kcp.keytype, kcp.clrkey.clrkey, kcp.protkey.protkey, &kcp.protkey.len, &kcp.protkey.type); - PKEY_DBF_DBG("%s pkey_clr2protkey()=%d\n", __func__, rc); + pr_debug("%s pkey_clr2protkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ucp, &kcp, sizeof(kcp))) @@ -1422,7 +1421,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = cca_findcard(kfc.seckey.seckey, &kfc.cardnr, &kfc.domain, 1); - PKEY_DBF_DBG("%s cca_findcard()=%d\n", __func__, rc); + pr_debug("%s cca_findcard()=%d\n", __func__, rc); if (rc < 0) break; if (copy_to_user(ufc, &kfc, sizeof(kfc))) @@ -1438,7 +1437,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, ksp.protkey.len = sizeof(ksp.protkey.protkey); rc = pkey_skey2pkey(ksp.seckey.seckey, ksp.protkey.protkey, &ksp.protkey.len, &ksp.protkey.type); - PKEY_DBF_DBG("%s pkey_skey2pkey()=%d\n", __func__, rc); + pr_debug("%s pkey_skey2pkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(usp, &ksp, sizeof(ksp))) @@ -1453,7 +1452,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = pkey_verifykey(&kvk.seckey, &kvk.cardnr, &kvk.domain, &kvk.keysize, &kvk.attributes); - PKEY_DBF_DBG("%s pkey_verifykey()=%d\n", __func__, rc); + pr_debug("%s pkey_verifykey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(uvk, &kvk, sizeof(kvk))) @@ -1469,7 +1468,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kgp.protkey.len = sizeof(kgp.protkey.protkey); rc = pkey_genprotkey(kgp.keytype, kgp.protkey.protkey, &kgp.protkey.len, &kgp.protkey.type); - PKEY_DBF_DBG("%s pkey_genprotkey()=%d\n", __func__, rc); + pr_debug("%s pkey_genprotkey()=%d\n", __func__, rc); if (rc) break; if (copy_to_user(ugp, &kgp, sizeof(kgp))) @@ -1484,7 +1483,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, return -EFAULT; rc = pkey_verifyprotkey(kvp.protkey.protkey, kvp.protkey.len, kvp.protkey.type); - PKEY_DBF_DBG("%s pkey_verifyprotkey()=%d\n", __func__, rc); + pr_debug("%s pkey_verifyprotkey()=%d\n", __func__, rc); break; } case PKEY_KBLOB2PROTK: { @@ -1500,7 +1499,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, ktp.protkey.len = sizeof(ktp.protkey.protkey); rc = pkey_keyblob2pkey(kkey, ktp.keylen, ktp.protkey.protkey, &ktp.protkey.len, &ktp.protkey.type); - PKEY_DBF_DBG("%s pkey_keyblob2pkey()=%d\n", __func__, rc); + pr_debug("%s pkey_keyblob2pkey()=%d\n", __func__, rc); memzero_explicit(kkey, ktp.keylen); kfree(kkey); if (rc) @@ -1529,7 +1528,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_genseckey2(apqns, kgs.apqn_entries, kgs.type, kgs.size, kgs.keygenflags, kkey, &klen); - PKEY_DBF_DBG("%s pkey_genseckey2()=%d\n", __func__, rc); + pr_debug("%s pkey_genseckey2()=%d\n", __func__, rc); kfree(apqns); if (rc) { kfree(kkey); @@ -1571,7 +1570,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_clr2seckey2(apqns, kcs.apqn_entries, kcs.type, kcs.size, kcs.keygenflags, kcs.clrkey.clrkey, kkey, &klen); - PKEY_DBF_DBG("%s pkey_clr2seckey2()=%d\n", __func__, rc); + pr_debug("%s pkey_clr2seckey2()=%d\n", __func__, rc); kfree(apqns); if (rc) { kfree(kkey); @@ -1607,7 +1606,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_verifykey2(kkey, kvk.keylen, &kvk.cardnr, &kvk.domain, &kvk.type, &kvk.size, &kvk.flags); - PKEY_DBF_DBG("%s pkey_verifykey2()=%d\n", __func__, rc); + pr_debug("%s pkey_verifykey2()=%d\n", __func__, rc); kfree(kkey); if (rc) break; @@ -1636,7 +1635,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, kkey, ktp.keylen, ktp.protkey.protkey, &ktp.protkey.len, &ktp.protkey.type); - PKEY_DBF_DBG("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); + pr_debug("%s pkey_keyblob2pkey2()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); kfree(kkey); @@ -1670,7 +1669,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } rc = pkey_apqns4key(kkey, kak.keylen, kak.flags, apqns, &nr_apqns); - PKEY_DBF_DBG("%s pkey_apqns4key()=%d\n", __func__, rc); + pr_debug("%s pkey_apqns4key()=%d\n", __func__, rc); kfree(kkey); if (rc && rc != -ENOSPC) { kfree(apqns); @@ -1713,7 +1712,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, } rc = pkey_apqns4keytype(kat.type, kat.cur_mkvp, kat.alt_mkvp, kat.flags, apqns, &nr_apqns); - PKEY_DBF_DBG("%s pkey_apqns4keytype()=%d\n", __func__, rc); + pr_debug("%s pkey_apqns4keytype()=%d\n", __func__, rc); if (rc && rc != -ENOSPC) { kfree(apqns); break; @@ -1763,7 +1762,7 @@ static long pkey_unlocked_ioctl(struct file *filp, unsigned int cmd, rc = pkey_keyblob2pkey3(apqns, ktp.apqn_entries, kkey, ktp.keylen, protkey, &protkeylen, &ktp.pkeytype); - PKEY_DBF_DBG("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); + pr_debug("%s pkey_keyblob2pkey3()=%d\n", __func__, rc); kfree(apqns); memzero_explicit(kkey, ktp.keylen); kfree(kkey); From 6a2892d09df545c6ea828bc8ad08112961b88f6d Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Fri, 9 Feb 2024 16:14:23 +0100 Subject: [PATCH 049/117] s390/ap: add debug possibility for AP messages This patch introduces two dynamic debug hexdump invocation possibilities to be able to a) dump an AP message immediately before it goes into the firmware queue and b) dump a fresh from the firmware queue received AP message. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_queue.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index d8268fee3bd5..6e4e8d324a6d 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -136,6 +136,8 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) switch (status.response_code) { case AP_RESPONSE_NORMAL: + print_hex_dump_debug("aprpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + aq->reply->msg, aq->reply->len, false); aq->queue_count = max_t(int, 0, aq->queue_count - 1); if (!status.queue_empty && !aq->queue_count) aq->queue_count++; @@ -246,6 +248,8 @@ static enum ap_sm_wait ap_sm_write(struct ap_queue *aq) /* Start the next request on the queue. */ ap_msg = list_entry(aq->requestq.next, struct ap_message, list); + print_hex_dump_debug("apreq: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg->msg, ap_msg->len, false); status = __ap_send(qid, ap_msg->psmid, ap_msg->msg, ap_msg->len, ap_msg->flags & AP_MSG_FLAG_SPECIAL); From b69b65f51148531ff3dd942a038614f77d9d60e3 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Fri, 9 Feb 2024 16:20:21 +0100 Subject: [PATCH 050/117] s390/zcrypt: add debug possibility for CCA and EP11 messages This patch introduces dynamic debug hexdump invocation possibilities to be able to: - dump an CCA or EP11 CPRB request as early as possible when received via ioctl from userspace but after the ap message has been collected together. - dump an CCA or EP11 CPRB reply short before it is transferred via ioctl into userspace. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/zcrypt_api.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 00e7e05a289c..e8742757085b 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -866,6 +866,8 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, rc = prep_cca_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain); if (rc) goto out; + print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); tdom = *domain; if (perms != &ap_perms && tdom < AP_DOMAINS) { @@ -953,6 +955,10 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, *domain = AP_QID_QUEUE(qid); rc = pref_zq->ops->send_cprb(userspace, pref_zq, xcrb, &ap_msg); + if (!rc) { + print_hex_dump_debug("ccarpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); + } spin_lock(&zcrypt_list_lock); zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); @@ -1046,6 +1052,8 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, rc = prep_ep11_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain); if (rc) goto out_free; + print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); if (perms != &ap_perms && domain < AUTOSEL_DOM) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { @@ -1130,6 +1138,10 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, qid = pref_zq->queue->qid; rc = pref_zq->ops->send_ep11_cprb(userspace, pref_zq, xcrb, &ap_msg); + if (!rc) { + print_hex_dump_debug("ep11rpl: ", DUMP_PREFIX_ADDRESS, 16, 1, + ap_msg.msg, ap_msg.len, false); + } spin_lock(&zcrypt_list_lock); zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt); From 39ceca158802f0891805f4e4e6724716294d36f2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:44:57 +0100 Subject: [PATCH 051/117] s390/fpu: fix VLGV macro The VLGV macro generates the VLGV instruction and has a vr parameter which correlates to the V3 vector register field of the instruction (bits 12-15). Due to its position in the VRS-c instruction format of the VLGV instruction, this field correlates to the second bit of the RXB byte of the instruction (see Principles of Operation, Chapter "Vector Overview and Support Instructions"). Within the VLGV macro the MRXBOPC macro is used to generate the RXB field of the instruction. The usage of the MRXBOPC macro is incorrect, since the vector register number is passed as third parameter (which correlates to the first bit of the RXB byte), while it should be passed as fourth parameter (second bit of the RXB byte). In result an incorrect instruction would be generated if the VLGV macro would be used for vector register numbers larger than 15. Fix this and pass the vector register number as fourth parameter. Currently there are no users within the kernel which use the macro in a way that broken code would be generated. Reviewed-by: Jens Remus Reviewed-by: Claudio Imbrenda Reviewed-by: Hendrik Brueckner Signed-off-by: Heiko Carstens --- arch/s390/include/asm/vx-insn-asm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/vx-insn-asm.h b/arch/s390/include/asm/vx-insn-asm.h index 360f8b36d962..d01be91e228b 100644 --- a/arch/s390/include/asm/vx-insn-asm.h +++ b/arch/s390/include/asm/vx-insn-asm.h @@ -350,7 +350,7 @@ VX_NUM v3, \vr .word 0xE700 | (r1 << 4) | (v3&15) .word (b2 << 12) | (\disp) - MRXBOPC \m, 0x21, v3 + MRXBOPC \m, 0x21, 0, v3 .endm .macro VLGVB gr, vr, disp, base="%r0" VLGV \gr, \vr, \disp, \base, 0 From 37edadee47bb36c6e7a72cfbe5535d3cf20e6e87 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:44:58 +0100 Subject: [PATCH 052/117] s390/fpu: improve description of RXB macro The v1, v2, v3, and v4 parameters of the RXB macro are a bit misleading, since the reader can assume that the parameters always correlate with the instructions format fields V1, V2, V3, and V4 as defined in the Principles of Operation. This is not the case for a couple of instructions, therefore improve the description of the macro. Suggested by Jens Remus, who also provided the improved description. Suggested-by: Jens Remus Reviewed-by: Jens Remus Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/vx-insn-asm.h | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/vx-insn-asm.h b/arch/s390/include/asm/vx-insn-asm.h index d01be91e228b..0d9f64b80127 100644 --- a/arch/s390/include/asm/vx-insn-asm.h +++ b/arch/s390/include/asm/vx-insn-asm.h @@ -195,10 +195,26 @@ /* RXB - Compute most significant bit used vector registers * * @rxb: Operand to store computed RXB value - * @v1: First vector register designated operand - * @v2: Second vector register designated operand - * @v3: Third vector register designated operand - * @v4: Fourth vector register designated operand + * @v1: Vector register designated operand whose MSB is stored in + * RXB bit 0 (instruction bit 36) and whose remaining bits + * are stored in instruction bits 8-11. + * @v2: Vector register designated operand whose MSB is stored in + * RXB bit 1 (instruction bit 37) and whose remaining bits + * are stored in instruction bits 12-15. + * @v3: Vector register designated operand whose MSB is stored in + * RXB bit 2 (instruction bit 38) and whose remaining bits + * are stored in instruction bits 16-19. + * @v4: Vector register designated operand whose MSB is stored in + * RXB bit 3 (instruction bit 39) and whose remaining bits + * are stored in instruction bits 32-35. + * + * Note: In most vector instruction formats [1] V1, V2, V3, and V4 directly + * correspond to @v1, @v2, @v3, and @v4. But there are exceptions, such as but + * not limited to the vector instruction formats VRR-g, VRR-h, VRS-a, VRS-d, + * and VSI. + * + * [1] IBM z/Architecture Principles of Operation, chapter "Program + * Execution, section "Instructions", subsection "Instruction Formats". */ .macro RXB rxb v1 v2=0 v3=0 v4=0 \rxb = 0 @@ -223,6 +239,9 @@ * @v2: Second vector register designated operand (for RXB) * @v3: Third vector register designated operand (for RXB) * @v4: Fourth vector register designated operand (for RXB) + * + * Note: For @v1, @v2, @v3, and @v4 also refer to the RXB macro + * description for further details. */ .macro MRXB m v1 v2=0 v3=0 v4=0 rxb = 0 @@ -238,6 +257,9 @@ * @v2: Second vector register designated operand (for RXB) * @v3: Third vector register designated operand (for RXB) * @v4: Fourth vector register designated operand (for RXB) + * + * Note: For @v1, @v2, @v3, and @v4 also refer to the RXB macro + * description for further details. */ .macro MRXBOPC m opc v1 v2=0 v3=0 v4=0 MRXB \m, \v1, \v2, \v3, \v4 From 9e96afab8c0fadafc3fa2dbb2f233a641d7fe229 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:44:59 +0100 Subject: [PATCH 053/117] s390/nmi: remove register validation code Remove the historic machine check handler code which validates registers. Registers are automatically validated as part of the machine check handling sequence (see Principles of Operation, Machine-Check Handling chapter, Validation). Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 5 -- arch/s390/kernel/nmi.c | 164 +++++++++++---------------------------- 2 files changed, 47 insertions(+), 122 deletions(-) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index f22cb6831f93..c6ccfc9d3704 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -489,16 +489,11 @@ SYM_FUNC_END(psw_idle) */ SYM_CODE_START(mcck_int_handler) BPOFF - la %r1,4095 # validate r1 - spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer - LBEAR __LC_LAST_BREAK_SAVE_AREA-4095(%r1) # validate bear - lmg %r0,%r15,__LC_GPREGS_SAVE_AREA # validate gprs lmg %r8,%r9,__LC_MCK_OLD_PSW TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE jo .Lmcck_panic # yes -> rest of mcck code invalid TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID jno .Lmcck_panic # control registers invalid -> panic - lctlg %c0,%c15,__LC_CREGS_SAVE_AREA # validate ctl regs ptlb lghi %r14,__LC_CPU_TIMER_SAVE_AREA mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index bbef8d954d01..6dd3020d4a2f 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -203,133 +203,63 @@ void s390_handle_mcck(void) } } -/* - * returns 0 if register contents could be validated - * returns 1 otherwise +/** + * nmi_registers_valid - verify if registers are valid + * @mci: machine check interruption code + * + * Inspect a machine check interruption code and verify if all required + * registers are valid. For some registers the corresponding validity bit is + * ignored and the registers are set to the expected value. + * Returns true if all registers are valid, otherwise false. */ -static int notrace s390_validate_registers(union mci mci) +static bool notrace nmi_registers_valid(union mci mci) { - struct mcesa *mcesa; - void *fpt_save_area; union ctlreg2 cr2; - int kill_task; - u64 zero; - kill_task = 0; - zero = 0; - - if (!mci.gr || !mci.fp) - kill_task = 1; - fpt_save_area = &S390_lowcore.floating_pt_save_area; - if (!mci.fc) { - kill_task = 1; - asm volatile( - " lfpc %0\n" - : - : "Q" (zero)); - } else { - asm volatile( - " lfpc %0\n" - : - : "Q" (S390_lowcore.fpt_creg_save_area)); - } - - mcesa = __va(S390_lowcore.mcesad & MCESA_ORIGIN_MASK); - if (!cpu_has_vx()) { - /* Validate floating point registers */ - asm volatile( - " ld 0,0(%0)\n" - " ld 1,8(%0)\n" - " ld 2,16(%0)\n" - " ld 3,24(%0)\n" - " ld 4,32(%0)\n" - " ld 5,40(%0)\n" - " ld 6,48(%0)\n" - " ld 7,56(%0)\n" - " ld 8,64(%0)\n" - " ld 9,72(%0)\n" - " ld 10,80(%0)\n" - " ld 11,88(%0)\n" - " ld 12,96(%0)\n" - " ld 13,104(%0)\n" - " ld 14,112(%0)\n" - " ld 15,120(%0)\n" - : - : "a" (fpt_save_area) - : "memory"); - } else { - /* Validate vector registers */ - union ctlreg0 cr0; - - /* - * The vector validity must only be checked if not running a - * KVM guest. For KVM guests the machine check is forwarded by - * KVM and it is the responsibility of the guest to take - * appropriate actions. The host vector or FPU values have been - * saved by KVM and will be restored by KVM. - */ - if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) - kill_task = 1; - cr0.reg = S390_lowcore.cregs_save_area[0]; - cr0.afp = cr0.vx = 1; - local_ctl_load(0, &cr0.reg); - asm volatile( - " la 1,%0\n" - " VLM 0,15,0,1\n" - " VLM 16,31,256,1\n" - : - : "Q" (*(struct vx_array *)mcesa->vector_save_area) - : "1"); - local_ctl_load(0, &S390_lowcore.cregs_save_area[0]); - } - /* Validate access registers */ - asm volatile( - " lam 0,15,0(%0)\n" - : - : "a" (&S390_lowcore.access_regs_save_area) - : "memory"); - if (!mci.ar) - kill_task = 1; - /* Validate guarded storage registers */ - cr2.reg = S390_lowcore.cregs_save_area[2]; - if (cr2.gse) { - if (!mci.gs) { - /* - * 2 cases: - * - machine check in kernel or userspace - * - machine check while running SIE (KVM guest) - * For kernel or userspace the userspace values of - * guarded storage control can not be recreated, the - * process must be terminated. - * For SIE the guest values of guarded storage can not - * be recreated. This is either due to a bug or due to - * GS being disabled in the guest. The guest will be - * notified by KVM code and the guests machine check - * handling must take care of this. The host values - * are saved by KVM and are not affected. - */ - if (!test_cpu_flag(CIF_MCCK_GUEST)) - kill_task = 1; - } else { - load_gs_cb((struct gs_cb *)mcesa->guarded_storage_save_area); - } - } /* - * The getcpu vdso syscall reads CPU number from the programmable + * The getcpu vdso syscall reads the CPU number from the programmable * field of the TOD clock. Disregard the TOD programmable register - * validity bit and load the CPU number into the TOD programmable - * field unconditionally. + * validity bit and load the CPU number into the TOD programmable field + * unconditionally. */ set_tod_programmable_field(raw_smp_processor_id()); - /* Validate clock comparator register */ + /* + * Set the clock comparator register to the next expected value. + */ set_clock_comparator(S390_lowcore.clock_comparator); - + if (!mci.gr || !mci.fp || !mci.fc) + return false; + /* + * The vector validity must only be checked if not running a + * KVM guest. For KVM guests the machine check is forwarded by + * KVM and it is the responsibility of the guest to take + * appropriate actions. The host vector or FPU values have been + * saved by KVM and will be restored by KVM. + */ + if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) + return false; + if (!mci.ar) + return false; + /* + * Two cases for guarded storage registers: + * - machine check in kernel or userspace + * - machine check while running SIE (KVM guest) + * For kernel or userspace the userspace values of guarded storage + * control can not be recreated, the process must be terminated. + * For SIE the guest values of guarded storage can not be recreated. + * This is either due to a bug or due to GS being disabled in the + * guest. The guest will be notified by KVM code and the guests machine + * check handling must take care of this. The host values are saved by + * KVM and are not affected. + */ + cr2.reg = S390_lowcore.cregs_save_area[2]; + if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST)) + return false; if (!mci.ms || !mci.pm || !mci.ia) - kill_task = 1; - - return kill_task; + return false; + return true; } -NOKPROBE_SYMBOL(s390_validate_registers); +NOKPROBE_SYMBOL(nmi_registers_valid); /* * Backup the guest's machine check info to its description block @@ -427,7 +357,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) s390_handle_damage(); } } - if (s390_validate_registers(mci)) { + if (!nmi_registers_valid(mci)) { if (!user_mode(regs)) s390_handle_damage(); /* From b6b842becd734301349b2fbe8d31099ea3d22a0f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:00 +0100 Subject: [PATCH 054/117] s390/fpu: use KERNEL_VXR_LOW instead of KERNEL_VXR_V0V7 Use KERNEL_VXR_LOW instead of KERNEL_VXR_V0V7 for configurations without vector registers in order to decide if floating point registers need to be saved and restored. Kernel FPU areas which use floating point registers are supposed to use the KERNEL_FPR mask, however users may also open-code this and specify KERNEL_VXR_V0V7 and/or KERNEL_VXR_V8V15. If only KERNEL_VXR_V8V15 is specified floating point registers wouldn't be saved and restored. Improve this and check for both bits. There are currently no users where this would fix a bug. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/kernel/fpu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index a4f3449cc814..21c9885cfb21 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -25,7 +25,7 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) asm volatile("stfpc %0" : "=Q" (state->fpc)); if (!cpu_has_vx()) { - if (flags & KERNEL_VXR_V0V7) { + if (flags & KERNEL_VXR_LOW) { /* Save floating-point registers */ asm volatile("std 0,%0" : "=Q" (state->fprs[0])); asm volatile("std 1,%0" : "=Q" (state->fprs[1])); @@ -107,7 +107,7 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) asm volatile("lfpc %0" : : "Q" (state->fpc)); if (!cpu_has_vx()) { - if (flags & KERNEL_VXR_V0V7) { + if (flags & KERNEL_VXR_LOW) { /* Restore floating-point registers */ asm volatile("ld 0,%0" : : "Q" (state->fprs[0])); asm volatile("ld 1,%0" : : "Q" (state->fprs[1])); From 31d3ec15dc95ad73ea403892840624465d377fd9 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:01 +0100 Subject: [PATCH 055/117] s390/fpu: various coding style changes Address various checkpatch warnings, adjust whitespace, and try to increase readability. This is just preparation, in order to avoid that subsequent patches contain any distracting drive-by coding style changes. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu/api.h | 19 ++++++++++--------- arch/s390/include/asm/fpu/internal.h | 10 ++++------ arch/s390/include/asm/fpu/types.h | 4 +++- arch/s390/kernel/fpu.c | 18 +++++++----------- 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/arch/s390/include/asm/fpu/api.h b/arch/s390/include/asm/fpu/api.h index d6ca8bc6ca68..24da55d33129 100644 --- a/arch/s390/include/asm/fpu/api.h +++ b/arch/s390/include/asm/fpu/api.h @@ -81,12 +81,12 @@ static inline void sfpc_safe(u32 fpc) #define KERNEL_VXR_V16V23 8 #define KERNEL_VXR_V24V31 16 -#define KERNEL_VXR_LOW (KERNEL_VXR_V0V7|KERNEL_VXR_V8V15) -#define KERNEL_VXR_MID (KERNEL_VXR_V8V15|KERNEL_VXR_V16V23) -#define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23|KERNEL_VXR_V24V31) +#define KERNEL_VXR_LOW (KERNEL_VXR_V0V7 | KERNEL_VXR_V8V15) +#define KERNEL_VXR_MID (KERNEL_VXR_V8V15 | KERNEL_VXR_V16V23) +#define KERNEL_VXR_HIGH (KERNEL_VXR_V16V23 | KERNEL_VXR_V24V31) -#define KERNEL_VXR (KERNEL_VXR_LOW|KERNEL_VXR_HIGH) -#define KERNEL_FPR (KERNEL_FPC|KERNEL_VXR_LOW) +#define KERNEL_VXR (KERNEL_VXR_LOW | KERNEL_VXR_HIGH) +#define KERNEL_FPR (KERNEL_FPC | KERNEL_VXR_LOW) struct kernel_fpu; @@ -100,26 +100,27 @@ struct kernel_fpu; void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); - static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { preempt_disable(); state->mask = S390_lowcore.fpu_flags; - if (!test_cpu_flag(CIF_FPU)) + if (!test_cpu_flag(CIF_FPU)) { /* Save user space FPU state and register contents */ save_fpu_regs(); - else if (state->mask & flags) + } else if (state->mask & flags) { /* Save FPU/vector register in-use by the kernel */ __kernel_fpu_begin(state, flags); + } S390_lowcore.fpu_flags |= flags; } static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) { S390_lowcore.fpu_flags = state->mask; - if (state->mask & flags) + if (state->mask & flags) { /* Restore FPU/vector register in-use by the kernel */ __kernel_fpu_end(state, flags); + } preempt_enable(); } diff --git a/arch/s390/include/asm/fpu/internal.h b/arch/s390/include/asm/fpu/internal.h index d511c4cf5afb..d7c0a100a444 100644 --- a/arch/s390/include/asm/fpu/internal.h +++ b/arch/s390/include/asm/fpu/internal.h @@ -20,11 +20,11 @@ static inline bool cpu_has_vx(void) static inline void save_vx_regs(__vector128 *vxrs) { - asm volatile( + asm volatile("\n" " la 1,%0\n" " .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ " .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ - : "=Q" (*(struct vx_array *) vxrs) : : "1"); + : "=Q" (*(struct vx_array *)vxrs) : : "1"); } static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) @@ -50,8 +50,7 @@ static inline void fpregs_store(_s390_fp_regs *fpregs, struct fpu *fpu) if (cpu_has_vx()) convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); else - memcpy((freg_t *)&fpregs->fprs, fpu->fprs, - sizeof(fpregs->fprs)); + memcpy((freg_t *)&fpregs->fprs, fpu->fprs, sizeof(fpregs->fprs)); } static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) @@ -60,8 +59,7 @@ static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) if (cpu_has_vx()) convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); else - memcpy(fpu->fprs, (freg_t *)&fpregs->fprs, - sizeof(fpregs->fprs)); + memcpy(fpu->fprs, (freg_t *)&fpregs->fprs, sizeof(fpregs->fprs)); } #endif /* _ASM_S390_FPU_INTERNAL_H */ diff --git a/arch/s390/include/asm/fpu/types.h b/arch/s390/include/asm/fpu/types.h index b1afa13c07b7..1caaf31209fc 100644 --- a/arch/s390/include/asm/fpu/types.h +++ b/arch/s390/include/asm/fpu/types.h @@ -23,7 +23,9 @@ struct fpu { }; /* VX array structure for address operand constraints in inline assemblies */ -struct vx_array { __vector128 _[__NUM_VXRS]; }; +struct vx_array { + __vector128 _[__NUM_VXRS]; +}; /* In-kernel FPU state structure */ struct kernel_fpu { diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 21c9885cfb21..d8e18a74b93d 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -16,14 +16,13 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { /* * Limit the save to the FPU/vector registers already - * in use by the previous context + * in use by the previous context. */ flags &= state->mask; - - if (flags & KERNEL_FPC) + if (flags & KERNEL_FPC) { /* Save floating point control */ asm volatile("stfpc %0" : "=Q" (state->fpc)); - + } if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) { /* Save floating-point registers */ @@ -46,7 +45,6 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) } return; } - /* Test and save vector registers */ asm volatile ( /* @@ -97,15 +95,14 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) { /* * Limit the restore to the FPU/vector registers of the - * previous context that have been overwritte by the - * current context + * previous context that have been overwritten by the + * current context. */ flags &= state->mask; - - if (flags & KERNEL_FPC) + if (flags & KERNEL_FPC) { /* Restore floating-point controls */ asm volatile("lfpc %0" : : "Q" (state->fpc)); - + } if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) { /* Restore floating-point registers */ @@ -128,7 +125,6 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) } return; } - /* Test and restore (load) vector registers */ asm volatile ( /* From fd2527f20915d041e838b6e4a08122dbc73c7abc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:02 +0100 Subject: [PATCH 056/117] s390/fpu: move, rename, and merge header files Move, rename, and merge the fpu and vx header files. This way fpu header files have a consistent naming scheme (fpu*.h). Also get rid of the fpu subdirectory and move header files to asm directory, so that all fpu and vx header files can be found at the same location. Merge internal.h header file into other header files, since the internal helpers are used at many locations. so those helper functions are really not internal. Signed-off-by: Heiko Carstens --- arch/s390/crypto/chacha-glue.c | 2 +- arch/s390/crypto/chacha-s390.S | 2 +- arch/s390/crypto/crc32-vx.c | 2 +- arch/s390/crypto/crc32be-vx.S | 2 +- arch/s390/crypto/crc32le-vx.S | 2 +- arch/s390/include/asm/asm-prototypes.h | 2 +- arch/s390/include/asm/entry-common.h | 2 +- .../asm/{vx-insn-asm.h => fpu-insn-asm.h} | 10 +-- arch/s390/include/asm/fpu-insn.h | 42 +++++++++ .../include/asm/{fpu/types.h => fpu-types.h} | 0 arch/s390/include/asm/{fpu/api.h => fpu.h} | 88 ++++++++++++------- arch/s390/include/asm/fpu/internal.h | 65 -------------- arch/s390/include/asm/kvm_host.h | 2 +- arch/s390/include/asm/processor.h | 3 +- arch/s390/include/asm/vx-insn.h | 19 ---- arch/s390/kernel/compat_signal.c | 2 +- arch/s390/kernel/crash_dump.c | 2 +- arch/s390/kernel/early.c | 2 +- arch/s390/kernel/entry.S | 2 +- arch/s390/kernel/fpu.c | 4 +- arch/s390/kernel/machine_kexec.c | 1 + arch/s390/kernel/nmi.c | 3 +- arch/s390/kernel/perf_regs.c | 3 +- arch/s390/kernel/process.c | 1 + arch/s390/kernel/ptrace.c | 2 +- arch/s390/kernel/smp.c | 1 + arch/s390/kernel/sysinfo.c | 2 +- arch/s390/kernel/traps.c | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- arch/s390/kvm/vsie.c | 2 +- lib/raid6/s390vx.uc | 3 +- 31 files changed, 130 insertions(+), 147 deletions(-) rename arch/s390/include/asm/{vx-insn-asm.h => fpu-insn-asm.h} (98%) create mode 100644 arch/s390/include/asm/fpu-insn.h rename arch/s390/include/asm/{fpu/types.h => fpu-types.h} (100%) rename arch/s390/include/asm/{fpu/api.h => fpu.h} (72%) delete mode 100644 arch/s390/include/asm/fpu/internal.h delete mode 100644 arch/s390/include/asm/vx-insn.h diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index a823132fc563..97098add2079 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include "chacha-s390.h" static void chacha20_crypt_s390(u32 *state, u8 *dst, const u8 *src, diff --git a/arch/s390/crypto/chacha-s390.S b/arch/s390/crypto/chacha-s390.S index 37cb63f25b17..63f3102678c0 100644 --- a/arch/s390/crypto/chacha-s390.S +++ b/arch/s390/crypto/chacha-s390.S @@ -8,7 +8,7 @@ #include #include -#include +#include #define SP %r15 #define FRAME (16 * 8 + 4 * 8) diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index 6ae3e3ff5b0a..dc2997f18e30 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #define CRC32_BLOCK_SIZE 1 diff --git a/arch/s390/crypto/crc32be-vx.S b/arch/s390/crypto/crc32be-vx.S index 34ee47926891..f2dc8a688afb 100644 --- a/arch/s390/crypto/crc32be-vx.S +++ b/arch/s390/crypto/crc32be-vx.S @@ -14,7 +14,7 @@ #include #include -#include +#include /* Vector register range containing CRC-32 constants */ #define CONST_R1R2 %v9 diff --git a/arch/s390/crypto/crc32le-vx.S b/arch/s390/crypto/crc32le-vx.S index 5a819ae09a0b..df19e06ff8bc 100644 --- a/arch/s390/crypto/crc32le-vx.S +++ b/arch/s390/crypto/crc32le-vx.S @@ -15,7 +15,7 @@ #include #include -#include +#include /* Vector register range containing CRC-32 constants */ #define CONST_PERM_LE2BE %v9 diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index a873e873e1ee..56096ae26f29 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include __int128_t __ashlti3(__int128_t a, int b); diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h index fdd319a622b0..a1dbab19c0bd 100644 --- a/arch/s390/include/asm/entry-common.h +++ b/arch/s390/include/asm/entry-common.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #define ARCH_EXIT_TO_USER_MODE_WORK (_TIF_GUARDED_STORAGE | _TIF_PER_TRAP) diff --git a/arch/s390/include/asm/vx-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h similarity index 98% rename from arch/s390/include/asm/vx-insn-asm.h rename to arch/s390/include/asm/fpu-insn-asm.h index 0d9f64b80127..789d626599ee 100644 --- a/arch/s390/include/asm/vx-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -9,11 +9,11 @@ * Author(s): Hendrik Brueckner */ -#ifndef __ASM_S390_VX_INSN_INTERNAL_H -#define __ASM_S390_VX_INSN_INTERNAL_H +#ifndef __ASM_S390_FPU_INSN_ASM_H +#define __ASM_S390_FPU_INSN_ASM_H -#ifndef __ASM_S390_VX_INSN_H -#error only can be included directly +#ifndef __ASM_S390_FPU_INSN_H +#error only can be included directly #endif #ifdef __ASSEMBLY__ @@ -700,4 +700,4 @@ .endm #endif /* __ASSEMBLY__ */ -#endif /* __ASM_S390_VX_INSN_INTERNAL_H */ +#endif /* __ASM_S390_FPU_INSN_ASM_H */ diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h new file mode 100644 index 000000000000..32d2894b60ca --- /dev/null +++ b/arch/s390/include/asm/fpu-insn.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Support for Floating Point and Vector Instructions + * + */ + +#ifndef __ASM_S390_FPU_INSN_H +#define __ASM_S390_FPU_INSN_H + +#include + +#ifndef __ASSEMBLY__ + +#include + +asm(".include \"asm/fpu-insn-asm.h\"\n"); + +/** + * sfpc_safe - Set floating point control register safely. + * @fpc: new value for floating point control register + * + * Set floating point control register. This may lead to an exception, + * since a saved value may have been modified by user space (ptrace, + * signal return, kvm registers) to an invalid value. In such a case + * set the floating point control register to zero. + */ +static inline void sfpc_safe(u32 fpc) +{ + asm volatile("\n" + "0: sfpc %[fpc]\n" + "1: nopr %%r7\n" + ".pushsection .fixup, \"ax\"\n" + "2: lghi %[fpc],0\n" + " jg 0b\n" + ".popsection\n" + EX_TABLE(1b, 2b) + : [fpc] "+d" (fpc) + : : "memory"); +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/include/asm/fpu/types.h b/arch/s390/include/asm/fpu-types.h similarity index 100% rename from arch/s390/include/asm/fpu/types.h rename to arch/s390/include/asm/fpu-types.h diff --git a/arch/s390/include/asm/fpu/api.h b/arch/s390/include/asm/fpu.h similarity index 72% rename from arch/s390/include/asm/fpu/api.h rename to arch/s390/include/asm/fpu.h index 24da55d33129..eed430c868df 100644 --- a/arch/s390/include/asm/fpu/api.h +++ b/arch/s390/include/asm/fpu.h @@ -41,40 +41,26 @@ * Author(s): Hendrik Brueckner */ -#ifndef _ASM_S390_FPU_API_H -#define _ASM_S390_FPU_API_H +#ifndef _ASM_S390_FPU_H +#define _ASM_S390_FPU_H +#include #include -#include -#include +#include +#include +#include +#include +#include + +static inline bool cpu_has_vx(void) +{ + return likely(test_facility(129)); +} void save_fpu_regs(void); void load_fpu_regs(void); void __load_fpu_regs(void); -/** - * sfpc_safe - Set floating point control register safely. - * @fpc: new value for floating point control register - * - * Set floating point control register. This may lead to an exception, - * since a saved value may have been modified by user space (ptrace, - * signal return, kvm registers) to an invalid value. In such a case - * set the floating point control register to zero. - */ -static inline void sfpc_safe(u32 fpc) -{ - asm volatile("\n" - "0: sfpc %[fpc]\n" - "1: nopr %%r7\n" - ".pushsection .fixup, \"ax\"\n" - "2: lghi %[fpc],0\n" - " jg 0b\n" - ".popsection\n" - EX_TABLE(1b, 2b) - : [fpc] "+d" (fpc) - : : "memory"); -} - #define KERNEL_FPC 1 #define KERNEL_VXR_V0V7 2 #define KERNEL_VXR_V8V15 4 @@ -88,8 +74,6 @@ static inline void sfpc_safe(u32 fpc) #define KERNEL_VXR (KERNEL_VXR_LOW | KERNEL_VXR_HIGH) #define KERNEL_FPR (KERNEL_FPC | KERNEL_VXR_LOW) -struct kernel_fpu; - /* * Note the functions below must be called with preemption disabled. * Do not enable preemption before calling __kernel_fpu_end() to prevent @@ -124,4 +108,48 @@ static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) preempt_enable(); } -#endif /* _ASM_S390_FPU_API_H */ +static inline void save_vx_regs(__vector128 *vxrs) +{ + asm volatile("\n" + " la 1,%0\n" + " .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ + " .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ + : "=Q" (*(struct vx_array *)vxrs) : : "1"); +} + +static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) +{ + int i; + + for (i = 0; i < __NUM_FPRS; i++) + fprs[i].ui = vxrs[i].high; +} + +static inline void convert_fp_to_vx(__vector128 *vxrs, freg_t *fprs) +{ + int i; + + for (i = 0; i < __NUM_FPRS; i++) + vxrs[i].high = fprs[i].ui; +} + +static inline void fpregs_store(_s390_fp_regs *fpregs, struct fpu *fpu) +{ + fpregs->pad = 0; + fpregs->fpc = fpu->fpc; + if (cpu_has_vx()) + convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); + else + memcpy((freg_t *)&fpregs->fprs, fpu->fprs, sizeof(fpregs->fprs)); +} + +static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) +{ + fpu->fpc = fpregs->fpc; + if (cpu_has_vx()) + convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); + else + memcpy(fpu->fprs, (freg_t *)&fpregs->fprs, sizeof(fpregs->fprs)); +} + +#endif /* _ASM_S390_FPU_H */ diff --git a/arch/s390/include/asm/fpu/internal.h b/arch/s390/include/asm/fpu/internal.h deleted file mode 100644 index d7c0a100a444..000000000000 --- a/arch/s390/include/asm/fpu/internal.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * FPU state and register content conversion primitives - * - * Copyright IBM Corp. 2015 - * Author(s): Hendrik Brueckner - */ - -#ifndef _ASM_S390_FPU_INTERNAL_H -#define _ASM_S390_FPU_INTERNAL_H - -#include -#include -#include - -static inline bool cpu_has_vx(void) -{ - return likely(test_facility(129)); -} - -static inline void save_vx_regs(__vector128 *vxrs) -{ - asm volatile("\n" - " la 1,%0\n" - " .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ - " .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ - : "=Q" (*(struct vx_array *)vxrs) : : "1"); -} - -static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) -{ - int i; - - for (i = 0; i < __NUM_FPRS; i++) - fprs[i].ui = vxrs[i].high; -} - -static inline void convert_fp_to_vx(__vector128 *vxrs, freg_t *fprs) -{ - int i; - - for (i = 0; i < __NUM_FPRS; i++) - vxrs[i].high = fprs[i].ui; -} - -static inline void fpregs_store(_s390_fp_regs *fpregs, struct fpu *fpu) -{ - fpregs->pad = 0; - fpregs->fpc = fpu->fpc; - if (cpu_has_vx()) - convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); - else - memcpy((freg_t *)&fpregs->fprs, fpu->fprs, sizeof(fpregs->fprs)); -} - -static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) -{ - fpu->fpc = fpregs->fpc; - if (cpu_has_vx()) - convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); - else - memcpy(fpu->fprs, (freg_t *)&fpregs->fprs, sizeof(fpregs->fprs)); -} - -#endif /* _ASM_S390_FPU_INTERNAL_H */ diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 52664105a473..e336715eb7d2 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index c0b6e74d899a..a422a2cf9d05 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -33,13 +33,12 @@ #include #include #include +#include #include #include #include #include #include -#include -#include #include typedef long (*sys_call_ptr_t)(struct pt_regs *regs); diff --git a/arch/s390/include/asm/vx-insn.h b/arch/s390/include/asm/vx-insn.h deleted file mode 100644 index 8c188f1c6d27..000000000000 --- a/arch/s390/include/asm/vx-insn.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Support for Vector Instructions - * - * This wrapper header file allows to use the vector instruction macros in - * both assembler files as well as in inline assemblies in C files. - */ - -#ifndef __ASM_S390_VX_INSN_H -#define __ASM_S390_VX_INSN_H - -#include - -#ifndef __ASSEMBLY__ - -asm(".include \"asm/vx-insn-asm.h\"\n"); - -#endif /* __ASSEMBLY__ */ -#endif /* __ASM_S390_VX_INSN_H */ diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index 13f41cb497be..6cd9bf925c82 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include "compat_linux.h" #include "compat_ptrace.h" #include "entry.h" diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 5c46c2659305..d09ebb6f5262 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #define PTR_ADD(x, y) (((char *) (x)) + ((unsigned long) (y))) #define PTR_SUB(x, y) (((char *) (x)) - ((unsigned long) (y))) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 63d5b51cb45a..c666271433fb 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -20,9 +20,9 @@ #include #include #include -#include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index c6ccfc9d3704..01c3b2d2821d 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index d8e18a74b93d..5d6a2339db40 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -8,9 +8,7 @@ #include #include #include -#include -#include -#include +#include void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 032a1124f04b..c5d0c1cf984b 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 6dd3020d4a2f..c77382a67325 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,6 @@ #include #include #include -#include -#include struct mcck_struct { unsigned int kill_task : 1; diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index 3d93656bd948..c8e8fb728ddb 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -5,8 +5,7 @@ #include #include #include -#include -#include +#include u64 perf_reg_value(struct pt_regs *regs, int idx) { diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 7227a73226f1..e502192da5f7 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 7c5e99be2155..2eafd6dcd592 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include "entry.h" diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index f77a54e1e4bd..0324649aae0a 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index b439f17516eb..061d45cf0261 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include int topology_max_mnest; diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 46dac4540ca8..08f8aee96d8f 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -28,8 +28,8 @@ #include #include #include -#include #include +#include #include "entry.h" static inline void __user *get_trap_ip(struct pt_regs *regs) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 6eab70b19abf..8f4414539756 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -43,9 +43,9 @@ #include #include #include +#include #include #include -#include #include "kvm-s390.h" #include "gaccess.h" #include "pci.h" diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index fef42e2a80a2..457d92c2949a 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include "kvm-s390.h" #include "gaccess.h" diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc index 7b0b355e1a26..bc2f4fbe5a82 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid6/s390vx.uc @@ -12,8 +12,7 @@ */ #include -#include -#include +#include #define NSIZE 16 From 045bad0800cec6098c30148e2873e20db742c750 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:03 +0100 Subject: [PATCH 057/117] s390/fpu: add documentation about fpu helper functions Add documentation which describes what the fpu helper functions are good for, and why they should be used. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 32d2894b60ca..ccdaacdeaa97 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -15,6 +15,26 @@ asm(".include \"asm/fpu-insn-asm.h\"\n"); +/* + * Various small helper functions, which can and should be used within + * kernel fpu code sections. Each function represents only one floating + * point or vector instruction (except for helper functions which require + * exception handling). + * + * This allows to use floating point and vector instructions like C + * functions, which has the advantage that all supporting code, like + * e.g. loops, can be written in easy to read C code. + * + * Each of the helper functions provides support for code instrumentation, + * like e.g. KASAN. Therefore instrumentation is also covered automatically + * when using these functions. + * + * In order to ensure that code generated with the helper functions stays + * within kernel fpu sections, which are guarded with kernel_fpu_begin() + * and kernel_fpu_end() calls, each function has a mandatory "memory" + * barrier. + */ + /** * sfpc_safe - Set floating point control register safely. * @fpc: new value for floating point control register From 13a8a519cacf04e970907687dcb1e85c03073ba8 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:04 +0100 Subject: [PATCH 058/117] s390/fpu: use lfpc instead of sfpc instruction The only user of sfpc_safe() needs to read the new fpc register value from memory before it is set with sfpc. Avoid this indirection and use lfpc, which reads the new value from memory. Also add the "fpu_" prefix to have a common name space for fpu related inline assemblies, and provide memory access instrumentation. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 22 ++++++++++++++-------- arch/s390/kernel/fpu.c | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index ccdaacdeaa97..64ba3d9bcd14 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -11,6 +11,7 @@ #ifndef __ASSEMBLY__ +#include #include asm(".include \"asm/fpu-insn-asm.h\"\n"); @@ -36,26 +37,31 @@ asm(".include \"asm/fpu-insn-asm.h\"\n"); */ /** - * sfpc_safe - Set floating point control register safely. + * fpu_lfpc_safe - Load floating point control register safely. * @fpc: new value for floating point control register * - * Set floating point control register. This may lead to an exception, + * Load floating point control register. This may lead to an exception, * since a saved value may have been modified by user space (ptrace, * signal return, kvm registers) to an invalid value. In such a case * set the floating point control register to zero. */ -static inline void sfpc_safe(u32 fpc) +static inline void fpu_lfpc_safe(unsigned int *fpc) { + u32 tmp; + + instrument_read(fpc, sizeof(*fpc)); asm volatile("\n" - "0: sfpc %[fpc]\n" + "0: lfpc %[fpc]\n" "1: nopr %%r7\n" ".pushsection .fixup, \"ax\"\n" - "2: lghi %[fpc],0\n" - " jg 0b\n" + "2: lghi %[tmp],0\n" + " sfpc %[tmp]\n" + " jg 1b\n" ".popsection\n" EX_TABLE(1b, 2b) - : [fpc] "+d" (fpc) - : : "memory"); + : [tmp] "=d" (tmp) + : [fpc] "Q" (*fpc) + : "memory"); } #endif /* __ASSEMBLY__ */ diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 5d6a2339db40..98dc9f593a14 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -174,7 +174,7 @@ void __load_fpu_regs(void) unsigned long *regs = current->thread.fpu.regs; struct fpu *state = ¤t->thread.fpu; - sfpc_safe(state->fpc); + fpu_lfpc_safe(&state->fpc); if (likely(cpu_has_vx())) { asm volatile("lgr 1,%0\n" "VLM 0,15,0,1\n" From 88d8136a0896e32fc39f90788eaa5c7bdccc9fb0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:05 +0100 Subject: [PATCH 059/117] s390/fpu: provide and use ld and std inline assemblies Deduplicate the 64 ld and std inline assemblies. Provide an fpu inline assembly for both instructions, and use them in the new save_fp_regs() and load_fp_regs() helper functions. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 18 +++++++ arch/s390/include/asm/fpu.h | 40 ++++++++++++++++ arch/s390/kernel/fpu.c | 81 ++++---------------------------- 3 files changed, 67 insertions(+), 72 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 64ba3d9bcd14..1ce8e2f9786c 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -36,6 +36,15 @@ asm(".include \"asm/fpu-insn-asm.h\"\n"); * barrier. */ +static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg) +{ + instrument_read(reg, sizeof(*reg)); + asm volatile("ld %[fpr],%[reg]\n" + : + : [fpr] "I" (fpr), [reg] "Q" (reg->ui) + : "memory"); +} + /** * fpu_lfpc_safe - Load floating point control register safely. * @fpc: new value for floating point control register @@ -64,5 +73,14 @@ static inline void fpu_lfpc_safe(unsigned int *fpc) : "memory"); } +static __always_inline void fpu_std(unsigned short fpr, freg_t *reg) +{ + instrument_write(reg, sizeof(*reg)); + asm volatile("std %[fpr],%[reg]\n" + : [reg] "=Q" (reg->ui) + : [fpr] "I" (fpr) + : "memory"); +} + #endif /* __ASSEMBLY__ */ #endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index eed430c868df..626695de6085 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -84,6 +84,46 @@ void __load_fpu_regs(void); void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); +static __always_inline void save_fp_regs(freg_t *fprs) +{ + fpu_std(0, &fprs[0]); + fpu_std(1, &fprs[1]); + fpu_std(2, &fprs[2]); + fpu_std(3, &fprs[3]); + fpu_std(4, &fprs[4]); + fpu_std(5, &fprs[5]); + fpu_std(6, &fprs[6]); + fpu_std(7, &fprs[7]); + fpu_std(8, &fprs[8]); + fpu_std(9, &fprs[9]); + fpu_std(10, &fprs[10]); + fpu_std(11, &fprs[11]); + fpu_std(12, &fprs[12]); + fpu_std(13, &fprs[13]); + fpu_std(14, &fprs[14]); + fpu_std(15, &fprs[15]); +} + +static __always_inline void load_fp_regs(freg_t *fprs) +{ + fpu_ld(0, &fprs[0]); + fpu_ld(1, &fprs[1]); + fpu_ld(2, &fprs[2]); + fpu_ld(3, &fprs[3]); + fpu_ld(4, &fprs[4]); + fpu_ld(5, &fprs[5]); + fpu_ld(6, &fprs[6]); + fpu_ld(7, &fprs[7]); + fpu_ld(8, &fprs[8]); + fpu_ld(9, &fprs[9]); + fpu_ld(10, &fprs[10]); + fpu_ld(11, &fprs[11]); + fpu_ld(12, &fprs[12]); + fpu_ld(13, &fprs[13]); + fpu_ld(14, &fprs[14]); + fpu_ld(15, &fprs[15]); +} + static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { preempt_disable(); diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 98dc9f593a14..f25c54caf32b 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -22,25 +22,8 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) asm volatile("stfpc %0" : "=Q" (state->fpc)); } if (!cpu_has_vx()) { - if (flags & KERNEL_VXR_LOW) { - /* Save floating-point registers */ - asm volatile("std 0,%0" : "=Q" (state->fprs[0])); - asm volatile("std 1,%0" : "=Q" (state->fprs[1])); - asm volatile("std 2,%0" : "=Q" (state->fprs[2])); - asm volatile("std 3,%0" : "=Q" (state->fprs[3])); - asm volatile("std 4,%0" : "=Q" (state->fprs[4])); - asm volatile("std 5,%0" : "=Q" (state->fprs[5])); - asm volatile("std 6,%0" : "=Q" (state->fprs[6])); - asm volatile("std 7,%0" : "=Q" (state->fprs[7])); - asm volatile("std 8,%0" : "=Q" (state->fprs[8])); - asm volatile("std 9,%0" : "=Q" (state->fprs[9])); - asm volatile("std 10,%0" : "=Q" (state->fprs[10])); - asm volatile("std 11,%0" : "=Q" (state->fprs[11])); - asm volatile("std 12,%0" : "=Q" (state->fprs[12])); - asm volatile("std 13,%0" : "=Q" (state->fprs[13])); - asm volatile("std 14,%0" : "=Q" (state->fprs[14])); - asm volatile("std 15,%0" : "=Q" (state->fprs[15])); - } + if (flags & KERNEL_VXR_LOW) + save_fp_regs(state->fprs); return; } /* Test and save vector registers */ @@ -102,25 +85,8 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) asm volatile("lfpc %0" : : "Q" (state->fpc)); } if (!cpu_has_vx()) { - if (flags & KERNEL_VXR_LOW) { - /* Restore floating-point registers */ - asm volatile("ld 0,%0" : : "Q" (state->fprs[0])); - asm volatile("ld 1,%0" : : "Q" (state->fprs[1])); - asm volatile("ld 2,%0" : : "Q" (state->fprs[2])); - asm volatile("ld 3,%0" : : "Q" (state->fprs[3])); - asm volatile("ld 4,%0" : : "Q" (state->fprs[4])); - asm volatile("ld 5,%0" : : "Q" (state->fprs[5])); - asm volatile("ld 6,%0" : : "Q" (state->fprs[6])); - asm volatile("ld 7,%0" : : "Q" (state->fprs[7])); - asm volatile("ld 8,%0" : : "Q" (state->fprs[8])); - asm volatile("ld 9,%0" : : "Q" (state->fprs[9])); - asm volatile("ld 10,%0" : : "Q" (state->fprs[10])); - asm volatile("ld 11,%0" : : "Q" (state->fprs[11])); - asm volatile("ld 12,%0" : : "Q" (state->fprs[12])); - asm volatile("ld 13,%0" : : "Q" (state->fprs[13])); - asm volatile("ld 14,%0" : : "Q" (state->fprs[14])); - asm volatile("ld 15,%0" : : "Q" (state->fprs[15])); - } + if (flags & KERNEL_VXR_LOW) + load_fp_regs(state->fprs); return; } /* Test and restore (load) vector registers */ @@ -171,8 +137,8 @@ EXPORT_SYMBOL(__kernel_fpu_end); void __load_fpu_regs(void) { - unsigned long *regs = current->thread.fpu.regs; struct fpu *state = ¤t->thread.fpu; + void *regs = current->thread.fpu.regs; fpu_lfpc_safe(&state->fpc); if (likely(cpu_has_vx())) { @@ -183,22 +149,7 @@ void __load_fpu_regs(void) : "d" (regs) : "1", "cc", "memory"); } else { - asm volatile("ld 0,%0" : : "Q" (regs[0])); - asm volatile("ld 1,%0" : : "Q" (regs[1])); - asm volatile("ld 2,%0" : : "Q" (regs[2])); - asm volatile("ld 3,%0" : : "Q" (regs[3])); - asm volatile("ld 4,%0" : : "Q" (regs[4])); - asm volatile("ld 5,%0" : : "Q" (regs[5])); - asm volatile("ld 6,%0" : : "Q" (regs[6])); - asm volatile("ld 7,%0" : : "Q" (regs[7])); - asm volatile("ld 8,%0" : : "Q" (regs[8])); - asm volatile("ld 9,%0" : : "Q" (regs[9])); - asm volatile("ld 10,%0" : : "Q" (regs[10])); - asm volatile("ld 11,%0" : : "Q" (regs[11])); - asm volatile("ld 12,%0" : : "Q" (regs[12])); - asm volatile("ld 13,%0" : : "Q" (regs[13])); - asm volatile("ld 14,%0" : : "Q" (regs[14])); - asm volatile("ld 15,%0" : : "Q" (regs[15])); + load_fp_regs(regs); } clear_cpu_flag(CIF_FPU); } @@ -213,8 +164,9 @@ EXPORT_SYMBOL(load_fpu_regs); void save_fpu_regs(void) { - unsigned long flags, *regs; + unsigned long flags; struct fpu *state; + void *regs; local_irq_save(flags); @@ -233,22 +185,7 @@ void save_fpu_regs(void) : "d" (regs) : "1", "cc", "memory"); } else { - asm volatile("std 0,%0" : "=Q" (regs[0])); - asm volatile("std 1,%0" : "=Q" (regs[1])); - asm volatile("std 2,%0" : "=Q" (regs[2])); - asm volatile("std 3,%0" : "=Q" (regs[3])); - asm volatile("std 4,%0" : "=Q" (regs[4])); - asm volatile("std 5,%0" : "=Q" (regs[5])); - asm volatile("std 6,%0" : "=Q" (regs[6])); - asm volatile("std 7,%0" : "=Q" (regs[7])); - asm volatile("std 8,%0" : "=Q" (regs[8])); - asm volatile("std 9,%0" : "=Q" (regs[9])); - asm volatile("std 10,%0" : "=Q" (regs[10])); - asm volatile("std 11,%0" : "=Q" (regs[11])); - asm volatile("std 12,%0" : "=Q" (regs[12])); - asm volatile("std 13,%0" : "=Q" (regs[13])); - asm volatile("std 14,%0" : "=Q" (regs[14])); - asm volatile("std 15,%0" : "=Q" (regs[15])); + save_fp_regs(regs); } set_cpu_flag(CIF_FPU); out: From f4e3de75d0c4ebe9bbbfef19d7845ee70cb017bd Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:06 +0100 Subject: [PATCH 060/117] s390/fpu: provide and use lfpc, sfpc, and stfpc inline assemblies Instead of open-coding lfpc, sfpc, and stfpc inline assemblies at several locations, provide an fpu_* function for each instruction and use the function instead. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 26 ++++++++++++++++++++++++++ arch/s390/kernel/fpu.c | 14 +++++--------- arch/s390/kernel/process.c | 2 +- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 1ce8e2f9786c..df2cad95b598 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -45,6 +45,15 @@ static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg) : "memory"); } +static __always_inline void fpu_lfpc(unsigned int *fpc) +{ + instrument_read(fpc, sizeof(*fpc)); + asm volatile("lfpc %[fpc]" + : + : [fpc] "Q" (*fpc) + : "memory"); +} + /** * fpu_lfpc_safe - Load floating point control register safely. * @fpc: new value for floating point control register @@ -82,5 +91,22 @@ static __always_inline void fpu_std(unsigned short fpr, freg_t *reg) : "memory"); } +static __always_inline void fpu_sfpc(unsigned int fpc) +{ + asm volatile("sfpc %[fpc]" + : + : [fpc] "d" (fpc) + : "memory"); +} + +static __always_inline void fpu_stfpc(unsigned int *fpc) +{ + instrument_write(fpc, sizeof(*fpc)); + asm volatile("stfpc %[fpc]" + : [fpc] "=Q" (*fpc) + : + : "memory"); +} + #endif /* __ASSEMBLY__ */ #endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index f25c54caf32b..6bfd4d0f33e1 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -17,10 +17,8 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) * in use by the previous context. */ flags &= state->mask; - if (flags & KERNEL_FPC) { - /* Save floating point control */ - asm volatile("stfpc %0" : "=Q" (state->fpc)); - } + if (flags & KERNEL_FPC) + fpu_stfpc(&state->fpc); if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) save_fp_regs(state->fprs); @@ -80,10 +78,8 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) * current context. */ flags &= state->mask; - if (flags & KERNEL_FPC) { - /* Restore floating-point controls */ - asm volatile("lfpc %0" : : "Q" (state->fpc)); - } + if (flags & KERNEL_FPC) + fpu_lfpc(&state->fpc); if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) load_fp_regs(state->fprs); @@ -176,7 +172,7 @@ void save_fpu_regs(void) state = ¤t->thread.fpu; regs = current->thread.fpu.regs; - asm volatile("stfpc %0" : "=Q" (state->fpc)); + fpu_stfpc(&state->fpc); if (likely(cpu_has_vx())) { asm volatile("lgr 1,%0\n" "VSTM 0,15,0,1\n" diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index e502192da5f7..b0578ea230e7 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -191,7 +191,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) void execve_tail(void) { current->thread.fpu.fpc = 0; - asm volatile("sfpc %0" : : "d" (0)); + fpu_sfpc(0); } struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) From 3a5866a001e83e1aa143fc0aeba0248247483962 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:07 +0100 Subject: [PATCH 061/117] s390/fpu: provide and use vlm and vstm inline assemblies Instead of open-coding vlm and vstm inline assemblies at several locations, provide an fpu_* function for each instruction, and use them in the new save_vx_regs() and load_vx_regs() helper functions. Note that "O" and "R" inline assembly operand modifiers are used in order to pass the displacement and base register of the memory operands to the existing VLM and VSTM macros. The two operand modifiers are not available for clang. Therefore provide two variants of each inline assembly. The clang variant always uses and clobbers general purpose register 1, like in the previous inline assemblies, so it can be used as base register with a zero displacement. This generates slightly less efficient code, but can be removed as soon as clang has support for the used operand modifiers. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 70 ++++++++++++++++++++++++++++++++ arch/s390/include/asm/fpu.h | 21 ++++++---- arch/s390/kernel/fpu.c | 24 +++-------- 3 files changed, 88 insertions(+), 27 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index df2cad95b598..538201864a47 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -108,5 +108,75 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : "memory"); } +#ifdef CONFIG_CC_IS_CLANG + +#define fpu_vlm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_read(_v, size); \ + asm volatile("\n" \ + " la 1,%[vxrs]\n" \ + " VLM %[v1],%[v3],0,1\n" \ + : \ + : [vxrs] "R" (*_v), \ + [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory", "1"); \ +} while (0) + +#else /* CONFIG_CC_IS_CLANG */ + +#define fpu_vlm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_read(_v, size); \ + asm volatile("VLM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \ + : \ + : [vxrs] "Q" (*_v), \ + [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + +#define fpu_vstm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_write(_v, size); \ + asm volatile("\n" \ + " la 1,%[vxrs]\n" \ + " VSTM %[v1],%[v3],0,1\n" \ + : [vxrs] "=R" (*_v) \ + : [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory", "1"); \ +} while (0) + +#else /* CONFIG_CC_IS_CLANG */ + +#define fpu_vstm(_v1, _v3, _vxrs) do { \ + unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ + struct { \ + __vector128 _v[(_v3) - (_v1) + 1]; \ + } *_v = (void *)(_vxrs); \ + \ + instrument_write(_v, size); \ + asm volatile("VSTM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \ + : [vxrs] "=Q" (*_v) \ + : [v1] "I" (_v1), [v3] "I" (_v3) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CC_IS_CLANG */ + #endif /* __ASSEMBLY__ */ #endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 626695de6085..6a0a23a28ce8 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -84,6 +84,18 @@ void __load_fpu_regs(void); void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); +static __always_inline void save_vx_regs(__vector128 *vxrs) +{ + fpu_vstm(0, 15, &vxrs[0]); + fpu_vstm(16, 31, &vxrs[16]); +} + +static __always_inline void load_vx_regs(__vector128 *vxrs) +{ + fpu_vlm(0, 15, &vxrs[0]); + fpu_vlm(16, 31, &vxrs[16]); +} + static __always_inline void save_fp_regs(freg_t *fprs) { fpu_std(0, &fprs[0]); @@ -148,15 +160,6 @@ static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) preempt_enable(); } -static inline void save_vx_regs(__vector128 *vxrs) -{ - asm volatile("\n" - " la 1,%0\n" - " .word 0xe70f,0x1000,0x003e\n" /* vstm 0,15,0(1) */ - " .word 0xe70f,0x1100,0x0c3e\n" /* vstm 16,31,256(1) */ - : "=Q" (*(struct vx_array *)vxrs) : : "1"); -} - static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) { int i; diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 6bfd4d0f33e1..092a4bdf88ed 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -137,16 +137,10 @@ void __load_fpu_regs(void) void *regs = current->thread.fpu.regs; fpu_lfpc_safe(&state->fpc); - if (likely(cpu_has_vx())) { - asm volatile("lgr 1,%0\n" - "VLM 0,15,0,1\n" - "VLM 16,31,256,1\n" - : - : "d" (regs) - : "1", "cc", "memory"); - } else { + if (likely(cpu_has_vx())) + load_vx_regs(regs); + else load_fp_regs(regs); - } clear_cpu_flag(CIF_FPU); } @@ -173,16 +167,10 @@ void save_fpu_regs(void) regs = current->thread.fpu.regs; fpu_stfpc(&state->fpc); - if (likely(cpu_has_vx())) { - asm volatile("lgr 1,%0\n" - "VSTM 0,15,0,1\n" - "VSTM 16,31,256,1\n" - : - : "d" (regs) - : "1", "cc", "memory"); - } else { + if (likely(cpu_has_vx())) + save_vx_regs(regs); + else save_fp_regs(regs); - } set_cpu_flag(CIF_FPU); out: local_irq_restore(flags); From 918c7cad66509c2170e38a088550fb4a525e0878 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:08 +0100 Subject: [PATCH 062/117] s390/fpu: convert __kernel_fpu_begin()/__kernel_fpu_end() to C Convert the rather large __kernel_fpu_begin()/__kernel_fpu_end() inline assemblies to C. The C variant is much more readable, and this also allows to get rid of the non-obvious usage of KERNEL_VXR_* constants within the inline assemblies. E.g. "tmll %[m],6" correlates with the two bits set in KERNEL_VXR_LOW. If the corresponding defines would be changed, the inline assembles would break in a subtle way. Therefore convert to C, use the proper defines, and allow the compiler to generate code using the (hopefully) most efficient instructions. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-types.h | 5 - arch/s390/include/asm/fpu.h | 18 +++- arch/s390/kernel/fpu.c | 148 +++++++++++++----------------- 3 files changed, 75 insertions(+), 96 deletions(-) diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h index 1caaf31209fc..743858dbc7fb 100644 --- a/arch/s390/include/asm/fpu-types.h +++ b/arch/s390/include/asm/fpu-types.h @@ -22,11 +22,6 @@ struct fpu { }; }; -/* VX array structure for address operand constraints in inline assemblies */ -struct vx_array { - __vector128 _[__NUM_VXRS]; -}; - /* In-kernel FPU state structure */ struct kernel_fpu { u32 mask; diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 6a0a23a28ce8..be85c28cdcde 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -61,11 +61,19 @@ void save_fpu_regs(void); void load_fpu_regs(void); void __load_fpu_regs(void); -#define KERNEL_FPC 1 -#define KERNEL_VXR_V0V7 2 -#define KERNEL_VXR_V8V15 4 -#define KERNEL_VXR_V16V23 8 -#define KERNEL_VXR_V24V31 16 +enum { + KERNEL_FPC_BIT = 0, + KERNEL_VXR_V0V7_BIT, + KERNEL_VXR_V8V15_BIT, + KERNEL_VXR_V16V23_BIT, + KERNEL_VXR_V24V31_BIT, +}; + +#define KERNEL_FPC BIT(KERNEL_FPC_BIT) +#define KERNEL_VXR_V0V7 BIT(KERNEL_VXR_V0V7_BIT) +#define KERNEL_VXR_V8V15 BIT(KERNEL_VXR_V8V15_BIT) +#define KERNEL_VXR_V16V23 BIT(KERNEL_VXR_V16V23_BIT) +#define KERNEL_VXR_V24V31 BIT(KERNEL_VXR_V24V31_BIT) #define KERNEL_VXR_LOW (KERNEL_VXR_V0V7 | KERNEL_VXR_V8V15) #define KERNEL_VXR_MID (KERNEL_VXR_V8V15 | KERNEL_VXR_V16V23) diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 092a4bdf88ed..0a31408a46f3 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -12,6 +12,9 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { + __vector128 *vxrs = state->vxrs; + u32 mask; + /* * Limit the save to the FPU/vector registers already * in use by the previous context. @@ -24,54 +27,42 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) save_fp_regs(state->fprs); return; } - /* Test and save vector registers */ - asm volatile ( - /* - * Test if any vector register must be saved and, if so, - * test if all register can be saved. - */ - " la 1,%[vxrs]\n" /* load save area */ - " tmll %[m],30\n" /* KERNEL_VXR */ - " jz 7f\n" /* no work -> done */ - " jo 5f\n" /* -> save V0..V31 */ - /* - * Test for special case KERNEL_FPU_MID only. In this - * case a vstm V8..V23 is the best instruction - */ - " chi %[m],12\n" /* KERNEL_VXR_MID */ - " jne 0f\n" /* -> save V8..V23 */ - " VSTM 8,23,128,1\n" /* vstm %v8,%v23,128(%r1) */ - " j 7f\n" - /* Test and save the first half of 16 vector registers */ - "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */ - " jz 3f\n" /* -> KERNEL_VXR_HIGH */ - " jo 2f\n" /* 11 -> save V0..V15 */ - " brc 2,1f\n" /* 10 -> save V8..V15 */ - " VSTM 0,7,0,1\n" /* vstm %v0,%v7,0(%r1) */ - " j 3f\n" - "1: VSTM 8,15,128,1\n" /* vstm %v8,%v15,128(%r1) */ - " j 3f\n" - "2: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */ - /* Test and save the second half of 16 vector registers */ - "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */ - " jz 7f\n" - " jo 6f\n" /* 11 -> save V16..V31 */ - " brc 2,4f\n" /* 10 -> save V24..V31 */ - " VSTM 16,23,256,1\n" /* vstm %v16,%v23,256(%r1) */ - " j 7f\n" - "4: VSTM 24,31,384,1\n" /* vstm %v24,%v31,384(%r1) */ - " j 7f\n" - "5: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */ - "6: VSTM 16,31,256,1\n" /* vstm %v16,%v31,256(%r1) */ - "7:" - : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) - : [m] "d" (flags) - : "1", "cc"); + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + fpu_vstm(0, 15, &vxrs[0]); + fpu_vstm(16, 31, &vxrs[16]); + return; + } + if (mask == KERNEL_VXR_MID) { + fpu_vstm(8, 23, &vxrs[8]); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + fpu_vstm(0, 15, &vxrs[0]); + else if (mask == KERNEL_VXR_V0V7) + fpu_vstm(0, 7, &vxrs[0]); + else + fpu_vstm(8, 15, &vxrs[8]); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + fpu_vstm(16, 31, &vxrs[16]); + else if (mask == KERNEL_VXR_V16V23) + fpu_vstm(16, 23, &vxrs[16]); + else + fpu_vstm(24, 31, &vxrs[24]); + } } EXPORT_SYMBOL(__kernel_fpu_begin); void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) { + __vector128 *vxrs = state->vxrs; + u32 mask; + /* * Limit the restore to the FPU/vector registers of the * previous context that have been overwritten by the @@ -85,49 +76,34 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) load_fp_regs(state->fprs); return; } - /* Test and restore (load) vector registers */ - asm volatile ( - /* - * Test if any vector register must be loaded and, if so, - * test if all registers can be loaded at once. - */ - " la 1,%[vxrs]\n" /* load restore area */ - " tmll %[m],30\n" /* KERNEL_VXR */ - " jz 7f\n" /* no work -> done */ - " jo 5f\n" /* -> restore V0..V31 */ - /* - * Test for special case KERNEL_FPU_MID only. In this - * case a vlm V8..V23 is the best instruction - */ - " chi %[m],12\n" /* KERNEL_VXR_MID */ - " jne 0f\n" /* -> restore V8..V23 */ - " VLM 8,23,128,1\n" /* vlm %v8,%v23,128(%r1) */ - " j 7f\n" - /* Test and restore the first half of 16 vector registers */ - "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */ - " jz 3f\n" /* -> KERNEL_VXR_HIGH */ - " jo 2f\n" /* 11 -> restore V0..V15 */ - " brc 2,1f\n" /* 10 -> restore V8..V15 */ - " VLM 0,7,0,1\n" /* vlm %v0,%v7,0(%r1) */ - " j 3f\n" - "1: VLM 8,15,128,1\n" /* vlm %v8,%v15,128(%r1) */ - " j 3f\n" - "2: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */ - /* Test and restore the second half of 16 vector registers */ - "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */ - " jz 7f\n" - " jo 6f\n" /* 11 -> restore V16..V31 */ - " brc 2,4f\n" /* 10 -> restore V24..V31 */ - " VLM 16,23,256,1\n" /* vlm %v16,%v23,256(%r1) */ - " j 7f\n" - "4: VLM 24,31,384,1\n" /* vlm %v24,%v31,384(%r1) */ - " j 7f\n" - "5: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */ - "6: VLM 16,31,256,1\n" /* vlm %v16,%v31,256(%r1) */ - "7:" - : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) - : [m] "d" (flags) - : "1", "cc"); + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + fpu_vlm(0, 15, &vxrs[0]); + fpu_vlm(16, 31, &vxrs[16]); + return; + } + if (mask == KERNEL_VXR_MID) { + fpu_vlm(8, 23, &vxrs[8]); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + fpu_vlm(0, 15, &vxrs[0]); + else if (mask == KERNEL_VXR_V0V7) + fpu_vlm(0, 7, &vxrs[0]); + else + fpu_vlm(8, 15, &vxrs[8]); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + fpu_vlm(16, 31, &vxrs[16]); + else if (mask == KERNEL_VXR_V16V23) + fpu_vlm(16, 23, &vxrs[16]); + else + fpu_vlm(24, 31, &vxrs[24]); + } } EXPORT_SYMBOL(__kernel_fpu_end); From 419abc4d3828813b58d047da146f519eedaa395b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:09 +0100 Subject: [PATCH 063/117] s390/fpu: convert FPU CIF flag to regular TIF flag The FPU state, as represented by the CIF_FPU flag reflects the FPU state of a task, not the CPU it is running on. Therefore convert the flag to a regular TIF flag. This removes the magic in switch_to() where a save_fpu_regs() call for the currently (previous) running task sets the per-cpu CIF_FPU flag, which is required to restore FPU register contents of the next task, when it returns to user space. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/entry-common.h | 2 +- arch/s390/include/asm/fpu.h | 2 +- arch/s390/include/asm/processor.h | 2 -- arch/s390/include/asm/thread_info.h | 2 ++ arch/s390/kernel/entry.S | 2 +- arch/s390/kernel/fpu.c | 6 +++--- arch/s390/kernel/process.c | 7 +------ arch/s390/kvm/kvm-s390.c | 2 +- arch/s390/kvm/vsie.c | 2 +- 9 files changed, 11 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h index a1dbab19c0bd..e479d39e1445 100644 --- a/arch/s390/include/asm/entry-common.h +++ b/arch/s390/include/asm/entry-common.h @@ -41,7 +41,7 @@ static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs, static __always_inline void arch_exit_to_user_mode(void) { - if (test_cpu_flag(CIF_FPU)) + if (test_thread_flag(TIF_FPU)) __load_fpu_regs(); if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index be85c28cdcde..3f076172a283 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -148,7 +148,7 @@ static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { preempt_disable(); state->mask = S390_lowcore.fpu_flags; - if (!test_cpu_flag(CIF_FPU)) { + if (!test_thread_flag(TIF_FPU)) { /* Save user space FPU state and register contents */ save_fpu_regs(); } else if (state->mask & flags) { diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index a422a2cf9d05..f25617cbc49e 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -15,13 +15,11 @@ #include #define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ -#define CIF_FPU 3 /* restore FPU registers */ #define CIF_ENABLED_WAIT 5 /* in enabled wait state */ #define CIF_MCCK_GUEST 6 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 7 /* this CPU is dedicated */ #define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) -#define _CIF_FPU BIT(CIF_FPU) #define _CIF_ENABLED_WAIT BIT(CIF_ENABLED_WAIT) #define _CIF_MCCK_GUEST BIT(CIF_MCCK_GUEST) #define _CIF_DEDICATED_CPU BIT(CIF_DEDICATED_CPU) diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index a674c7d25da5..e3f70b94a79b 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -69,6 +69,7 @@ void arch_setup_new_exec(void); #define TIF_PATCH_PENDING 5 /* pending live patching update */ #define TIF_PGSTE 6 /* New mm's will use 4K page tables */ #define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ +#define TIF_FPU 8 /* restore FPU registers on exit to usermode */ #define TIF_ISOLATE_BP_GUEST 9 /* Run KVM guests with isolated BP */ #define TIF_PER_TRAP 10 /* Need to handle PER trap on exit to usermode */ @@ -92,6 +93,7 @@ void arch_setup_new_exec(void); #define _TIF_UPROBE BIT(TIF_UPROBE) #define _TIF_GUARDED_STORAGE BIT(TIF_GUARDED_STORAGE) #define _TIF_PATCH_PENDING BIT(TIF_PATCH_PENDING) +#define _TIF_FPU BIT(TIF_FPU) #define _TIF_ISOLATE_BP_GUEST BIT(TIF_ISOLATE_BP_GUEST) #define _TIF_PER_TRAP BIT(TIF_PER_TRAP) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 01c3b2d2821d..00f2e1741501 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -220,7 +220,7 @@ SYM_FUNC_START(__sie64a) oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now tm __SIE_PROG20+3(%r14),3 # last exit... jnz .Lsie_skip - TSTMSK __LC_CPU_FLAGS,_CIF_FPU + TSTMSK __SF_SIE_FLAGS(%r15),_TIF_FPU jo .Lsie_skip # exit if fp/vx regs changed lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 0a31408a46f3..12d6e9d97104 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -117,7 +117,7 @@ void __load_fpu_regs(void) load_vx_regs(regs); else load_fp_regs(regs); - clear_cpu_flag(CIF_FPU); + clear_thread_flag(TIF_FPU); } void load_fpu_regs(void) @@ -136,7 +136,7 @@ void save_fpu_regs(void) local_irq_save(flags); - if (test_cpu_flag(CIF_FPU)) + if (test_thread_flag(TIF_FPU)) goto out; state = ¤t->thread.fpu; @@ -147,7 +147,7 @@ void save_fpu_regs(void) save_vx_regs(regs); else save_fp_regs(regs); - set_cpu_flag(CIF_FPU); + set_thread_flag(TIF_FPU); out: local_irq_restore(flags); } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index b0578ea230e7..f4c355f080f2 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -88,7 +88,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { /* * Save the floating-point or vector register state of the current - * task and set the CIF_FPU flag to lazy restore the FPU register + * task and set the TIF_FPU flag to lazy restore the FPU register * state when returning to user space. */ save_fpu_regs(); @@ -196,11 +196,6 @@ void execve_tail(void) struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) { - /* - * save_fpu_regs() sets the CIF_FPU flag, which enforces - * a restore of the floating point / vector registers as - * soon as the next task returns to user space. - */ save_fpu_regs(); save_access_regs(&prev->thread.acrs[0]); save_ri_cb(prev->thread.ri_cb); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 8f4414539756..0cee7192a1c2 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4829,7 +4829,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->run->s.regs.gprs, sizeof(sie_page->pv_grregs)); } - if (test_cpu_flag(CIF_FPU)) + if (test_thread_flag(TIF_FPU)) load_fpu_regs(); exit_reason = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 457d92c2949a..b8a242360ed0 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1149,7 +1149,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) */ vcpu->arch.sie_block->prog0c |= PROG_IN_SIE; barrier(); - if (test_cpu_flag(CIF_FPU)) + if (test_thread_flag(TIF_FPU)) load_fpu_regs(); if (!kvm_s390_vcpu_sie_inhibited(vcpu)) rc = sie64a(scb_s, vcpu->run->s.regs.gprs); From 87c5c70036813d26e6e7e4393747a4fdc63cf193 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:10 +0100 Subject: [PATCH 064/117] s390/fpu: rename save_fpu_regs() to save_user_fpu_regs(), etc Rename save_fpu_regs(), load_fpu_regs(), and struct thread_struct's fpu member to save_user_fpu_regs(), load_user_fpu_regs(), and ufpu. This way the function and variable names reflect for which context they are supposed to be used. This large and trivial conversion is a prerequisite for making the kernel fpu usage preemptible. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/entry-common.h | 2 +- arch/s390/include/asm/fpu.h | 8 +-- arch/s390/include/asm/processor.h | 4 +- arch/s390/kernel/compat_signal.c | 18 +++---- arch/s390/kernel/fpu.c | 20 +++---- arch/s390/kernel/perf_regs.c | 6 +-- arch/s390/kernel/process.c | 8 +-- arch/s390/kernel/ptrace.c | 78 ++++++++++++++-------------- arch/s390/kernel/signal.c | 18 +++---- arch/s390/kernel/traps.c | 10 ++-- arch/s390/kvm/interrupt.c | 4 +- arch/s390/kvm/kvm-s390.c | 26 +++++----- arch/s390/kvm/vsie.c | 2 +- 13 files changed, 102 insertions(+), 102 deletions(-) diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h index e479d39e1445..659e07d7f31a 100644 --- a/arch/s390/include/asm/entry-common.h +++ b/arch/s390/include/asm/entry-common.h @@ -42,7 +42,7 @@ static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs, static __always_inline void arch_exit_to_user_mode(void) { if (test_thread_flag(TIF_FPU)) - __load_fpu_regs(); + __load_user_fpu_regs(); if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) debug_user_asce(1); diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 3f076172a283..5d3533569925 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -57,9 +57,9 @@ static inline bool cpu_has_vx(void) return likely(test_facility(129)); } -void save_fpu_regs(void); -void load_fpu_regs(void); -void __load_fpu_regs(void); +void save_user_fpu_regs(void); +void load_user_fpu_regs(void); +void __load_user_fpu_regs(void); enum { KERNEL_FPC_BIT = 0, @@ -150,7 +150,7 @@ static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) state->mask = S390_lowcore.fpu_flags; if (!test_thread_flag(TIF_FPU)) { /* Save user space FPU state and register contents */ - save_fpu_regs(); + save_user_fpu_regs(); } else if (state->mask & flags) { /* Save FPU/vector register in-use by the kernel */ __kernel_fpu_begin(state, flags); diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index f25617cbc49e..2e716bf34bf8 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -181,7 +181,7 @@ struct thread_struct { struct gs_cb *gs_cb; /* Current guarded storage cb */ struct gs_cb *gs_bc_cb; /* Broadcast guarded storage cb */ struct pgm_tdb trap_tdb; /* Transaction abort diagnose block */ - struct fpu fpu; /* FP and VX register save area */ + struct fpu ufpu; /* User FP and VX register save area */ }; /* Flag to disable transactions. */ @@ -200,7 +200,7 @@ typedef struct thread_struct thread_struct; #define INIT_THREAD { \ .ksp = sizeof(init_stack) + (unsigned long) &init_stack, \ - .fpu.regs = (void *) init_task.thread.fpu.fprs, \ + .ufpu.regs = (void *)init_task.thread.ufpu.fprs, \ .last_break = 1, \ } diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index 6cd9bf925c82..1942e2a9f8db 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -56,7 +56,7 @@ typedef struct static void store_sigregs(void) { save_access_regs(current->thread.acrs); - save_fpu_regs(); + save_user_fpu_regs(); } /* Load registers after signal return */ @@ -79,7 +79,7 @@ static int save_sigregs32(struct pt_regs *regs, _sigregs32 __user *sregs) user_sregs.regs.gprs[i] = (__u32) regs->gprs[i]; memcpy(&user_sregs.regs.acrs, current->thread.acrs, sizeof(user_sregs.regs.acrs)); - fpregs_store((_s390_fp_regs *) &user_sregs.fpregs, ¤t->thread.fpu); + fpregs_store((_s390_fp_regs *) &user_sregs.fpregs, ¤t->thread.ufpu); if (__copy_to_user(sregs, &user_sregs, sizeof(_sigregs32))) return -EFAULT; return 0; @@ -113,7 +113,7 @@ static int restore_sigregs32(struct pt_regs *regs,_sigregs32 __user *sregs) regs->gprs[i] = (__u64) user_sregs.regs.gprs[i]; memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, sizeof(current->thread.acrs)); - fpregs_load((_s390_fp_regs *) &user_sregs.fpregs, ¤t->thread.fpu); + fpregs_load((_s390_fp_regs *)&user_sregs.fpregs, ¤t->thread.ufpu); clear_pt_regs_flag(regs, PIF_SYSCALL); /* No longer in a system call */ return 0; @@ -136,11 +136,11 @@ static int save_sigregs_ext32(struct pt_regs *regs, /* Save vector registers to signal stack */ if (cpu_has_vx()) { for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = current->thread.fpu.vxrs[i].low; + vxrs[i] = current->thread.ufpu.vxrs[i].low; if (__copy_to_user(&sregs_ext->vxrs_low, vxrs, sizeof(sregs_ext->vxrs_low)) || __copy_to_user(&sregs_ext->vxrs_high, - current->thread.fpu.vxrs + __NUM_VXRS_LOW, + current->thread.ufpu.vxrs + __NUM_VXRS_LOW, sizeof(sregs_ext->vxrs_high))) return -EFAULT; } @@ -165,12 +165,12 @@ static int restore_sigregs_ext32(struct pt_regs *regs, if (cpu_has_vx()) { if (__copy_from_user(vxrs, &sregs_ext->vxrs_low, sizeof(sregs_ext->vxrs_low)) || - __copy_from_user(current->thread.fpu.vxrs + __NUM_VXRS_LOW, + __copy_from_user(current->thread.ufpu.vxrs + __NUM_VXRS_LOW, &sregs_ext->vxrs_high, sizeof(sregs_ext->vxrs_high))) return -EFAULT; for (i = 0; i < __NUM_VXRS_LOW; i++) - current->thread.fpu.vxrs[i].low = vxrs[i]; + current->thread.ufpu.vxrs[i].low = vxrs[i]; } return 0; } @@ -184,7 +184,7 @@ COMPAT_SYSCALL_DEFINE0(sigreturn) if (get_compat_sigset(&set, (compat_sigset_t __user *)frame->sc.oldmask)) goto badframe; set_current_blocked(&set); - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs32(regs, &frame->sregs)) goto badframe; if (restore_sigregs_ext32(regs, &frame->sregs_ext)) @@ -207,7 +207,7 @@ COMPAT_SYSCALL_DEFINE0(rt_sigreturn) set_current_blocked(&set); if (compat_restore_altstack(&frame->uc.uc_stack)) goto badframe; - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs32(regs, &frame->uc.uc_mcontext)) goto badframe; if (restore_sigregs_ext32(regs, &frame->uc.uc_mcontext_ext)) diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 12d6e9d97104..62c9b2809057 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -107,10 +107,10 @@ void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) } EXPORT_SYMBOL(__kernel_fpu_end); -void __load_fpu_regs(void) +void __load_user_fpu_regs(void) { - struct fpu *state = ¤t->thread.fpu; - void *regs = current->thread.fpu.regs; + struct fpu *state = ¤t->thread.ufpu; + void *regs = current->thread.ufpu.regs; fpu_lfpc_safe(&state->fpc); if (likely(cpu_has_vx())) @@ -120,15 +120,15 @@ void __load_fpu_regs(void) clear_thread_flag(TIF_FPU); } -void load_fpu_regs(void) +void load_user_fpu_regs(void) { raw_local_irq_disable(); - __load_fpu_regs(); + __load_user_fpu_regs(); raw_local_irq_enable(); } -EXPORT_SYMBOL(load_fpu_regs); +EXPORT_SYMBOL(load_user_fpu_regs); -void save_fpu_regs(void) +void save_user_fpu_regs(void) { unsigned long flags; struct fpu *state; @@ -139,8 +139,8 @@ void save_fpu_regs(void) if (test_thread_flag(TIF_FPU)) goto out; - state = ¤t->thread.fpu; - regs = current->thread.fpu.regs; + state = ¤t->thread.ufpu; + regs = current->thread.ufpu.regs; fpu_stfpc(&state->fpc); if (likely(cpu_has_vx())) @@ -151,4 +151,4 @@ void save_fpu_regs(void) out: local_irq_restore(flags); } -EXPORT_SYMBOL(save_fpu_regs); +EXPORT_SYMBOL(save_user_fpu_regs); diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index c8e8fb728ddb..511349b8bc5c 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -20,9 +20,9 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) idx -= PERF_REG_S390_FP0; if (cpu_has_vx()) - fp = *(freg_t *)(current->thread.fpu.vxrs + idx); + fp = *(freg_t *)(current->thread.ufpu.vxrs + idx); else - fp = current->thread.fpu.fprs[idx]; + fp = current->thread.ufpu.fprs[idx]; return fp.ui; } @@ -64,6 +64,6 @@ void perf_get_regs_user(struct perf_regs *regs_user, */ regs_user->regs = task_pt_regs(current); if (user_mode(regs_user->regs)) - save_fpu_regs(); + save_user_fpu_regs(); regs_user->abi = perf_reg_abi(current); } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index f4c355f080f2..62146daf9051 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -91,10 +91,10 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) * task and set the TIF_FPU flag to lazy restore the FPU register * state when returning to user space. */ - save_fpu_regs(); + save_user_fpu_regs(); *dst = *src; - dst->thread.fpu.regs = dst->thread.fpu.fprs; + dst->thread.ufpu.regs = dst->thread.ufpu.fprs; /* * Don't transfer over the runtime instrumentation or the guarded @@ -190,13 +190,13 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) void execve_tail(void) { - current->thread.fpu.fpc = 0; + current->thread.ufpu.fpc = 0; fpu_sfpc(0); } struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) { - save_fpu_regs(); + save_user_fpu_regs(); save_access_regs(&prev->thread.acrs[0]); save_ri_cb(prev->thread.ri_cb); save_gs_cb(prev->thread.gs_cb); diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 2eafd6dcd592..f1ca79073173 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -247,21 +247,21 @@ static unsigned long __peek_user(struct task_struct *child, addr_t addr) /* * floating point control reg. is in the thread structure */ - tmp = child->thread.fpu.fpc; + tmp = child->thread.ufpu.fpc; tmp <<= BITS_PER_LONG - 32; } else if (addr < offsetof(struct user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are either in child->thread.ufpu + * or the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct user, regs.fp_regs.fprs); if (cpu_has_vx()) tmp = *(addr_t *) - ((addr_t) child->thread.fpu.vxrs + 2*offset); + ((addr_t)child->thread.ufpu.vxrs + 2 * offset); else tmp = *(addr_t *) - ((addr_t) child->thread.fpu.fprs + offset); + ((addr_t)child->thread.ufpu.fprs + offset); } else if (addr < offsetof(struct user, regs.per_info) + sizeof(per_struct)) { /* @@ -396,20 +396,20 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) */ if ((unsigned int)data != 0) return -EINVAL; - child->thread.fpu.fpc = data >> (BITS_PER_LONG - 32); + child->thread.ufpu.fpc = data >> (BITS_PER_LONG - 32); } else if (addr < offsetof(struct user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are either in child->thread.ufpu + * or the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct user, regs.fp_regs.fprs); if (cpu_has_vx()) *(addr_t *)((addr_t) - child->thread.fpu.vxrs + 2*offset) = data; + child->thread.ufpu.vxrs + 2 * offset) = data; else *(addr_t *)((addr_t) - child->thread.fpu.fprs + offset) = data; + child->thread.ufpu.fprs + offset) = data; } else if (addr < offsetof(struct user, regs.per_info) + sizeof(per_struct)) { /* @@ -623,20 +623,20 @@ static u32 __peek_user_compat(struct task_struct *child, addr_t addr) /* * floating point control reg. is in the thread structure */ - tmp = child->thread.fpu.fpc; + tmp = child->thread.ufpu.fpc; } else if (addr < offsetof(struct compat_user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are either in child->thread.ufpu + * or the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct compat_user, regs.fp_regs.fprs); if (cpu_has_vx()) tmp = *(__u32 *) - ((addr_t) child->thread.fpu.vxrs + 2*offset); + ((addr_t)child->thread.ufpu.vxrs + 2 * offset); else tmp = *(__u32 *) - ((addr_t) child->thread.fpu.fprs + offset); + ((addr_t)child->thread.ufpu.fprs + offset); } else if (addr < offsetof(struct compat_user, regs.per_info) + sizeof(struct compat_per_struct_kernel)) { /* @@ -749,20 +749,20 @@ static int __poke_user_compat(struct task_struct *child, /* * floating point control reg. is in the thread structure */ - child->thread.fpu.fpc = data; + child->thread.ufpu.fpc = data; } else if (addr < offsetof(struct compat_user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.fpu - * or the child->thread.fpu.vxrs array + * floating point regs. are either in child->thread.ufpu + * or the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct compat_user, regs.fp_regs.fprs); if (cpu_has_vx()) *(__u32 *)((addr_t) - child->thread.fpu.vxrs + 2*offset) = tmp; + child->thread.ufpu.vxrs + 2 * offset) = tmp; else *(__u32 *)((addr_t) - child->thread.fpu.fprs + offset) = tmp; + child->thread.ufpu.fprs + offset) = tmp; } else if (addr < offsetof(struct compat_user, regs.per_info) + sizeof(struct compat_per_struct_kernel)) { /* @@ -894,10 +894,10 @@ static int s390_fpregs_get(struct task_struct *target, _s390_fp_regs fp_regs; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); - fp_regs.fpc = target->thread.fpu.fpc; - fpregs_store(&fp_regs, &target->thread.fpu); + fp_regs.fpc = target->thread.ufpu.fpc; + fpregs_store(&fp_regs, &target->thread.ufpu); return membuf_write(&to, &fp_regs, sizeof(fp_regs)); } @@ -911,22 +911,22 @@ static int s390_fpregs_set(struct task_struct *target, freg_t fprs[__NUM_FPRS]; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); if (cpu_has_vx()) - convert_vx_to_fp(fprs, target->thread.fpu.vxrs); + convert_vx_to_fp(fprs, target->thread.ufpu.vxrs); else - memcpy(&fprs, target->thread.fpu.fprs, sizeof(fprs)); + memcpy(&fprs, target->thread.ufpu.fprs, sizeof(fprs)); if (count > 0 && pos < offsetof(s390_fp_regs, fprs)) { - u32 ufpc[2] = { target->thread.fpu.fpc, 0 }; + u32 ufpc[2] = { target->thread.ufpu.fpc, 0 }; rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ufpc, 0, offsetof(s390_fp_regs, fprs)); if (rc) return rc; if (ufpc[1] != 0) return -EINVAL; - target->thread.fpu.fpc = ufpc[0]; + target->thread.ufpu.fpc = ufpc[0]; } if (rc == 0 && count > 0) @@ -936,9 +936,9 @@ static int s390_fpregs_set(struct task_struct *target, return rc; if (cpu_has_vx()) - convert_fp_to_vx(target->thread.fpu.vxrs, fprs); + convert_fp_to_vx(target->thread.ufpu.vxrs, fprs); else - memcpy(target->thread.fpu.fprs, &fprs, sizeof(fprs)); + memcpy(target->thread.ufpu.fprs, &fprs, sizeof(fprs)); return rc; } @@ -989,9 +989,9 @@ static int s390_vxrs_low_get(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = target->thread.fpu.vxrs[i].low; + vxrs[i] = target->thread.ufpu.vxrs[i].low; return membuf_write(&to, vxrs, sizeof(vxrs)); } @@ -1006,15 +1006,15 @@ static int s390_vxrs_low_set(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = target->thread.fpu.vxrs[i].low; + vxrs[i] = target->thread.ufpu.vxrs[i].low; rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1); if (rc == 0) for (i = 0; i < __NUM_VXRS_LOW; i++) - target->thread.fpu.vxrs[i].low = vxrs[i]; + target->thread.ufpu.vxrs[i].low = vxrs[i]; return rc; } @@ -1026,8 +1026,8 @@ static int s390_vxrs_high_get(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); - return membuf_write(&to, target->thread.fpu.vxrs + __NUM_VXRS_LOW, + save_user_fpu_regs(); + return membuf_write(&to, target->thread.ufpu.vxrs + __NUM_VXRS_LOW, __NUM_VXRS_HIGH * sizeof(__vector128)); } @@ -1041,10 +1041,10 @@ static int s390_vxrs_high_set(struct task_struct *target, if (!cpu_has_vx()) return -ENODEV; if (target == current) - save_fpu_regs(); + save_user_fpu_regs(); rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - target->thread.fpu.vxrs + __NUM_VXRS_LOW, 0, -1); + target->thread.ufpu.vxrs + __NUM_VXRS_LOW, 0, -1); return rc; } diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index 1517aa70678b..6c2cb345402f 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -109,7 +109,7 @@ struct rt_sigframe static void store_sigregs(void) { save_access_regs(current->thread.acrs); - save_fpu_regs(); + save_user_fpu_regs(); } /* Load registers after signal return */ @@ -131,7 +131,7 @@ static int save_sigregs(struct pt_regs *regs, _sigregs __user *sregs) memcpy(&user_sregs.regs.gprs, ®s->gprs, sizeof(sregs->regs.gprs)); memcpy(&user_sregs.regs.acrs, current->thread.acrs, sizeof(user_sregs.regs.acrs)); - fpregs_store(&user_sregs.fpregs, ¤t->thread.fpu); + fpregs_store(&user_sregs.fpregs, ¤t->thread.ufpu); if (__copy_to_user(sregs, &user_sregs, sizeof(_sigregs))) return -EFAULT; return 0; @@ -165,7 +165,7 @@ static int restore_sigregs(struct pt_regs *regs, _sigregs __user *sregs) memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, sizeof(current->thread.acrs)); - fpregs_load(&user_sregs.fpregs, ¤t->thread.fpu); + fpregs_load(&user_sregs.fpregs, ¤t->thread.ufpu); clear_pt_regs_flag(regs, PIF_SYSCALL); /* No longer in a system call */ return 0; @@ -181,11 +181,11 @@ static int save_sigregs_ext(struct pt_regs *regs, /* Save vector registers to signal stack */ if (cpu_has_vx()) { for (i = 0; i < __NUM_VXRS_LOW; i++) - vxrs[i] = current->thread.fpu.vxrs[i].low; + vxrs[i] = current->thread.ufpu.vxrs[i].low; if (__copy_to_user(&sregs_ext->vxrs_low, vxrs, sizeof(sregs_ext->vxrs_low)) || __copy_to_user(&sregs_ext->vxrs_high, - current->thread.fpu.vxrs + __NUM_VXRS_LOW, + current->thread.ufpu.vxrs + __NUM_VXRS_LOW, sizeof(sregs_ext->vxrs_high))) return -EFAULT; } @@ -202,12 +202,12 @@ static int restore_sigregs_ext(struct pt_regs *regs, if (cpu_has_vx()) { if (__copy_from_user(vxrs, &sregs_ext->vxrs_low, sizeof(sregs_ext->vxrs_low)) || - __copy_from_user(current->thread.fpu.vxrs + __NUM_VXRS_LOW, + __copy_from_user(current->thread.ufpu.vxrs + __NUM_VXRS_LOW, &sregs_ext->vxrs_high, sizeof(sregs_ext->vxrs_high))) return -EFAULT; for (i = 0; i < __NUM_VXRS_LOW; i++) - current->thread.fpu.vxrs[i].low = vxrs[i]; + current->thread.ufpu.vxrs[i].low = vxrs[i]; } return 0; } @@ -222,7 +222,7 @@ SYSCALL_DEFINE0(sigreturn) if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE)) goto badframe; set_current_blocked(&set); - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs(regs, &frame->sregs)) goto badframe; if (restore_sigregs_ext(regs, &frame->sregs_ext)) @@ -246,7 +246,7 @@ SYSCALL_DEFINE0(rt_sigreturn) set_current_blocked(&set); if (restore_altstack(&frame->uc.uc_stack)) goto badframe; - save_fpu_regs(); + save_user_fpu_regs(); if (restore_sigregs(regs, &frame->uc.uc_mcontext)) goto badframe; if (restore_sigregs_ext(regs, &frame->uc.uc_mcontext_ext)) diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 08f8aee96d8f..52578b5cecbd 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -201,8 +201,8 @@ static void vector_exception(struct pt_regs *regs) } /* get vector interrupt code from fpc */ - save_fpu_regs(); - vic = (current->thread.fpu.fpc & 0xf00) >> 8; + save_user_fpu_regs(); + vic = (current->thread.ufpu.fpc & 0xf00) >> 8; switch (vic) { case 1: /* invalid vector operation */ si_code = FPE_FLTINV; @@ -227,9 +227,9 @@ static void vector_exception(struct pt_regs *regs) static void data_exception(struct pt_regs *regs) { - save_fpu_regs(); - if (current->thread.fpu.fpc & FPC_DXC_MASK) - do_fp_trap(regs, current->thread.fpu.fpc); + save_user_fpu_regs(); + if (current->thread.ufpu.fpc & FPC_DXC_MASK) + do_fp_trap(regs, current->thread.ufpu.fpc); else do_trap(regs, SIGILL, ILL_ILLOPN, "data exception"); } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 6e435bf0c27e..9315203c2786 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -584,7 +584,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, mci.val = mchk->mcic; /* take care of lazy register loading */ - save_fpu_regs(); + save_user_fpu_regs(); save_access_regs(vcpu->run->s.regs.acrs); if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); @@ -648,7 +648,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, } rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, vcpu->run->s.regs.gprs, 128); - rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, + rc |= put_guest_lc(vcpu, current->thread.ufpu.fpc, (u32 __user *) __LC_FP_CREG_SAVE_AREA); rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 0cee7192a1c2..3ce4029cabc2 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4830,7 +4830,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) sizeof(sie_page->pv_grregs)); } if (test_thread_flag(TIF_FPU)) - load_fpu_regs(); + load_user_fpu_regs(); exit_reason = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); if (kvm_s390_pv_cpu_is_protected(vcpu)) { @@ -4952,14 +4952,14 @@ static void sync_regs(struct kvm_vcpu *vcpu) save_access_regs(vcpu->arch.host_acrs); restore_access_regs(vcpu->run->s.regs.acrs); /* save host (userspace) fprs/vrs */ - save_fpu_regs(); - vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; - vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; + save_user_fpu_regs(); + vcpu->arch.host_fpregs.fpc = current->thread.ufpu.fpc; + vcpu->arch.host_fpregs.regs = current->thread.ufpu.regs; if (cpu_has_vx()) - current->thread.fpu.regs = vcpu->run->s.regs.vrs; + current->thread.ufpu.regs = vcpu->run->s.regs.vrs; else - current->thread.fpu.regs = vcpu->run->s.regs.fprs; - current->thread.fpu.fpc = vcpu->run->s.regs.fpc; + current->thread.ufpu.regs = vcpu->run->s.regs.fprs; + current->thread.ufpu.fpc = vcpu->run->s.regs.fpc; /* Sync fmt2 only data */ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { @@ -5022,11 +5022,11 @@ static void store_regs(struct kvm_vcpu *vcpu) save_access_regs(vcpu->run->s.regs.acrs); restore_access_regs(vcpu->arch.host_acrs); /* Save guest register state */ - save_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.fpu.fpc; + save_user_fpu_regs(); + vcpu->run->s.regs.fpc = current->thread.ufpu.fpc; /* Restore will be done lazily at return */ - current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; - current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; + current->thread.ufpu.fpc = vcpu->arch.host_fpregs.fpc; + current->thread.ufpu.regs = vcpu->arch.host_fpregs.regs; if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) store_regs_fmt2(vcpu); } @@ -5172,8 +5172,8 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) * switch in the run ioctl. Let's update our copies before we save * it into the save area */ - save_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.fpu.fpc; + save_user_fpu_regs(); + vcpu->run->s.regs.fpc = current->thread.ufpu.fpc; save_access_regs(vcpu->run->s.regs.acrs); return kvm_s390_store_status_unloaded(vcpu, addr); diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index b8a242360ed0..e0f79c9a4852 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1150,7 +1150,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) vcpu->arch.sie_block->prog0c |= PROG_IN_SIE; barrier(); if (test_thread_flag(TIF_FPU)) - load_fpu_regs(); + load_user_fpu_regs(); if (!kvm_s390_vcpu_sie_inhibited(vcpu)) rc = sie64a(scb_s, vcpu->run->s.regs.gprs); barrier(); From c038b984a9af1010555986d6fe32d4da9db9fc3d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:11 +0100 Subject: [PATCH 065/117] s390/fpu: change type of fpu mask from u32 to int Change type of fpu mask consistently from u32 to int. This is a prerequisite to make the kernel fpu usage preemptible. Upcoming code uses __atomic* ops which work with int pointers. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-types.h | 2 +- arch/s390/include/asm/fpu.h | 8 ++++---- arch/s390/kernel/fpu.c | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h index 743858dbc7fb..fee4468a85d3 100644 --- a/arch/s390/include/asm/fpu-types.h +++ b/arch/s390/include/asm/fpu-types.h @@ -24,7 +24,7 @@ struct fpu { /* In-kernel FPU state structure */ struct kernel_fpu { - u32 mask; + int mask; u32 fpc; union { freg_t fprs[__NUM_FPRS]; diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 5d3533569925..447d68fb41b2 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -89,8 +89,8 @@ enum { * * Prefer using the kernel_fpu_begin()/kernel_fpu_end() pair of functions. */ -void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags); -void __kernel_fpu_end(struct kernel_fpu *state, u32 flags); +void __kernel_fpu_begin(struct kernel_fpu *state, int flags); +void __kernel_fpu_end(struct kernel_fpu *state, int flags); static __always_inline void save_vx_regs(__vector128 *vxrs) { @@ -144,7 +144,7 @@ static __always_inline void load_fp_regs(freg_t *fprs) fpu_ld(15, &fprs[15]); } -static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) +static inline void kernel_fpu_begin(struct kernel_fpu *state, int flags) { preempt_disable(); state->mask = S390_lowcore.fpu_flags; @@ -158,7 +158,7 @@ static inline void kernel_fpu_begin(struct kernel_fpu *state, u32 flags) S390_lowcore.fpu_flags |= flags; } -static inline void kernel_fpu_end(struct kernel_fpu *state, u32 flags) +static inline void kernel_fpu_end(struct kernel_fpu *state, int flags) { S390_lowcore.fpu_flags = state->mask; if (state->mask & flags) { diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 62c9b2809057..b976da5bf71b 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -10,10 +10,10 @@ #include #include -void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) +void __kernel_fpu_begin(struct kernel_fpu *state, int flags) { __vector128 *vxrs = state->vxrs; - u32 mask; + int mask; /* * Limit the save to the FPU/vector registers already @@ -58,10 +58,10 @@ void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) } EXPORT_SYMBOL(__kernel_fpu_begin); -void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) +void __kernel_fpu_end(struct kernel_fpu *state, int flags) { __vector128 *vxrs = state->vxrs; - u32 mask; + int mask; /* * Limit the restore to the FPU/vector registers of the From 4eed43de9ba0ae3af6716544408d185a152424cd Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:12 +0100 Subject: [PATCH 066/117] s390/fpu: make kernel fpu context preemptible Make the kernel fpu context preemptible. Add another fpu structure to the thread_struct, and use it to save and restore the kernel fpu context if its task uses fpu registers when it is preempted. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu.h | 42 ++++++++++++++++++++++--------- arch/s390/include/asm/lowcore.h | 2 +- arch/s390/include/asm/processor.h | 2 ++ arch/s390/kernel/process.c | 3 +++ 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 447d68fb41b2..4300eef243f9 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -82,13 +83,6 @@ enum { #define KERNEL_VXR (KERNEL_VXR_LOW | KERNEL_VXR_HIGH) #define KERNEL_FPR (KERNEL_FPC | KERNEL_VXR_LOW) -/* - * Note the functions below must be called with preemption disabled. - * Do not enable preemption before calling __kernel_fpu_end() to prevent - * an corruption of an existing kernel FPU state. - * - * Prefer using the kernel_fpu_begin()/kernel_fpu_end() pair of functions. - */ void __kernel_fpu_begin(struct kernel_fpu *state, int flags); void __kernel_fpu_end(struct kernel_fpu *state, int flags); @@ -146,8 +140,7 @@ static __always_inline void load_fp_regs(freg_t *fprs) static inline void kernel_fpu_begin(struct kernel_fpu *state, int flags) { - preempt_disable(); - state->mask = S390_lowcore.fpu_flags; + state->mask = READ_ONCE(current->thread.kfpu_flags); if (!test_thread_flag(TIF_FPU)) { /* Save user space FPU state and register contents */ save_user_fpu_regs(); @@ -155,17 +148,42 @@ static inline void kernel_fpu_begin(struct kernel_fpu *state, int flags) /* Save FPU/vector register in-use by the kernel */ __kernel_fpu_begin(state, flags); } - S390_lowcore.fpu_flags |= flags; + __atomic_or(flags, ¤t->thread.kfpu_flags); } static inline void kernel_fpu_end(struct kernel_fpu *state, int flags) { - S390_lowcore.fpu_flags = state->mask; + WRITE_ONCE(current->thread.kfpu_flags, state->mask); if (state->mask & flags) { /* Restore FPU/vector register in-use by the kernel */ __kernel_fpu_end(state, flags); } - preempt_enable(); +} + +static inline void save_kernel_fpu_regs(struct thread_struct *thread) +{ + struct fpu *state = &thread->kfpu; + + if (!thread->kfpu_flags) + return; + fpu_stfpc(&state->fpc); + if (likely(cpu_has_vx())) + save_vx_regs(state->vxrs); + else + save_fp_regs(state->fprs); +} + +static inline void restore_kernel_fpu_regs(struct thread_struct *thread) +{ + struct fpu *state = &thread->kfpu; + + if (!thread->kfpu_flags) + return; + fpu_lfpc(&state->fpc); + if (likely(cpu_has_vx())) + load_vx_regs(state->vxrs); + else + load_fp_regs(state->fprs); } static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 5dc1b6345006..8c5f16857539 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -157,7 +157,7 @@ struct lowcore { __s32 preempt_count; /* 0x03a8 */ __u32 spinlock_lockval; /* 0x03ac */ __u32 spinlock_index; /* 0x03b0 */ - __u32 fpu_flags; /* 0x03b4 */ + __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ __u8 pad_0x03c0[0x03c8-0x03c0]; /* 0x03c0 */ __u64 machine_flags; /* 0x03c8 */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 2e716bf34bf8..eee0a1eec620 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -166,6 +166,7 @@ struct thread_struct { unsigned int gmap_write_flag; /* gmap fault write indication */ unsigned int gmap_int_code; /* int code of last gmap fault */ unsigned int gmap_pfault; /* signal of a pending guest pfault */ + int kfpu_flags; /* kernel fpu flags */ /* Per-thread information related to debugging */ struct per_regs per_user; /* User specified PER registers */ @@ -182,6 +183,7 @@ struct thread_struct { struct gs_cb *gs_bc_cb; /* Broadcast guarded storage cb */ struct pgm_tdb trap_tdb; /* Transaction abort diagnose block */ struct fpu ufpu; /* User FP and VX register save area */ + struct fpu kfpu; /* Kernel FP and VX register save area */ }; /* Flag to disable transactions. */ diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 62146daf9051..b7b623818753 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -95,6 +95,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; dst->thread.ufpu.regs = dst->thread.ufpu.fprs; + dst->thread.kfpu_flags = 0; /* * Don't transfer over the runtime instrumentation or the guarded @@ -197,10 +198,12 @@ void execve_tail(void) struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next) { save_user_fpu_regs(); + save_kernel_fpu_regs(&prev->thread); save_access_regs(&prev->thread.acrs[0]); save_ri_cb(prev->thread.ri_cb); save_gs_cb(prev->thread.gs_cb); update_cr_regs(next); + restore_kernel_fpu_regs(&next->thread); restore_access_regs(&next->thread.acrs[0]); restore_ri_cb(next->thread.ri_cb, prev->thread.ri_cb); restore_gs_cb(next->thread.gs_cb); From ed3a0a011a9c33d81a0d024882ee433c42bfccae Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:13 +0100 Subject: [PATCH 067/117] s390/kvm: convert to regular kernel fpu user KVM modifies the kernel fpu's regs pointer to its own area to implement its custom version of preemtible kernel fpu context. With general support for preemptible kernel fpu context there is no need for the extra complexity in KVM code anymore. Therefore convert KVM to a regular kernel fpu user. In particular this means that all TIF_FPU checks can be removed, since the fpu register context will never be changed by other kernel fpu users, and also the fpu register context will be restored if a thread is preempted. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/kvm_host.h | 1 - arch/s390/kernel/entry.S | 2 -- arch/s390/kvm/interrupt.c | 8 ++++++-- arch/s390/kvm/kvm-s390.c | 34 +++++++++++++++----------------- arch/s390/kvm/vsie.c | 3 --- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index e336715eb7d2..56c2efb41cf2 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -743,7 +743,6 @@ struct kvm_vcpu_arch { struct kvm_s390_sie_block *vsie_block; unsigned int host_acrs[NUM_ACRS]; struct gs_cb *host_gscb; - struct fpu host_fpregs; struct kvm_s390_local_interrupt local_int; struct hrtimer ckc_timer; struct kvm_s390_pgm_info pgm; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 00f2e1741501..fc5277eab554 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -220,8 +220,6 @@ SYM_FUNC_START(__sie64a) oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now tm __SIE_PROG20+3(%r14),3 # last exit... jnz .Lsie_skip - TSTMSK __SF_SIE_FLAGS(%r15),_TIF_FPU - jo .Lsie_skip # exit if fp/vx regs changed lg %r14,__SF_SIE_CONTROL_PHYS(%r15) # get sie block phys addr BPEXIT __SF_SIE_FLAGS(%r15),_TIF_ISOLATE_BP_GUEST .Lsie_entry: diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 9315203c2786..c81708acd1f4 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -584,7 +584,11 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, mci.val = mchk->mcic; /* take care of lazy register loading */ - save_user_fpu_regs(); + fpu_stfpc(&vcpu->run->s.regs.fpc); + if (cpu_has_vx()) + save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); + else + save_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); save_access_regs(vcpu->run->s.regs.acrs); if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); @@ -648,7 +652,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, } rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, vcpu->run->s.regs.gprs, 128); - rc |= put_guest_lc(vcpu, current->thread.ufpu.fpc, + rc |= put_guest_lc(vcpu, vcpu->run->s.regs.fpc, (u32 __user *) __LC_FP_CREG_SAVE_AREA); rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 3ce4029cabc2..8467945344b5 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4829,8 +4829,6 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) vcpu->run->s.regs.gprs, sizeof(sie_page->pv_grregs)); } - if (test_thread_flag(TIF_FPU)) - load_user_fpu_regs(); exit_reason = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs); if (kvm_s390_pv_cpu_is_protected(vcpu)) { @@ -4951,16 +4949,11 @@ static void sync_regs(struct kvm_vcpu *vcpu) } save_access_regs(vcpu->arch.host_acrs); restore_access_regs(vcpu->run->s.regs.acrs); - /* save host (userspace) fprs/vrs */ - save_user_fpu_regs(); - vcpu->arch.host_fpregs.fpc = current->thread.ufpu.fpc; - vcpu->arch.host_fpregs.regs = current->thread.ufpu.regs; + fpu_lfpc_safe(&vcpu->run->s.regs.fpc); if (cpu_has_vx()) - current->thread.ufpu.regs = vcpu->run->s.regs.vrs; + load_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); else - current->thread.ufpu.regs = vcpu->run->s.regs.fprs; - current->thread.ufpu.fpc = vcpu->run->s.regs.fpc; - + load_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); /* Sync fmt2 only data */ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { sync_regs_fmt2(vcpu); @@ -5021,12 +5014,11 @@ static void store_regs(struct kvm_vcpu *vcpu) kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; save_access_regs(vcpu->run->s.regs.acrs); restore_access_regs(vcpu->arch.host_acrs); - /* Save guest register state */ - save_user_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.ufpu.fpc; - /* Restore will be done lazily at return */ - current->thread.ufpu.fpc = vcpu->arch.host_fpregs.fpc; - current->thread.ufpu.regs = vcpu->arch.host_fpregs.regs; + fpu_stfpc(&vcpu->run->s.regs.fpc); + if (cpu_has_vx()) + save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); + else + save_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) store_regs_fmt2(vcpu); } @@ -5034,6 +5026,7 @@ static void store_regs(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; + DECLARE_KERNEL_FPU_ONSTACK(fpu); int rc; /* @@ -5075,6 +5068,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) goto out; } + kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); sync_regs(vcpu); enable_cpu_timer_accounting(vcpu); @@ -5098,6 +5092,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) disable_cpu_timer_accounting(vcpu); store_regs(vcpu); + kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); kvm_sigset_deactivate(vcpu); @@ -5172,8 +5167,11 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) * switch in the run ioctl. Let's update our copies before we save * it into the save area */ - save_user_fpu_regs(); - vcpu->run->s.regs.fpc = current->thread.ufpu.fpc; + fpu_stfpc(&vcpu->run->s.regs.fpc); + if (cpu_has_vx()) + save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); + else + save_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); save_access_regs(vcpu->run->s.regs.acrs); return kvm_s390_store_status_unloaded(vcpu, addr); diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index e0f79c9a4852..3ec11612805d 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "kvm-s390.h" #include "gaccess.h" @@ -1149,8 +1148,6 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) */ vcpu->arch.sie_block->prog0c |= PROG_IN_SIE; barrier(); - if (test_thread_flag(TIF_FPU)) - load_user_fpu_regs(); if (!kvm_s390_vcpu_sie_inhibited(vcpu)) rc = sie64a(scb_s, vcpu->run->s.regs.gprs); barrier(); From 9cbff7f2214d16af5c10f1f55ac72d4c1a8bd787 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:14 +0100 Subject: [PATCH 068/117] s390/fpu: remove regs member from struct fpu KVM was the only user which modified the regs pointer in struct fpu. Remove the pointer and convert the rest of the core fpu code to directly access the save area embedded within struct fpu. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-types.h | 1 - arch/s390/include/asm/processor.h | 1 - arch/s390/kernel/fpu.c | 11 ++++------- arch/s390/kernel/process.c | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h index fee4468a85d3..f5b6fab30401 100644 --- a/arch/s390/include/asm/fpu-types.h +++ b/arch/s390/include/asm/fpu-types.h @@ -13,7 +13,6 @@ struct fpu { __u32 fpc; /* Floating-point control */ - void *regs; /* Pointer to the current save area */ union { /* Floating-point register save area */ freg_t fprs[__NUM_FPRS]; diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index eee0a1eec620..ecce58abf3db 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -202,7 +202,6 @@ typedef struct thread_struct thread_struct; #define INIT_THREAD { \ .ksp = sizeof(init_stack) + (unsigned long) &init_stack, \ - .ufpu.regs = (void *)init_task.thread.ufpu.fprs, \ .last_break = 1, \ } diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index b976da5bf71b..a0ef3fc5d90f 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -110,13 +110,12 @@ EXPORT_SYMBOL(__kernel_fpu_end); void __load_user_fpu_regs(void) { struct fpu *state = ¤t->thread.ufpu; - void *regs = current->thread.ufpu.regs; fpu_lfpc_safe(&state->fpc); if (likely(cpu_has_vx())) - load_vx_regs(regs); + load_vx_regs(state->vxrs); else - load_fp_regs(regs); + load_fp_regs(state->fprs); clear_thread_flag(TIF_FPU); } @@ -132,7 +131,6 @@ void save_user_fpu_regs(void) { unsigned long flags; struct fpu *state; - void *regs; local_irq_save(flags); @@ -140,13 +138,12 @@ void save_user_fpu_regs(void) goto out; state = ¤t->thread.ufpu; - regs = current->thread.ufpu.regs; fpu_stfpc(&state->fpc); if (likely(cpu_has_vx())) - save_vx_regs(regs); + save_vx_regs(state->vxrs); else - save_fp_regs(regs); + save_fp_regs(state->fprs); set_thread_flag(TIF_FPU); out: local_irq_restore(flags); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index b7b623818753..dd456b475861 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -94,7 +94,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) save_user_fpu_regs(); *dst = *src; - dst->thread.ufpu.regs = dst->thread.ufpu.fprs; dst->thread.kfpu_flags = 0; /* From bdbd3acb33f5b09b99d75b0f0edeb7db98a05c89 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:15 +0100 Subject: [PATCH 069/117] s390/fpu: remove anonymous union from struct fpu The anonymous union within struct fpu contains a floating point register array and a vector register array. Given that the vector register is always present remove the floating point register array. For configurations without vector registers save the floating point register contents within their corresponding vector register location. This allows to remove the union, and also to simplify ptrace and perf code. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-types.h | 14 +--- arch/s390/include/asm/fpu.h | 102 ++++++++++++++++++------------ arch/s390/kernel/fpu.c | 8 +-- arch/s390/kernel/perf_regs.c | 5 +- arch/s390/kernel/ptrace.c | 58 +++-------------- 5 files changed, 78 insertions(+), 109 deletions(-) diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h index f5b6fab30401..8e6927c23bdc 100644 --- a/arch/s390/include/asm/fpu-types.h +++ b/arch/s390/include/asm/fpu-types.h @@ -12,23 +12,15 @@ #include struct fpu { - __u32 fpc; /* Floating-point control */ - union { - /* Floating-point register save area */ - freg_t fprs[__NUM_FPRS]; - /* Vector register save area */ - __vector128 vxrs[__NUM_VXRS]; - }; + u32 fpc; + __vector128 vxrs[__NUM_VXRS] __aligned(8); }; /* In-kernel FPU state structure */ struct kernel_fpu { int mask; u32 fpc; - union { - freg_t fprs[__NUM_FPRS]; - __vector128 vxrs[__NUM_VXRS]; - }; + __vector128 vxrs[__NUM_VXRS] __aligned(8); }; #define DECLARE_KERNEL_FPU_ONSTACK(name) \ diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index 4300eef243f9..e706af26c5d0 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -98,44 +98,68 @@ static __always_inline void load_vx_regs(__vector128 *vxrs) fpu_vlm(16, 31, &vxrs[16]); } +static __always_inline void __save_fp_regs(freg_t *fprs, unsigned int offset) +{ + fpu_std(0, &fprs[0 * offset]); + fpu_std(1, &fprs[1 * offset]); + fpu_std(2, &fprs[2 * offset]); + fpu_std(3, &fprs[3 * offset]); + fpu_std(4, &fprs[4 * offset]); + fpu_std(5, &fprs[5 * offset]); + fpu_std(6, &fprs[6 * offset]); + fpu_std(7, &fprs[7 * offset]); + fpu_std(8, &fprs[8 * offset]); + fpu_std(9, &fprs[9 * offset]); + fpu_std(10, &fprs[10 * offset]); + fpu_std(11, &fprs[11 * offset]); + fpu_std(12, &fprs[12 * offset]); + fpu_std(13, &fprs[13 * offset]); + fpu_std(14, &fprs[14 * offset]); + fpu_std(15, &fprs[15 * offset]); +} + +static __always_inline void __load_fp_regs(freg_t *fprs, unsigned int offset) +{ + fpu_ld(0, &fprs[0 * offset]); + fpu_ld(1, &fprs[1 * offset]); + fpu_ld(2, &fprs[2 * offset]); + fpu_ld(3, &fprs[3 * offset]); + fpu_ld(4, &fprs[4 * offset]); + fpu_ld(5, &fprs[5 * offset]); + fpu_ld(6, &fprs[6 * offset]); + fpu_ld(7, &fprs[7 * offset]); + fpu_ld(8, &fprs[8 * offset]); + fpu_ld(9, &fprs[9 * offset]); + fpu_ld(10, &fprs[10 * offset]); + fpu_ld(11, &fprs[11 * offset]); + fpu_ld(12, &fprs[12 * offset]); + fpu_ld(13, &fprs[13 * offset]); + fpu_ld(14, &fprs[14 * offset]); + fpu_ld(15, &fprs[15 * offset]); +} + static __always_inline void save_fp_regs(freg_t *fprs) { - fpu_std(0, &fprs[0]); - fpu_std(1, &fprs[1]); - fpu_std(2, &fprs[2]); - fpu_std(3, &fprs[3]); - fpu_std(4, &fprs[4]); - fpu_std(5, &fprs[5]); - fpu_std(6, &fprs[6]); - fpu_std(7, &fprs[7]); - fpu_std(8, &fprs[8]); - fpu_std(9, &fprs[9]); - fpu_std(10, &fprs[10]); - fpu_std(11, &fprs[11]); - fpu_std(12, &fprs[12]); - fpu_std(13, &fprs[13]); - fpu_std(14, &fprs[14]); - fpu_std(15, &fprs[15]); + __save_fp_regs(fprs, sizeof(freg_t) / sizeof(freg_t)); } static __always_inline void load_fp_regs(freg_t *fprs) { - fpu_ld(0, &fprs[0]); - fpu_ld(1, &fprs[1]); - fpu_ld(2, &fprs[2]); - fpu_ld(3, &fprs[3]); - fpu_ld(4, &fprs[4]); - fpu_ld(5, &fprs[5]); - fpu_ld(6, &fprs[6]); - fpu_ld(7, &fprs[7]); - fpu_ld(8, &fprs[8]); - fpu_ld(9, &fprs[9]); - fpu_ld(10, &fprs[10]); - fpu_ld(11, &fprs[11]); - fpu_ld(12, &fprs[12]); - fpu_ld(13, &fprs[13]); - fpu_ld(14, &fprs[14]); - fpu_ld(15, &fprs[15]); + __load_fp_regs(fprs, sizeof(freg_t) / sizeof(freg_t)); +} + +static __always_inline void save_fp_regs_vx(__vector128 *vxrs) +{ + freg_t *fprs = (freg_t *)&vxrs[0].high; + + __save_fp_regs(fprs, sizeof(__vector128) / sizeof(freg_t)); +} + +static __always_inline void load_fp_regs_vx(__vector128 *vxrs) +{ + freg_t *fprs = (freg_t *)&vxrs[0].high; + + __load_fp_regs(fprs, sizeof(__vector128) / sizeof(freg_t)); } static inline void kernel_fpu_begin(struct kernel_fpu *state, int flags) @@ -170,7 +194,7 @@ static inline void save_kernel_fpu_regs(struct thread_struct *thread) if (likely(cpu_has_vx())) save_vx_regs(state->vxrs); else - save_fp_regs(state->fprs); + save_fp_regs_vx(state->vxrs); } static inline void restore_kernel_fpu_regs(struct thread_struct *thread) @@ -183,7 +207,7 @@ static inline void restore_kernel_fpu_regs(struct thread_struct *thread) if (likely(cpu_has_vx())) load_vx_regs(state->vxrs); else - load_fp_regs(state->fprs); + load_fp_regs_vx(state->vxrs); } static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) @@ -206,19 +230,13 @@ static inline void fpregs_store(_s390_fp_regs *fpregs, struct fpu *fpu) { fpregs->pad = 0; fpregs->fpc = fpu->fpc; - if (cpu_has_vx()) - convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); - else - memcpy((freg_t *)&fpregs->fprs, fpu->fprs, sizeof(fpregs->fprs)); + convert_vx_to_fp((freg_t *)&fpregs->fprs, fpu->vxrs); } static inline void fpregs_load(_s390_fp_regs *fpregs, struct fpu *fpu) { fpu->fpc = fpregs->fpc; - if (cpu_has_vx()) - convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); - else - memcpy(fpu->fprs, (freg_t *)&fpregs->fprs, sizeof(fpregs->fprs)); + convert_fp_to_vx(fpu->vxrs, (freg_t *)&fpregs->fprs); } #endif /* _ASM_S390_FPU_H */ diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index a0ef3fc5d90f..733e188951b7 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -24,7 +24,7 @@ void __kernel_fpu_begin(struct kernel_fpu *state, int flags) fpu_stfpc(&state->fpc); if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) - save_fp_regs(state->fprs); + save_fp_regs_vx(state->vxrs); return; } mask = flags & KERNEL_VXR; @@ -73,7 +73,7 @@ void __kernel_fpu_end(struct kernel_fpu *state, int flags) fpu_lfpc(&state->fpc); if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) - load_fp_regs(state->fprs); + load_fp_regs_vx(state->vxrs); return; } mask = flags & KERNEL_VXR; @@ -115,7 +115,7 @@ void __load_user_fpu_regs(void) if (likely(cpu_has_vx())) load_vx_regs(state->vxrs); else - load_fp_regs(state->fprs); + load_fp_regs_vx(state->vxrs); clear_thread_flag(TIF_FPU); } @@ -143,7 +143,7 @@ void save_user_fpu_regs(void) if (likely(cpu_has_vx())) save_vx_regs(state->vxrs); else - save_fp_regs(state->fprs); + save_fp_regs_vx(state->vxrs); set_thread_flag(TIF_FPU); out: local_irq_restore(flags); diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index 511349b8bc5c..a6b058ee4a36 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -19,10 +19,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return 0; idx -= PERF_REG_S390_FP0; - if (cpu_has_vx()) - fp = *(freg_t *)(current->thread.ufpu.vxrs + idx); - else - fp = current->thread.ufpu.fprs[idx]; + fp = *(freg_t *)(current->thread.ufpu.vxrs + idx); return fp.ui; } diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index f1ca79073173..1cfed8b710b8 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -252,17 +252,10 @@ static unsigned long __peek_user(struct task_struct *child, addr_t addr) } else if (addr < offsetof(struct user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.ufpu - * or the child->thread.ufpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct user, regs.fp_regs.fprs); - if (cpu_has_vx()) - tmp = *(addr_t *) - ((addr_t)child->thread.ufpu.vxrs + 2 * offset); - else - tmp = *(addr_t *) - ((addr_t)child->thread.ufpu.fprs + offset); - + tmp = *(addr_t *)((addr_t)child->thread.ufpu.vxrs + 2 * offset); } else if (addr < offsetof(struct user, regs.per_info) + sizeof(per_struct)) { /* * Handle access to the per_info structure. @@ -400,17 +393,10 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) } else if (addr < offsetof(struct user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.ufpu - * or the child->thread.ufpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct user, regs.fp_regs.fprs); - if (cpu_has_vx()) - *(addr_t *)((addr_t) - child->thread.ufpu.vxrs + 2 * offset) = data; - else - *(addr_t *)((addr_t) - child->thread.ufpu.fprs + offset) = data; - + *(addr_t *)((addr_t)child->thread.ufpu.vxrs + 2 * offset) = data; } else if (addr < offsetof(struct user, regs.per_info) + sizeof(per_struct)) { /* * Handle access to the per_info structure. @@ -627,17 +613,10 @@ static u32 __peek_user_compat(struct task_struct *child, addr_t addr) } else if (addr < offsetof(struct compat_user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.ufpu - * or the child->thread.ufpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct compat_user, regs.fp_regs.fprs); - if (cpu_has_vx()) - tmp = *(__u32 *) - ((addr_t)child->thread.ufpu.vxrs + 2 * offset); - else - tmp = *(__u32 *) - ((addr_t)child->thread.ufpu.fprs + offset); - + tmp = *(__u32 *)((addr_t)child->thread.ufpu.vxrs + 2 * offset); } else if (addr < offsetof(struct compat_user, regs.per_info) + sizeof(struct compat_per_struct_kernel)) { /* * Handle access to the per_info structure. @@ -753,17 +732,10 @@ static int __poke_user_compat(struct task_struct *child, } else if (addr < offsetof(struct compat_user, regs.fp_regs) + sizeof(s390_fp_regs)) { /* - * floating point regs. are either in child->thread.ufpu - * or the child->thread.ufpu.vxrs array + * floating point regs. are in the child->thread.ufpu.vxrs array */ offset = addr - offsetof(struct compat_user, regs.fp_regs.fprs); - if (cpu_has_vx()) - *(__u32 *)((addr_t) - child->thread.ufpu.vxrs + 2 * offset) = tmp; - else - *(__u32 *)((addr_t) - child->thread.ufpu.fprs + offset) = tmp; - + *(__u32 *)((addr_t)child->thread.ufpu.vxrs + 2 * offset) = tmp; } else if (addr < offsetof(struct compat_user, regs.per_info) + sizeof(struct compat_per_struct_kernel)) { /* * Handle access to the per_info structure. @@ -912,12 +884,7 @@ static int s390_fpregs_set(struct task_struct *target, if (target == current) save_user_fpu_regs(); - - if (cpu_has_vx()) - convert_vx_to_fp(fprs, target->thread.ufpu.vxrs); - else - memcpy(&fprs, target->thread.ufpu.fprs, sizeof(fprs)); - + convert_vx_to_fp(fprs, target->thread.ufpu.vxrs); if (count > 0 && pos < offsetof(s390_fp_regs, fprs)) { u32 ufpc[2] = { target->thread.ufpu.fpc, 0 }; rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ufpc, @@ -934,12 +901,7 @@ static int s390_fpregs_set(struct task_struct *target, fprs, offsetof(s390_fp_regs, fprs), -1); if (rc) return rc; - - if (cpu_has_vx()) - convert_fp_to_vx(target->thread.ufpu.vxrs, fprs); - else - memcpy(target->thread.ufpu.fprs, &fprs, sizeof(fprs)); - + convert_fp_to_vx(target->thread.ufpu.vxrs, fprs); return rc; } From cad8c3abaac3848f46ba9d1b21f79fab87098da2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:16 +0100 Subject: [PATCH 070/117] s390/fpu: let fpu_vlm() and fpu_vstm() return number of registers Let fpu_vlm() and fpu_vstm() macros return the number of registers saved / loaded. This is helpful to read easy to read code in case there are several subsequent fpu_vlm() or fpu_vstm() calls: __vector128 *vxrs = .... vxrs += fpu_vstm(0, 15, vxrs); vxrs += fpu_vstm(16, 31, vxrs); Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 538201864a47..803ce4e2aab4 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -110,7 +110,8 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) #ifdef CONFIG_CC_IS_CLANG -#define fpu_vlm(_v1, _v3, _vxrs) do { \ +#define fpu_vlm(_v1, _v3, _vxrs) \ +({ \ unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ struct { \ __vector128 _v[(_v3) - (_v1) + 1]; \ @@ -124,11 +125,13 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : [vxrs] "R" (*_v), \ [v1] "I" (_v1), [v3] "I" (_v3) \ : "memory", "1"); \ -} while (0) + (_v3) - (_v1) + 1; \ +}) #else /* CONFIG_CC_IS_CLANG */ -#define fpu_vlm(_v1, _v3, _vxrs) do { \ +#define fpu_vlm(_v1, _v3, _vxrs) \ +({ \ unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ struct { \ __vector128 _v[(_v3) - (_v1) + 1]; \ @@ -140,13 +143,15 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : [vxrs] "Q" (*_v), \ [v1] "I" (_v1), [v3] "I" (_v3) \ : "memory"); \ -} while (0) + (_v3) - (_v1) + 1; \ +}) #endif /* CONFIG_CC_IS_CLANG */ #ifdef CONFIG_CC_IS_CLANG -#define fpu_vstm(_v1, _v3, _vxrs) do { \ +#define fpu_vstm(_v1, _v3, _vxrs) \ +({ \ unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ struct { \ __vector128 _v[(_v3) - (_v1) + 1]; \ @@ -159,11 +164,13 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : [vxrs] "=R" (*_v) \ : [v1] "I" (_v1), [v3] "I" (_v3) \ : "memory", "1"); \ -} while (0) + (_v3) - (_v1) + 1; \ +}) #else /* CONFIG_CC_IS_CLANG */ -#define fpu_vstm(_v1, _v3, _vxrs) do { \ +#define fpu_vstm(_v1, _v3, _vxrs) \ +({ \ unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ struct { \ __vector128 _v[(_v3) - (_v1) + 1]; \ @@ -174,7 +181,8 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : [vxrs] "=Q" (*_v) \ : [v1] "I" (_v1), [v3] "I" (_v3) \ : "memory"); \ -} while (0) + (_v3) - (_v1) + 1; \ +}) #endif /* CONFIG_CC_IS_CLANG */ From 066c40918bb495de8f2e45bd7eec06737a142712 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:17 +0100 Subject: [PATCH 071/117] s390/fpu: decrease stack usage for some cases The kernel_fpu structure has a quite large size of 520 bytes. In order to reduce stack footprint introduce several kernel fpu structures with different and also smaller sizes. This way every kernel fpu user must use the correct variant. A compile time check verifies that the correct variant is used. There are several users which use only 16 instead of all 32 vector registers. For those users the new kernel_fpu_16 structure with a size of only 266 bytes can be used. Signed-off-by: Heiko Carstens --- arch/s390/crypto/chacha-glue.c | 2 +- arch/s390/crypto/crc32-vx.c | 2 +- arch/s390/include/asm/fpu-types.h | 32 ++++++++++++++++----- arch/s390/include/asm/fpu.h | 48 +++++++++++++++++++++++++++---- arch/s390/kernel/fpu.c | 48 +++++++++++++++---------------- arch/s390/kernel/sysinfo.c | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- lib/raid6/s390vx.uc | 4 +-- 8 files changed, 97 insertions(+), 43 deletions(-) diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index 97098add2079..f8b0c52e77a4 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -22,7 +22,7 @@ static void chacha20_crypt_s390(u32 *state, u8 *dst, const u8 *src, unsigned int nbytes, const u32 *key, u32 *counter) { - DECLARE_KERNEL_FPU_ONSTACK(vxstate); + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); kernel_fpu_begin(&vxstate, KERNEL_VXR); chacha20_vx(dst, src, nbytes, key, counter); diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index dc2997f18e30..d9f1fdb66691 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -50,7 +50,7 @@ u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); unsigned char const *data, size_t datalen) \ { \ unsigned long prealign, aligned, remaining; \ - DECLARE_KERNEL_FPU_ONSTACK(vxstate); \ + DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ \ if (datalen < VX_MIN_LEN + VX_ALIGN_MASK) \ return ___crc32_sw(crc, data, datalen); \ diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h index 8e6927c23bdc..04c32b9fc849 100644 --- a/arch/s390/include/asm/fpu-types.h +++ b/arch/s390/include/asm/fpu-types.h @@ -16,14 +16,32 @@ struct fpu { __vector128 vxrs[__NUM_VXRS] __aligned(8); }; -/* In-kernel FPU state structure */ -struct kernel_fpu { - int mask; - u32 fpc; - __vector128 vxrs[__NUM_VXRS] __aligned(8); +struct kernel_fpu_hdr { + int mask; + u32 fpc; }; -#define DECLARE_KERNEL_FPU_ONSTACK(name) \ - struct kernel_fpu name __uninitialized +struct kernel_fpu { + struct kernel_fpu_hdr hdr; + __vector128 vxrs[] __aligned(8); +}; + +#define KERNEL_FPU_STRUCT(vxr_size) \ +struct kernel_fpu_##vxr_size { \ + struct kernel_fpu_hdr hdr; \ + __vector128 vxrs[vxr_size] __aligned(8); \ +} + +KERNEL_FPU_STRUCT(16); +KERNEL_FPU_STRUCT(32); + +#define DECLARE_KERNEL_FPU_ONSTACK(vxr_size, name) \ + struct kernel_fpu_##vxr_size name __uninitialized + +#define DECLARE_KERNEL_FPU_ONSTACK16(name) \ + DECLARE_KERNEL_FPU_ONSTACK(16, name) + +#define DECLARE_KERNEL_FPU_ONSTACK32(name) \ + DECLARE_KERNEL_FPU_ONSTACK(32, name) #endif /* _ASM_S390_FPU_TYPES_H */ diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index e706af26c5d0..c1b3920092a1 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -162,28 +162,64 @@ static __always_inline void load_fp_regs_vx(__vector128 *vxrs) __load_fp_regs(fprs, sizeof(__vector128) / sizeof(freg_t)); } -static inline void kernel_fpu_begin(struct kernel_fpu *state, int flags) +static inline void _kernel_fpu_begin(struct kernel_fpu *state, int flags) { - state->mask = READ_ONCE(current->thread.kfpu_flags); + state->hdr.mask = READ_ONCE(current->thread.kfpu_flags); if (!test_thread_flag(TIF_FPU)) { /* Save user space FPU state and register contents */ save_user_fpu_regs(); - } else if (state->mask & flags) { + } else if (state->hdr.mask & flags) { /* Save FPU/vector register in-use by the kernel */ __kernel_fpu_begin(state, flags); } __atomic_or(flags, ¤t->thread.kfpu_flags); } -static inline void kernel_fpu_end(struct kernel_fpu *state, int flags) +static inline void _kernel_fpu_end(struct kernel_fpu *state, int flags) { - WRITE_ONCE(current->thread.kfpu_flags, state->mask); - if (state->mask & flags) { + WRITE_ONCE(current->thread.kfpu_flags, state->hdr.mask); + if (state->hdr.mask & flags) { /* Restore FPU/vector register in-use by the kernel */ __kernel_fpu_end(state, flags); } } +void __kernel_fpu_invalid_size(void); + +static __always_inline void kernel_fpu_check_size(int flags, unsigned int size) +{ + unsigned int cnt = 0; + + if (flags & KERNEL_VXR_V0V7) + cnt += 8; + if (flags & KERNEL_VXR_V8V15) + cnt += 8; + if (flags & KERNEL_VXR_V16V23) + cnt += 8; + if (flags & KERNEL_VXR_V24V31) + cnt += 8; + if (cnt != size) + __kernel_fpu_invalid_size(); +} + +#define kernel_fpu_begin(state, flags) \ +{ \ + typeof(state) s = (state); \ + int _flags = (flags); \ + \ + kernel_fpu_check_size(_flags, ARRAY_SIZE(s->vxrs)); \ + _kernel_fpu_begin((struct kernel_fpu *)s, _flags); \ +} + +#define kernel_fpu_end(state, flags) \ +{ \ + typeof(state) s = (state); \ + int _flags = (flags); \ + \ + kernel_fpu_check_size(_flags, ARRAY_SIZE(s->vxrs)); \ + _kernel_fpu_end((struct kernel_fpu *)s, _flags); \ +} + static inline void save_kernel_fpu_regs(struct thread_struct *thread) { struct fpu *state = &thread->kfpu; diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 733e188951b7..62e9befe7890 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -19,41 +19,41 @@ void __kernel_fpu_begin(struct kernel_fpu *state, int flags) * Limit the save to the FPU/vector registers already * in use by the previous context. */ - flags &= state->mask; + flags &= state->hdr.mask; if (flags & KERNEL_FPC) - fpu_stfpc(&state->fpc); + fpu_stfpc(&state->hdr.fpc); if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) - save_fp_regs_vx(state->vxrs); + save_fp_regs_vx(vxrs); return; } mask = flags & KERNEL_VXR; if (mask == KERNEL_VXR) { - fpu_vstm(0, 15, &vxrs[0]); - fpu_vstm(16, 31, &vxrs[16]); + vxrs += fpu_vstm(0, 15, vxrs); + vxrs += fpu_vstm(16, 31, vxrs); return; } if (mask == KERNEL_VXR_MID) { - fpu_vstm(8, 23, &vxrs[8]); + vxrs += fpu_vstm(8, 23, vxrs); return; } mask = flags & KERNEL_VXR_LOW; if (mask) { if (mask == KERNEL_VXR_LOW) - fpu_vstm(0, 15, &vxrs[0]); + vxrs += fpu_vstm(0, 15, vxrs); else if (mask == KERNEL_VXR_V0V7) - fpu_vstm(0, 7, &vxrs[0]); + vxrs += fpu_vstm(0, 7, vxrs); else - fpu_vstm(8, 15, &vxrs[8]); + vxrs += fpu_vstm(8, 15, vxrs); } mask = flags & KERNEL_VXR_HIGH; if (mask) { if (mask == KERNEL_VXR_HIGH) - fpu_vstm(16, 31, &vxrs[16]); + vxrs += fpu_vstm(16, 31, vxrs); else if (mask == KERNEL_VXR_V16V23) - fpu_vstm(16, 23, &vxrs[16]); + vxrs += fpu_vstm(16, 23, vxrs); else - fpu_vstm(24, 31, &vxrs[24]); + vxrs += fpu_vstm(24, 31, vxrs); } } EXPORT_SYMBOL(__kernel_fpu_begin); @@ -68,41 +68,41 @@ void __kernel_fpu_end(struct kernel_fpu *state, int flags) * previous context that have been overwritten by the * current context. */ - flags &= state->mask; + flags &= state->hdr.mask; if (flags & KERNEL_FPC) - fpu_lfpc(&state->fpc); + fpu_lfpc(&state->hdr.fpc); if (!cpu_has_vx()) { if (flags & KERNEL_VXR_LOW) - load_fp_regs_vx(state->vxrs); + load_fp_regs_vx(vxrs); return; } mask = flags & KERNEL_VXR; if (mask == KERNEL_VXR) { - fpu_vlm(0, 15, &vxrs[0]); - fpu_vlm(16, 31, &vxrs[16]); + vxrs += fpu_vlm(0, 15, vxrs); + vxrs += fpu_vlm(16, 31, vxrs); return; } if (mask == KERNEL_VXR_MID) { - fpu_vlm(8, 23, &vxrs[8]); + vxrs += fpu_vlm(8, 23, vxrs); return; } mask = flags & KERNEL_VXR_LOW; if (mask) { if (mask == KERNEL_VXR_LOW) - fpu_vlm(0, 15, &vxrs[0]); + vxrs += fpu_vlm(0, 15, vxrs); else if (mask == KERNEL_VXR_V0V7) - fpu_vlm(0, 7, &vxrs[0]); + vxrs += fpu_vlm(0, 7, vxrs); else - fpu_vlm(8, 15, &vxrs[8]); + vxrs += fpu_vlm(8, 15, vxrs); } mask = flags & KERNEL_VXR_HIGH; if (mask) { if (mask == KERNEL_VXR_HIGH) - fpu_vlm(16, 31, &vxrs[16]); + vxrs += fpu_vlm(16, 31, vxrs); else if (mask == KERNEL_VXR_V16V23) - fpu_vlm(16, 23, &vxrs[16]); + vxrs += fpu_vlm(16, 23, vxrs); else - fpu_vlm(24, 31, &vxrs[24]); + vxrs += fpu_vlm(24, 31, vxrs); } } EXPORT_SYMBOL(__kernel_fpu_end); diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 061d45cf0261..4cd6428bfab2 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -426,7 +426,7 @@ subsys_initcall(create_proc_service_level); */ void s390_adjust_jiffies(void) { - DECLARE_KERNEL_FPU_ONSTACK(fpu); + DECLARE_KERNEL_FPU_ONSTACK16(fpu); struct sysinfo_1_2_2 *info; unsigned long capability; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 8467945344b5..8c222b0dfbf2 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -5026,7 +5026,7 @@ static void store_regs(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { struct kvm_run *kvm_run = vcpu->run; - DECLARE_KERNEL_FPU_ONSTACK(fpu); + DECLARE_KERNEL_FPU_ONSTACK32(fpu); int rc; /* diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc index bc2f4fbe5a82..92c05b7596bc 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid6/s390vx.uc @@ -80,7 +80,7 @@ static inline void COPY_VEC(int x, int y) static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) { - DECLARE_KERNEL_FPU_ONSTACK(vxstate); + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); u8 **dptr, *p, *q; int d, z, z0; @@ -113,7 +113,7 @@ static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, size_t bytes, void **ptrs) { - DECLARE_KERNEL_FPU_ONSTACK(vxstate); + DECLARE_KERNEL_FPU_ONSTACK32(vxstate); u8 **dptr, *p, *q; int d, z, z0; From 8c09871a950a3fe686e0e27fd4193179c5f74f37 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:18 +0100 Subject: [PATCH 072/117] s390/fpu: limit save and restore to used registers The first invocation of kernel_fpu_begin() after switching from user to kernel context will save all vector registers, even if only parts of the vector registers are used within the kernel fpu context. Given that save and restore of all vector registers is quite expensive change the current approach in several ways: - Instead of saving and restoring all user registers limit this to those registers which are actually used within an kernel fpu context. - On context switch save all remaining user fpu registers, so they can be restored when the task is rescheduled. - Saving user registers within kernel_fpu_begin() is done without disabling and enabling interrupts - which also slightly reduces runtime. In worst case (e.g. interrupt context uses the same registers) this may lead to the situation that registers are saved several times, however the assumption is that this will not happen frequently, so that the new method is faster in nearly all cases. - save_user_fpu_regs() can still be called from all contexts and saves all (or all remaining) user registers to a tasks ufpu user fpu save area. Overall this reduces the time required to save and restore the user fpu context for nearly all cases. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/entry-common.h | 3 +- arch/s390/include/asm/fpu.h | 83 +++++++++++-------- arch/s390/include/asm/processor.h | 1 + arch/s390/kernel/fpu.c | 114 ++++++++++++++++++--------- 4 files changed, 130 insertions(+), 71 deletions(-) diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h index 659e07d7f31a..7f5004065e8a 100644 --- a/arch/s390/include/asm/entry-common.h +++ b/arch/s390/include/asm/entry-common.h @@ -41,8 +41,7 @@ static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs, static __always_inline void arch_exit_to_user_mode(void) { - if (test_thread_flag(TIF_FPU)) - __load_user_fpu_regs(); + load_user_fpu_regs(); if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) debug_user_asce(1); diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h index c1b3920092a1..c84cb33913e2 100644 --- a/arch/s390/include/asm/fpu.h +++ b/arch/s390/include/asm/fpu.h @@ -58,10 +58,6 @@ static inline bool cpu_has_vx(void) return likely(test_facility(129)); } -void save_user_fpu_regs(void); -void load_user_fpu_regs(void); -void __load_user_fpu_regs(void); - enum { KERNEL_FPC_BIT = 0, KERNEL_VXR_V0V7_BIT, @@ -83,6 +79,8 @@ enum { #define KERNEL_VXR (KERNEL_VXR_LOW | KERNEL_VXR_HIGH) #define KERNEL_FPR (KERNEL_FPC | KERNEL_VXR_LOW) +void load_fpu_state(struct fpu *state, int flags); +void save_fpu_state(struct fpu *state, int flags); void __kernel_fpu_begin(struct kernel_fpu *state, int flags); void __kernel_fpu_end(struct kernel_fpu *state, int flags); @@ -162,26 +160,57 @@ static __always_inline void load_fp_regs_vx(__vector128 *vxrs) __load_fp_regs(fprs, sizeof(__vector128) / sizeof(freg_t)); } -static inline void _kernel_fpu_begin(struct kernel_fpu *state, int flags) +static inline void load_user_fpu_regs(void) { - state->hdr.mask = READ_ONCE(current->thread.kfpu_flags); - if (!test_thread_flag(TIF_FPU)) { - /* Save user space FPU state and register contents */ - save_user_fpu_regs(); - } else if (state->hdr.mask & flags) { - /* Save FPU/vector register in-use by the kernel */ - __kernel_fpu_begin(state, flags); - } - __atomic_or(flags, ¤t->thread.kfpu_flags); + struct thread_struct *thread = ¤t->thread; + + if (!thread->ufpu_flags) + return; + load_fpu_state(&thread->ufpu, thread->ufpu_flags); + thread->ufpu_flags = 0; } -static inline void _kernel_fpu_end(struct kernel_fpu *state, int flags) +static __always_inline void __save_user_fpu_regs(struct thread_struct *thread, int flags) { - WRITE_ONCE(current->thread.kfpu_flags, state->hdr.mask); - if (state->hdr.mask & flags) { - /* Restore FPU/vector register in-use by the kernel */ + save_fpu_state(&thread->ufpu, flags); + __atomic_or(flags, &thread->ufpu_flags); +} + +static inline void save_user_fpu_regs(void) +{ + struct thread_struct *thread = ¤t->thread; + int mask, flags; + + mask = __atomic_or(KERNEL_FPC | KERNEL_VXR, &thread->kfpu_flags); + flags = ~READ_ONCE(thread->ufpu_flags) & (KERNEL_FPC | KERNEL_VXR); + if (flags) + __save_user_fpu_regs(thread, flags); + barrier(); + WRITE_ONCE(thread->kfpu_flags, mask); +} + +static __always_inline void _kernel_fpu_begin(struct kernel_fpu *state, int flags) +{ + struct thread_struct *thread = ¤t->thread; + int mask, uflags; + + mask = __atomic_or(flags, &thread->kfpu_flags); + state->hdr.mask = mask; + uflags = READ_ONCE(thread->ufpu_flags); + if ((uflags & flags) != flags) + __save_user_fpu_regs(thread, ~uflags & flags); + if (mask & flags) + __kernel_fpu_begin(state, flags); +} + +static __always_inline void _kernel_fpu_end(struct kernel_fpu *state, int flags) +{ + int mask = state->hdr.mask; + + if (mask & flags) __kernel_fpu_end(state, flags); - } + barrier(); + WRITE_ONCE(current->thread.kfpu_flags, mask); } void __kernel_fpu_invalid_size(void); @@ -222,28 +251,16 @@ static __always_inline void kernel_fpu_check_size(int flags, unsigned int size) static inline void save_kernel_fpu_regs(struct thread_struct *thread) { - struct fpu *state = &thread->kfpu; - if (!thread->kfpu_flags) return; - fpu_stfpc(&state->fpc); - if (likely(cpu_has_vx())) - save_vx_regs(state->vxrs); - else - save_fp_regs_vx(state->vxrs); + save_fpu_state(&thread->kfpu, thread->kfpu_flags); } static inline void restore_kernel_fpu_regs(struct thread_struct *thread) { - struct fpu *state = &thread->kfpu; - if (!thread->kfpu_flags) return; - fpu_lfpc(&state->fpc); - if (likely(cpu_has_vx())) - load_vx_regs(state->vxrs); - else - load_fp_regs_vx(state->vxrs); + load_fpu_state(&thread->kfpu, thread->kfpu_flags); } static inline void convert_vx_to_fp(freg_t *fprs, __vector128 *vxrs) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index ecce58abf3db..7cf00cf8fb0b 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -166,6 +166,7 @@ struct thread_struct { unsigned int gmap_write_flag; /* gmap fault write indication */ unsigned int gmap_int_code; /* int code of last gmap fault */ unsigned int gmap_pfault; /* signal of a pending guest pfault */ + int ufpu_flags; /* user fpu flags */ int kfpu_flags; /* kernel fpu flags */ /* Per-thread information related to debugging */ diff --git a/arch/s390/kernel/fpu.c b/arch/s390/kernel/fpu.c index 62e9befe7890..fa90bbdc5ef9 100644 --- a/arch/s390/kernel/fpu.c +++ b/arch/s390/kernel/fpu.c @@ -107,45 +107,87 @@ void __kernel_fpu_end(struct kernel_fpu *state, int flags) } EXPORT_SYMBOL(__kernel_fpu_end); -void __load_user_fpu_regs(void) +void load_fpu_state(struct fpu *state, int flags) { - struct fpu *state = ¤t->thread.ufpu; + __vector128 *vxrs = &state->vxrs[0]; + int mask; - fpu_lfpc_safe(&state->fpc); - if (likely(cpu_has_vx())) - load_vx_regs(state->vxrs); - else - load_fp_regs_vx(state->vxrs); - clear_thread_flag(TIF_FPU); + if (flags & KERNEL_FPC) + fpu_lfpc(&state->fpc); + if (!cpu_has_vx()) { + if (flags & KERNEL_VXR_V0V7) + load_fp_regs_vx(state->vxrs); + return; + } + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + fpu_vlm(0, 15, &vxrs[0]); + fpu_vlm(16, 31, &vxrs[16]); + return; + } + if (mask == KERNEL_VXR_MID) { + fpu_vlm(8, 23, &vxrs[8]); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + fpu_vlm(0, 15, &vxrs[0]); + else if (mask == KERNEL_VXR_V0V7) + fpu_vlm(0, 7, &vxrs[0]); + else + fpu_vlm(8, 15, &vxrs[8]); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + fpu_vlm(16, 31, &vxrs[16]); + else if (mask == KERNEL_VXR_V16V23) + fpu_vlm(16, 23, &vxrs[16]); + else + fpu_vlm(24, 31, &vxrs[24]); + } } -void load_user_fpu_regs(void) +void save_fpu_state(struct fpu *state, int flags) { - raw_local_irq_disable(); - __load_user_fpu_regs(); - raw_local_irq_enable(); + __vector128 *vxrs = &state->vxrs[0]; + int mask; + + if (flags & KERNEL_FPC) + fpu_stfpc(&state->fpc); + if (!cpu_has_vx()) { + if (flags & KERNEL_VXR_LOW) + save_fp_regs_vx(state->vxrs); + return; + } + mask = flags & KERNEL_VXR; + if (mask == KERNEL_VXR) { + fpu_vstm(0, 15, &vxrs[0]); + fpu_vstm(16, 31, &vxrs[16]); + return; + } + if (mask == KERNEL_VXR_MID) { + fpu_vstm(8, 23, &vxrs[8]); + return; + } + mask = flags & KERNEL_VXR_LOW; + if (mask) { + if (mask == KERNEL_VXR_LOW) + fpu_vstm(0, 15, &vxrs[0]); + else if (mask == KERNEL_VXR_V0V7) + fpu_vstm(0, 7, &vxrs[0]); + else + fpu_vstm(8, 15, &vxrs[8]); + } + mask = flags & KERNEL_VXR_HIGH; + if (mask) { + if (mask == KERNEL_VXR_HIGH) + fpu_vstm(16, 31, &vxrs[16]); + else if (mask == KERNEL_VXR_V16V23) + fpu_vstm(16, 23, &vxrs[16]); + else + fpu_vstm(24, 31, &vxrs[24]); + } } -EXPORT_SYMBOL(load_user_fpu_regs); - -void save_user_fpu_regs(void) -{ - unsigned long flags; - struct fpu *state; - - local_irq_save(flags); - - if (test_thread_flag(TIF_FPU)) - goto out; - - state = ¤t->thread.ufpu; - - fpu_stfpc(&state->fpc); - if (likely(cpu_has_vx())) - save_vx_regs(state->vxrs); - else - save_fp_regs_vx(state->vxrs); - set_thread_flag(TIF_FPU); -out: - local_irq_restore(flags); -} -EXPORT_SYMBOL(save_user_fpu_regs); +EXPORT_SYMBOL(save_fpu_state); From 2c6b96762fbd3fd597279950026c6f23ef14acf5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:19 +0100 Subject: [PATCH 073/117] s390/fpu: remove TIF_FPU TIF_FPU is unused - remove it. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/thread_info.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index e3f70b94a79b..a674c7d25da5 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -69,7 +69,6 @@ void arch_setup_new_exec(void); #define TIF_PATCH_PENDING 5 /* pending live patching update */ #define TIF_PGSTE 6 /* New mm's will use 4K page tables */ #define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ -#define TIF_FPU 8 /* restore FPU registers on exit to usermode */ #define TIF_ISOLATE_BP_GUEST 9 /* Run KVM guests with isolated BP */ #define TIF_PER_TRAP 10 /* Need to handle PER trap on exit to usermode */ @@ -93,7 +92,6 @@ void arch_setup_new_exec(void); #define _TIF_UPROBE BIT(TIF_UPROBE) #define _TIF_GUARDED_STORAGE BIT(TIF_GUARDED_STORAGE) #define _TIF_PATCH_PENDING BIT(TIF_PATCH_PENDING) -#define _TIF_FPU BIT(TIF_FPU) #define _TIF_ISOLATE_BP_GUEST BIT(TIF_ISOLATE_BP_GUEST) #define _TIF_PER_TRAP BIT(TIF_PER_TRAP) From 4ce69fcf17d067112f754414aa563e0cd74cc49d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:20 +0100 Subject: [PATCH 074/117] s390/checksum: call instrument_read() instead of kasan_check_read() Call instrument_read() from csum_partial() instead of kasan_check_read(). instrument_read() covers all memory access instrumentation methods. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/checksum.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/checksum.h b/arch/s390/include/asm/checksum.h index 69837eec2ff5..fcef9ae433a7 100644 --- a/arch/s390/include/asm/checksum.h +++ b/arch/s390/include/asm/checksum.h @@ -12,7 +12,7 @@ #ifndef _S390_CHECKSUM_H #define _S390_CHECKSUM_H -#include +#include #include /* @@ -34,7 +34,7 @@ static inline __wsum csum_partial(const void *buff, int len, __wsum sum) .odd = (unsigned long) len, }; - kasan_check_read(buff, len); + instrument_read(buff, len); asm volatile( "0: cksm %[sum],%[rp]\n" " jo 0b\n" From 3a74f44de2c901e1536d227d29257cae1a6ed18f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:21 +0100 Subject: [PATCH 075/117] s390/checksum: provide and use cksm() inline assembly Convert those callers of csum_partial() to use the cksm instruction, which are either very early or in critical paths, like panic/dump, so they don't have to rely on a working kernel infrastructure, which will be introduced with a subsequent patch. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/checksum.h | 27 ++++++++++++++++----------- arch/s390/kernel/ipl.c | 3 +-- arch/s390/kernel/os_info.c | 6 +++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/checksum.h b/arch/s390/include/asm/checksum.h index fcef9ae433a7..414264b3ed6c 100644 --- a/arch/s390/include/asm/checksum.h +++ b/arch/s390/include/asm/checksum.h @@ -15,6 +15,21 @@ #include #include +static inline __wsum cksm(const void *buff, int len, __wsum sum) +{ + union register_pair rp = { + .even = (unsigned long)buff, + .odd = (unsigned long)len, + }; + + instrument_read(buff, len); + asm volatile("\n" + "0: cksm %[sum],%[rp]\n" + " jo 0b\n" + : [sum] "+&d" (sum), [rp] "+&d" (rp.pair) : : "cc", "memory"); + return sum; +} + /* * Computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit). @@ -29,17 +44,7 @@ */ static inline __wsum csum_partial(const void *buff, int len, __wsum sum) { - union register_pair rp = { - .even = (unsigned long) buff, - .odd = (unsigned long) len, - }; - - instrument_read(buff, len); - asm volatile( - "0: cksm %[sum],%[rp]\n" - " jo 0b\n" - : [sum] "+&d" (sum), [rp] "+&d" (rp.pair) : : "cc", "memory"); - return sum; + return cksm(buff, len, sum); } /* diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index ba75f6bee774..1486350a4177 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1941,8 +1941,7 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) reipl_type == IPL_TYPE_UNKNOWN) os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; os_info_entry_add(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); - csum = (__force unsigned int) - csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); + csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); abs_lc = get_abs_lowcore(); abs_lc->ipib = __pa(reipl_block_actual); abs_lc->ipib_checksum = csum; diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 6e1824141b29..a801e6bd5341 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -29,7 +29,7 @@ static struct os_info os_info __page_aligned_data; u32 os_info_csum(struct os_info *os_info) { int size = sizeof(*os_info) - offsetof(struct os_info, version_major); - return (__force u32)csum_partial(&os_info->version_major, size, 0); + return (__force u32)cksm(&os_info->version_major, size, 0); } /* @@ -49,7 +49,7 @@ void os_info_entry_add(int nr, void *ptr, u64 size) { os_info.entry[nr].addr = __pa(ptr); os_info.entry[nr].size = size; - os_info.entry[nr].csum = (__force u32)csum_partial(ptr, size, 0); + os_info.entry[nr].csum = (__force u32)cksm(ptr, size, 0); os_info.csum = os_info_csum(&os_info); } @@ -98,7 +98,7 @@ static void os_info_old_alloc(int nr, int align) msg = "copy failed"; goto fail_free; } - csum = (__force u32)csum_partial(buf_align, size, 0); + csum = (__force u32)cksm(buf_align, size, 0); if (csum != os_info_old->entry[nr].csum) { msg = "checksum failed"; goto fail_free; From cb2a1dd589a0ce97429bf2beeb560e5b030c2ccc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:22 +0100 Subject: [PATCH 076/117] s390/checksum: provide vector register variant of csum_partial() Provide a faster variant of csum_partial() which uses vector registers instead of the cksm instruction. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/checksum.h | 17 +---- arch/s390/include/asm/fpu-insn-asm.h | 19 ++++++ arch/s390/include/asm/fpu-insn.h | 99 ++++++++++++++++++++++++++++ arch/s390/include/asm/fpu-types.h | 4 ++ arch/s390/lib/Makefile | 1 + arch/s390/lib/csum-partial.c | 63 ++++++++++++++++++ 6 files changed, 187 insertions(+), 16 deletions(-) create mode 100644 arch/s390/lib/csum-partial.c diff --git a/arch/s390/include/asm/checksum.h b/arch/s390/include/asm/checksum.h index 414264b3ed6c..00095cc20afa 100644 --- a/arch/s390/include/asm/checksum.h +++ b/arch/s390/include/asm/checksum.h @@ -30,22 +30,7 @@ static inline __wsum cksm(const void *buff, int len, __wsum sum) return sum; } -/* - * Computes the checksum of a memory block at buff, length len, - * and adds in "sum" (32-bit). - * - * Returns a 32-bit number suitable for feeding into itself - * or csum_tcpudp_magic. - * - * This function must be called with even lengths, except - * for the last fragment, which may be odd. - * - * It's best to have buff aligned on a 32-bit boundary. - */ -static inline __wsum csum_partial(const void *buff, int len, __wsum sum) -{ - return cksm(buff, len, sum); -} +__wsum csum_partial(const void *buff, int len, __wsum sum); /* * Fold a partial checksum without adding pseudo headers. diff --git a/arch/s390/include/asm/fpu-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index 789d626599ee..aaf42c513a21 100644 --- a/arch/s390/include/asm/fpu-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -521,6 +521,15 @@ VMRL \vr1, \vr2, \vr3, 3 .endm +/* VECTOR LOAD WITH LENGTH */ +.macro VLL v, gr, disp, base + VX_NUM v1, \v + GR_NUM b2, \base + GR_NUM r3, \gr + .word 0xE700 | ((v1&15) << 4) | r3 + .word (b2 << 12) | (\disp) + MRXBOPC 0, 0x37, v1 +.endm /* Vector integer instructions */ @@ -534,6 +543,16 @@ MRXBOPC 0, 0x68, v1, v2, v3 .endm +/* VECTOR CHECKSUM */ +.macro VCKSM vr1, vr2, vr3 + VX_NUM v1, \vr1 + VX_NUM v2, \vr2 + VX_NUM v3, \vr3 + .word 0xE700 | ((v1&15) << 4) | (v2&15) + .word ((v3&15) << 12) + MRXBOPC 0, 0x66, v1, v2, v3 +.endm + /* VECTOR EXCLUSIVE OR */ .macro VX vr1, vr2, vr3 VX_NUM v1, \vr1 diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 803ce4e2aab4..7e9997fa45d3 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -108,6 +108,89 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : "memory"); } +static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VCKSM %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vl(u8 v1, const void *vxr) +{ + instrument_read(vxr, sizeof(__vector128)); + asm volatile("\n" + " la 1,%[vxr]\n" + " VL %[v1],0,,1\n" + : + : [vxr] "R" (*(__vector128 *)vxr), + [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vl(u8 v1, const void *vxr) +{ + instrument_read(vxr, sizeof(__vector128)); + asm volatile("VL %[v1],%O[vxr],,%R[vxr]\n" + : + : [vxr] "Q" (*(__vector128 *)vxr), + [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +static __always_inline u64 fpu_vlgvf(u8 v, u16 index) +{ + u64 val; + + asm volatile("VLGVF %[val],%[v],%[index]" + : [val] "=d" (val) + : [v] "I" (v), [index] "L" (index) + : "memory"); + return val; +} + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_read(vxr, size); + asm volatile("\n" + " la 1,%[vxr]\n" + " VLL %[v1],%[index],0,1\n" + : + : [vxr] "R" (*(u8 *)vxr), + [index] "d" (index), + [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_read(vxr, size); + asm volatile("VLL %[v1],%[index],%O[vxr],%R[vxr]\n" + : + : [vxr] "Q" (*(u8 *)vxr), + [index] "d" (index), + [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + #ifdef CONFIG_CC_IS_CLANG #define fpu_vlm(_v1, _v3, _vxrs) \ @@ -148,6 +231,14 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) #endif /* CONFIG_CC_IS_CLANG */ +static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index) +{ + asm volatile("VLVGF %[v],%[val],%[index]" + : + : [v] "I" (v), [val] "d" (val), [index] "L" (index) + : "memory"); +} + #ifdef CONFIG_CC_IS_CLANG #define fpu_vstm(_v1, _v3, _vxrs) \ @@ -186,5 +277,13 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) #endif /* CONFIG_CC_IS_CLANG */ +static __always_inline void fpu_vzero(u8 v) +{ + asm volatile("VZERO %[v]" + : + : [v] "I" (v) + : "memory"); +} + #endif /* __ASSEMBLY__ */ #endif /* __ASM_S390_FPU_INSN_H */ diff --git a/arch/s390/include/asm/fpu-types.h b/arch/s390/include/asm/fpu-types.h index 04c32b9fc849..8d58d5a95399 100644 --- a/arch/s390/include/asm/fpu-types.h +++ b/arch/s390/include/asm/fpu-types.h @@ -32,12 +32,16 @@ struct kernel_fpu_##vxr_size { \ __vector128 vxrs[vxr_size] __aligned(8); \ } +KERNEL_FPU_STRUCT(8); KERNEL_FPU_STRUCT(16); KERNEL_FPU_STRUCT(32); #define DECLARE_KERNEL_FPU_ONSTACK(vxr_size, name) \ struct kernel_fpu_##vxr_size name __uninitialized +#define DECLARE_KERNEL_FPU_ONSTACK8(name) \ + DECLARE_KERNEL_FPU_ONSTACK(8, name) + #define DECLARE_KERNEL_FPU_ONSTACK16(name) \ DECLARE_KERNEL_FPU_ONSTACK(16, name) diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 7c50eca85ca4..90eac15ea62a 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -4,6 +4,7 @@ # lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o +lib-y += csum-partial.o obj-y += mem.o xor.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c new file mode 100644 index 000000000000..3ea009cbc3b7 --- /dev/null +++ b/arch/s390/lib/csum-partial.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +/* + * Computes the checksum of a memory block at buff, length len, + * and adds in "sum" (32-bit). + * + * Returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic. + * + * This function must be called with even lengths, except + * for the last fragment, which may be odd. + * + * It's best to have buff aligned on a 64-bit boundary. + */ +__wsum csum_partial(const void *buff, int len, __wsum sum) +{ + DECLARE_KERNEL_FPU_ONSTACK8(vxstate); + + if (!cpu_has_vx()) + return cksm(buff, len, sum); + kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); + fpu_vlvgf(16, (__force u32)sum, 1); + fpu_vzero(17); + fpu_vzero(18); + fpu_vzero(19); + while (len >= 64) { + fpu_vlm(20, 23, buff); + fpu_vcksm(16, 20, 16); + fpu_vcksm(17, 21, 17); + fpu_vcksm(18, 22, 18); + fpu_vcksm(19, 23, 19); + buff += 64; + len -= 64; + } + while (len >= 32) { + fpu_vlm(20, 21, buff); + fpu_vcksm(16, 20, 16); + fpu_vcksm(17, 21, 17); + buff += 32; + len -= 32; + } + while (len >= 16) { + fpu_vl(20, buff); + fpu_vcksm(16, 20, 16); + buff += 16; + len -= 16; + } + if (len) { + fpu_vll(20, len - 1, buff); + fpu_vcksm(16, 20, 16); + } + fpu_vcksm(18, 19, 18); + fpu_vcksm(16, 17, 16); + fpu_vcksm(16, 18, 16); + sum = (__force __wsum)fpu_vlgvf(16, 1); + kernel_fpu_end(&vxstate, KERNEL_VXR_V16V23); + return sum; +} +EXPORT_SYMBOL(csum_partial); From dcd3e1de9d17dc43dfed87a9fc814b9dec508043 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:23 +0100 Subject: [PATCH 077/117] s390/checksum: provide csum_partial_copy_nocheck() With csum_partial(), which reads all bytes into registers it is easy to also implement csum_partial_copy_nocheck() which copies the buffer while calculating its checksum. For a 512 byte buffer this reduces the runtime by 19%. Compared to the old generic variant (memcpy() + cksm instruction) runtime is reduced by 42%). Signed-off-by: Heiko Carstens --- arch/s390/include/asm/checksum.h | 3 ++ arch/s390/include/asm/fpu-insn-asm.h | 10 +++++ arch/s390/include/asm/fpu-insn.h | 58 ++++++++++++++++++++++++++++ arch/s390/lib/csum-partial.c | 54 +++++++++++++++++++------- 4 files changed, 112 insertions(+), 13 deletions(-) diff --git a/arch/s390/include/asm/checksum.h b/arch/s390/include/asm/checksum.h index 00095cc20afa..b89159591ca0 100644 --- a/arch/s390/include/asm/checksum.h +++ b/arch/s390/include/asm/checksum.h @@ -32,6 +32,9 @@ static inline __wsum cksm(const void *buff, int len, __wsum sum) __wsum csum_partial(const void *buff, int len, __wsum sum); +#define _HAVE_ARCH_CSUM_AND_COPY +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len); + /* * Fold a partial checksum without adding pseudo headers. */ diff --git a/arch/s390/include/asm/fpu-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index aaf42c513a21..02ccfe46050a 100644 --- a/arch/s390/include/asm/fpu-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -531,6 +531,16 @@ MRXBOPC 0, 0x37, v1 .endm +/* VECTOR STORE WITH LENGTH */ +.macro VSTL v, gr, disp, base + VX_NUM v1, \v + GR_NUM b2, \base + GR_NUM r3, \gr + .word 0xE700 | ((v1&15) << 4) | r3 + .word (b2 << 12) | (\disp) + MRXBOPC 0, 0x3f, v1 +.endm + /* Vector integer instructions */ /* VECTOR AND */ diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 7e9997fa45d3..35c4fbe0bdd6 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -241,6 +241,64 @@ static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index) #ifdef CONFIG_CC_IS_CLANG +static __always_inline void fpu_vst(u8 v1, const void *vxr) +{ + instrument_write(vxr, sizeof(__vector128)); + asm volatile("\n" + " la 1,%[vxr]\n" + " VST %[v1],0,,1\n" + : [vxr] "=R" (*(__vector128 *)vxr) + : [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vst(u8 v1, const void *vxr) +{ + instrument_write(vxr, sizeof(__vector128)); + asm volatile("VST %[v1],%O[vxr],,%R[vxr]\n" + : [vxr] "=Q" (*(__vector128 *)vxr) + : [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + +static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_write(vxr, size); + asm volatile("\n" + " la 1,%[vxr]\n" + " VSTL %[v1],%[index],0,1\n" + : [vxr] "=R" (*(u8 *)vxr) + : [index] "d" (index), [v1] "I" (v1) + : "memory", "1"); +} + +#else /* CONFIG_CC_IS_CLANG */ + +static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) +{ + unsigned int size; + + size = min(index + 1, sizeof(__vector128)); + instrument_write(vxr, size); + asm volatile("VSTL %[v1],%[index],%O[vxr],%R[vxr]\n" + : [vxr] "=Q" (*(u8 *)vxr) + : [index] "d" (index), [v1] "I" (v1) + : "memory"); +} + +#endif /* CONFIG_CC_IS_CLANG */ + +#ifdef CONFIG_CC_IS_CLANG + #define fpu_vstm(_v1, _v3, _vxrs) \ ({ \ unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \ diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c index 3ea009cbc3b7..458abd9bac70 100644 --- a/arch/s390/lib/csum-partial.c +++ b/arch/s390/lib/csum-partial.c @@ -5,8 +5,8 @@ #include /* - * Computes the checksum of a memory block at buff, length len, - * and adds in "sum" (32-bit). + * Computes the checksum of a memory block at src, length len, + * and adds in "sum" (32-bit). If copy is true copies to dst. * * Returns a 32-bit number suitable for feeding into itself * or csum_tcpudp_magic. @@ -14,43 +14,60 @@ * This function must be called with even lengths, except * for the last fragment, which may be odd. * - * It's best to have buff aligned on a 64-bit boundary. + * It's best to have src and dst aligned on a 64-bit boundary. */ -__wsum csum_partial(const void *buff, int len, __wsum sum) +static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __wsum sum, bool copy) { DECLARE_KERNEL_FPU_ONSTACK8(vxstate); - if (!cpu_has_vx()) - return cksm(buff, len, sum); + if (!cpu_has_vx()) { + if (copy) + memcpy(dst, src, len); + return cksm(dst, len, sum); + } kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); fpu_vlvgf(16, (__force u32)sum, 1); fpu_vzero(17); fpu_vzero(18); fpu_vzero(19); while (len >= 64) { - fpu_vlm(20, 23, buff); + fpu_vlm(20, 23, src); + if (copy) { + fpu_vstm(20, 23, dst); + dst += 64; + } fpu_vcksm(16, 20, 16); fpu_vcksm(17, 21, 17); fpu_vcksm(18, 22, 18); fpu_vcksm(19, 23, 19); - buff += 64; + src += 64; len -= 64; } while (len >= 32) { - fpu_vlm(20, 21, buff); + fpu_vlm(20, 21, src); + if (copy) { + fpu_vstm(20, 21, dst); + dst += 32; + } fpu_vcksm(16, 20, 16); fpu_vcksm(17, 21, 17); - buff += 32; + src += 32; len -= 32; } while (len >= 16) { - fpu_vl(20, buff); + fpu_vl(20, src); + if (copy) { + fpu_vst(20, dst); + dst += 16; + } fpu_vcksm(16, 20, 16); - buff += 16; + src += 16; len -= 16; } if (len) { - fpu_vll(20, len - 1, buff); + fpu_vll(20, len - 1, src); + if (copy) + fpu_vstl(20, len - 1, dst); fpu_vcksm(16, 20, 16); } fpu_vcksm(18, 19, 18); @@ -60,4 +77,15 @@ __wsum csum_partial(const void *buff, int len, __wsum sum) kernel_fpu_end(&vxstate, KERNEL_VXR_V16V23); return sum; } + +__wsum csum_partial(const void *buff, int len, __wsum sum) +{ + return csum_copy(NULL, buff, len, sum, false); +} EXPORT_SYMBOL(csum_partial); + +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len) +{ + return csum_copy(dst, src, len, 0, true); +} +EXPORT_SYMBOL(csum_partial_copy_nocheck); From c8dde11df19192c421f5b70c2b8ba55d32e07c66 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:24 +0100 Subject: [PATCH 078/117] s390/raid6: convert to use standard fpu_*() inline assemblies Move the s390 specific raid6 inline assemblies, make them generic, and reuse them to implement the raid6 gen/xor implementation. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 48 ++++++++++++++++++++++++++++ lib/raid6/s390vx.uc | 55 ++++++-------------------------- 2 files changed, 58 insertions(+), 45 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 35c4fbe0bdd6..028d28570170 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -108,6 +108,14 @@ static __always_inline void fpu_stfpc(unsigned int *fpc) : "memory"); } +static __always_inline void fpu_vab(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VAB %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3) { asm volatile("VCKSM %[v1],%[v2],%[v3]" @@ -116,6 +124,14 @@ static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3) : "memory"); } +static __always_inline void fpu_vesravb(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VESRAVB %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + #ifdef CONFIG_CC_IS_CLANG static __always_inline void fpu_vl(u8 v1, const void *vxr) @@ -231,6 +247,14 @@ static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr) #endif /* CONFIG_CC_IS_CLANG */ +static __always_inline void fpu_vlr(u8 v1, u8 v2) +{ + asm volatile("VLR %[v1],%[v2]" + : + : [v1] "I" (v1), [v2] "I" (v2) + : "memory"); +} + static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index) { asm volatile("VLVGF %[v],%[val],%[index]" @@ -239,6 +263,22 @@ static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index) : "memory"); } +static __always_inline void fpu_vn(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VN %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + +static __always_inline void fpu_vrepib(u8 v1, s16 i2) +{ + asm volatile("VREPIB %[v1],%[i2]" + : + : [v1] "I" (v1), [i2] "K" (i2) + : "memory"); +} + #ifdef CONFIG_CC_IS_CLANG static __always_inline void fpu_vst(u8 v1, const void *vxr) @@ -335,6 +375,14 @@ static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) #endif /* CONFIG_CC_IS_CLANG */ +static __always_inline void fpu_vx(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VX %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + static __always_inline void fpu_vzero(u8 v) { asm volatile("VZERO %[v]" diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc index 92c05b7596bc..863e2d320938 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid6/s390vx.uc @@ -16,10 +16,10 @@ #define NSIZE 16 -static inline void LOAD_CONST(void) +static __always_inline void LOAD_CONST(void) { - asm volatile("VREPIB %v24,7"); - asm volatile("VREPIB %v25,0x1d"); + fpu_vrepib(24, 0x07); + fpu_vrepib(25, 0x1d); } /* @@ -27,10 +27,7 @@ static inline void LOAD_CONST(void) * vector register y left by 1 bit and stores the result in * vector register x. */ -static inline void SHLBYTE(int x, int y) -{ - asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y)); -} +#define SHLBYTE(x, y) fpu_vab(x, y, y) /* * For each of the 16 bytes in the vector register y the MASK() @@ -38,45 +35,13 @@ static inline void SHLBYTE(int x, int y) * or 0x00 if the high bit is 0. The result is stored in vector * register x. */ -static inline void MASK(int x, int y) -{ - asm volatile ("VESRAVB %0,%1,24" : : "i" (x), "i" (y)); -} +#define MASK(x, y) fpu_vesravb(x, y, 24) -static inline void AND(int x, int y, int z) -{ - asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z)); -} - -static inline void XOR(int x, int y, int z) -{ - asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z)); -} - -static inline void LOAD_DATA(int x, u8 *ptr) -{ - typedef struct { u8 _[16 * $#]; } addrtype; - register addrtype *__ptr asm("1") = (addrtype *) ptr; - - asm volatile ("VLM %2,%3,0,%1" - : : "m" (*__ptr), "a" (__ptr), "i" (x), - "i" (x + $# - 1)); -} - -static inline void STORE_DATA(int x, u8 *ptr) -{ - typedef struct { u8 _[16 * $#]; } addrtype; - register addrtype *__ptr asm("1") = (addrtype *) ptr; - - asm volatile ("VSTM %2,%3,0,1" - : "=m" (*__ptr) : "a" (__ptr), "i" (x), - "i" (x + $# - 1)); -} - -static inline void COPY_VEC(int x, int y) -{ - asm volatile ("VLR %0,%1" : : "i" (x), "i" (y)); -} +#define AND(x, y, z) fpu_vn(x, y, z) +#define XOR(x, y, z) fpu_vx(x, y, z) +#define LOAD_DATA(x, ptr) fpu_vlm(x, x + $# - 1, ptr) +#define STORE_DATA(x, ptr) fpu_vstm(x, x + $# - 1, ptr) +#define COPY_VEC(x, y) fpu_vlr(x, y) static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) { From ea8b75d2893681c91ffd89806caaa2ec5b22e073 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:25 +0100 Subject: [PATCH 079/117] s390/sysinfo: convert bogomips calculation to C Provide several one instruction fpu inline assemebles and use them to implement the bogomips calculation in C like style. This is more for illustration purposes on how kernel fpu code can be written in C. This has the advantage that the author only has to take care of the floating point instructions, but doesn't need to take care of general purpose register allocation (if needed), and the semantics of all other instructions not related to fpu. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 35 ++++++++++++++++++++++++++++++++ arch/s390/kernel/sysinfo.c | 23 ++++++++------------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 028d28570170..bbf782a2965d 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -36,6 +36,33 @@ asm(".include \"asm/fpu-insn-asm.h\"\n"); * barrier. */ +static __always_inline void fpu_cefbr(u8 f1, s32 val) +{ + asm volatile("cefbr %[f1],%[val]\n" + : + : [f1] "I" (f1), [val] "d" (val) + : "memory"); +} + +static __always_inline unsigned long fpu_cgebr(u8 f2, u8 mode) +{ + unsigned long val; + + asm volatile("cgebr %[val],%[mode],%[f2]\n" + : [val] "=d" (val) + : [f2] "I" (f2), [mode] "I" (mode) + : "memory"); + return val; +} + +static __always_inline void fpu_debr(u8 f1, u8 f2) +{ + asm volatile("debr %[f1],%[f2]\n" + : + : [f1] "I" (f1), [f2] "I" (f2) + : "memory"); +} + static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg) { instrument_read(reg, sizeof(*reg)); @@ -45,6 +72,14 @@ static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg) : "memory"); } +static __always_inline void fpu_ldgr(u8 f1, u32 val) +{ + asm volatile("ldgr %[f1],%[val]\n" + : + : [f1] "I" (f1), [val] "d" (val) + : "memory"); +} + static __always_inline void fpu_lfpc(unsigned int *fpc) { instrument_read(fpc, sizeof(*fpc)); diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 4cd6428bfab2..1b1be3110cfc 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -447,21 +447,14 @@ void s390_adjust_jiffies(void) * point division .. */ kernel_fpu_begin(&fpu, KERNEL_FPR); - asm volatile( - " sfpc %3\n" - " l %0,%1\n" - " tmlh %0,0xff80\n" - " jnz 0f\n" - " cefbr %%f2,%0\n" - " j 1f\n" - "0: le %%f2,%1\n" - "1: cefbr %%f0,%2\n" - " debr %%f0,%%f2\n" - " cgebr %0,5,%%f0\n" - : "=&d" (capability) - : "Q" (info->capability), "d" (10000000), "d" (0) - : "cc" - ); + fpu_sfpc(0); + if (info->capability & 0xff800000) + fpu_ldgr(2, info->capability); + else + fpu_cefbr(2, info->capability); + fpu_cefbr(0, 10000000); + fpu_debr(0, 2); + capability = fpu_cgebr(0, 5); kernel_fpu_end(&fpu, KERNEL_FPR); } else /* From 37346951a89ae0a6054d4286f5a90dadc57eee5d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:26 +0100 Subject: [PATCH 080/117] s390/fpu: add vector instruction inline assemblies for crc32 Provide various vector instruction inline assemblies for crc32 calculations. This is just preparation to keep the conversion of the existing crc32 implementations from assembly to C small. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index bbf782a2965d..c1e2e521d9af 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -167,6 +167,22 @@ static __always_inline void fpu_vesravb(u8 v1, u8 v2, u8 v3) : "memory"); } +static __always_inline void fpu_vgfmag(u8 v1, u8 v2, u8 v3, u8 v4) +{ + asm volatile("VGFMAG %[v1],%[v2],%[v3],%[v4]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4) + : "memory"); +} + +static __always_inline void fpu_vgfmg(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VGFMG %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + #ifdef CONFIG_CC_IS_CLANG static __always_inline void fpu_vl(u8 v1, const void *vxr) @@ -195,6 +211,22 @@ static __always_inline void fpu_vl(u8 v1, const void *vxr) #endif /* CONFIG_CC_IS_CLANG */ +static __always_inline void fpu_vleib(u8 v, s16 val, u8 index) +{ + asm volatile("VLEIB %[v],%[val],%[index]" + : + : [v] "I" (v), [val] "K" (val), [index] "I" (index) + : "memory"); +} + +static __always_inline void fpu_vleig(u8 v, s16 val, u8 index) +{ + asm volatile("VLEIG %[v],%[val],%[index]" + : + : [v] "I" (v), [val] "K" (val), [index] "I" (index) + : "memory"); +} + static __always_inline u64 fpu_vlgvf(u8 v, u16 index) { u64 val; @@ -306,6 +338,14 @@ static __always_inline void fpu_vn(u8 v1, u8 v2, u8 v3) : "memory"); } +static __always_inline void fpu_vperm(u8 v1, u8 v2, u8 v3, u8 v4) +{ + asm volatile("VPERM %[v1],%[v2],%[v3],%[v4]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4) + : "memory"); +} + static __always_inline void fpu_vrepib(u8 v1, s16 i2) { asm volatile("VREPIB %[v1],%[i2]" @@ -314,6 +354,14 @@ static __always_inline void fpu_vrepib(u8 v1, s16 i2) : "memory"); } +static __always_inline void fpu_vsrlb(u8 v1, u8 v2, u8 v3) +{ + asm volatile("VSRLB %[v1],%[v2],%[v3]" + : + : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3) + : "memory"); +} + #ifdef CONFIG_CC_IS_CLANG static __always_inline void fpu_vst(u8 v1, const void *vxr) @@ -410,6 +458,14 @@ static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr) #endif /* CONFIG_CC_IS_CLANG */ +static __always_inline void fpu_vupllf(u8 v1, u8 v2) +{ + asm volatile("VUPLLF %[v1],%[v2]" + : + : [v1] "I" (v1), [v2] "I" (v2) + : "memory"); +} + static __always_inline void fpu_vx(u8 v1, u8 v2, u8 v3) { asm volatile("VX %[v1],%[v2],%[v3]" From c59bf4de01b67184c19a9f6f04caa1a8d5b55afb Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:27 +0100 Subject: [PATCH 081/117] s390/crc32be: convert to C Convert CRC-32 BE variant to C. Signed-off-by: Heiko Carstens --- arch/s390/crypto/crc32-vx.c | 3 +- arch/s390/crypto/crc32-vx.h | 10 + .../crypto/{crc32be-vx.S => crc32be-vx.c} | 171 +++++++----------- 3 files changed, 77 insertions(+), 107 deletions(-) create mode 100644 arch/s390/crypto/crc32-vx.h rename arch/s390/crypto/{crc32be-vx.S => crc32be-vx.c} (56%) diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index d9f1fdb66691..0f3e6094174e 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -14,7 +14,7 @@ #include #include #include - +#include "crc32-vx.h" #define CRC32_BLOCK_SIZE 1 #define CRC32_DIGEST_SIZE 4 @@ -33,7 +33,6 @@ struct crc_desc_ctx { /* Prototypes for functions in assembly files */ u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); -u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); /* diff --git a/arch/s390/crypto/crc32-vx.h b/arch/s390/crypto/crc32-vx.h new file mode 100644 index 000000000000..eba754c2821b --- /dev/null +++ b/arch/s390/crypto/crc32-vx.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _CRC32_VX_S390_H +#define _CRC32_VX_S390_H + +#include + +u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); + +#endif /* _CRC32_VX_S390_H */ diff --git a/arch/s390/crypto/crc32be-vx.S b/arch/s390/crypto/crc32be-vx.c similarity index 56% rename from arch/s390/crypto/crc32be-vx.S rename to arch/s390/crypto/crc32be-vx.c index f2dc8a688afb..fed7c9c70d05 100644 --- a/arch/s390/crypto/crc32be-vx.S +++ b/arch/s390/crypto/crc32be-vx.c @@ -12,20 +12,17 @@ * Author(s): Hendrik Brueckner */ -#include -#include -#include +#include +#include +#include "crc32-vx.h" /* Vector register range containing CRC-32 constants */ -#define CONST_R1R2 %v9 -#define CONST_R3R4 %v10 -#define CONST_R5 %v11 -#define CONST_R6 %v12 -#define CONST_RU_POLY %v13 -#define CONST_CRC_POLY %v14 - - .data - .balign 8 +#define CONST_R1R2 9 +#define CONST_R3R4 10 +#define CONST_R5 11 +#define CONST_R6 12 +#define CONST_RU_POLY 13 +#define CONST_CRC_POLY 14 /* * The CRC-32 constant block contains reduction constants to fold and @@ -58,105 +55,74 @@ * P'(x) = 0xEDB88320 */ -SYM_DATA_START_LOCAL(constants_CRC_32_BE) - .quad 0x08833794c, 0x0e6228b11 # R1, R2 - .quad 0x0c5b9cd4c, 0x0e8a45605 # R3, R4 - .quad 0x0f200aa66, 1 << 32 # R5, x32 - .quad 0x0490d678d, 1 # R6, 1 - .quad 0x104d101df, 0 # u - .quad 0x104C11DB7, 0 # P(x) -SYM_DATA_END(constants_CRC_32_BE) +static unsigned long constants_CRC_32_BE[] = { + 0x08833794c, 0x0e6228b11, /* R1, R2 */ + 0x0c5b9cd4c, 0x0e8a45605, /* R3, R4 */ + 0x0f200aa66, 1UL << 32, /* R5, x32 */ + 0x0490d678d, 1, /* R6, 1 */ + 0x104d101df, 0, /* u */ + 0x104C11DB7, 0, /* P(x) */ +}; - .previous - - GEN_BR_THUNK %r14 - - .text -/* - * The CRC-32 function(s) use these calling conventions: - * - * Parameters: - * - * %r2: Initial CRC value, typically ~0; and final CRC (return) value. - * %r3: Input buffer pointer, performance might be improved if the - * buffer is on a doubleword boundary. - * %r4: Length of the buffer, must be 64 bytes or greater. +/** + * crc32_be_vgfm_16 - Compute CRC-32 (BE variant) with vector registers + * @crc: Initial CRC value, typically ~0. + * @buf: Input buffer pointer, performance might be improved if the + * buffer is on a doubleword boundary. + * @size: Size of the buffer, must be 64 bytes or greater. * * Register usage: - * - * %r5: CRC-32 constant pool base pointer. * V0: Initial CRC value and intermediate constants and results. * V1..V4: Data for CRC computation. * V5..V8: Next data chunks that are fetched from the input buffer. - * * V9..V14: CRC-32 constants. */ -SYM_FUNC_START(crc32_be_vgfm_16) +u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size) +{ /* Load CRC-32 constants */ - larl %r5,constants_CRC_32_BE - VLM CONST_R1R2,CONST_CRC_POLY,0,%r5 + fpu_vlm(CONST_R1R2, CONST_CRC_POLY, &constants_CRC_32_BE); + fpu_vzero(0); /* Load the initial CRC value into the leftmost word of V0. */ - VZERO %v0 - VLVGF %v0,%r2,0 + fpu_vlvgf(0, crc, 0); /* Load a 64-byte data chunk and XOR with CRC */ - VLM %v1,%v4,0,%r3 /* 64-bytes into V1..V4 */ - VX %v1,%v0,%v1 /* V1 ^= CRC */ - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ + fpu_vlm(1, 4, buf); + fpu_vx(1, 0, 1); + buf += 64; + size -= 64; - /* Check remaining buffer size and jump to proper folding method */ - cghi %r4,64 - jl .Lless_than_64bytes + while (size >= 64) { + /* Load the next 64-byte data chunk into V5 to V8 */ + fpu_vlm(5, 8, buf); -.Lfold_64bytes_loop: - /* Load the next 64-byte data chunk into V5 to V8 */ - VLM %v5,%v8,0,%r3 + /* + * Perform a GF(2) multiplication of the doublewords in V1 with + * the reduction constants in V0. The intermediate result is + * then folded (accumulated) with the next data chunk in V5 and + * stored in V1. Repeat this step for the register contents + * in V2, V3, and V4 respectively. + */ + fpu_vgfmag(1, CONST_R1R2, 1, 5); + fpu_vgfmag(2, CONST_R1R2, 2, 6); + fpu_vgfmag(3, CONST_R1R2, 3, 7); + fpu_vgfmag(4, CONST_R1R2, 4, 8); + buf += 64; + size -= 64; + } - /* - * Perform a GF(2) multiplication of the doublewords in V1 with - * the reduction constants in V0. The intermediate result is - * then folded (accumulated) with the next data chunk in V5 and - * stored in V1. Repeat this step for the register contents - * in V2, V3, and V4 respectively. - */ - VGFMAG %v1,CONST_R1R2,%v1,%v5 - VGFMAG %v2,CONST_R1R2,%v2,%v6 - VGFMAG %v3,CONST_R1R2,%v3,%v7 - VGFMAG %v4,CONST_R1R2,%v4,%v8 - - /* Adjust buffer pointer and length for next loop */ - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ - - cghi %r4,64 - jnl .Lfold_64bytes_loop - -.Lless_than_64bytes: /* Fold V1 to V4 into a single 128-bit value in V1 */ - VGFMAG %v1,CONST_R3R4,%v1,%v2 - VGFMAG %v1,CONST_R3R4,%v1,%v3 - VGFMAG %v1,CONST_R3R4,%v1,%v4 + fpu_vgfmag(1, CONST_R3R4, 1, 2); + fpu_vgfmag(1, CONST_R3R4, 1, 3); + fpu_vgfmag(1, CONST_R3R4, 1, 4); - /* Check whether to continue with 64-bit folding */ - cghi %r4,16 - jl .Lfinal_fold + while (size >= 16) { + fpu_vl(2, buf); + fpu_vgfmag(1, CONST_R3R4, 1, 2); + buf += 16; + size -= 16; + } -.Lfold_16bytes_loop: - - VL %v2,0,,%r3 /* Load next data chunk */ - VGFMAG %v1,CONST_R3R4,%v1,%v2 /* Fold next data chunk */ - - /* Adjust buffer pointer and size for folding next data chunk */ - aghi %r3,16 - aghi %r4,-16 - - /* Process remaining data chunks */ - cghi %r4,16 - jnl .Lfold_16bytes_loop - -.Lfinal_fold: /* * The R5 constant is used to fold a 128-bit value into an 96-bit value * that is XORed with the next 96-bit input data chunk. To use a single @@ -164,7 +130,7 @@ SYM_FUNC_START(crc32_be_vgfm_16) * form an intermediate 96-bit value (with appended zeros) which is then * XORed with the intermediate reduction result. */ - VGFMG %v1,CONST_R5,%v1 + fpu_vgfmg(1, CONST_R5, 1); /* * Further reduce the remaining 96-bit value to a 64-bit value using a @@ -173,7 +139,7 @@ SYM_FUNC_START(crc32_be_vgfm_16) * doubleword with R6. The result is a 64-bit value and is subject to * the Barret reduction. */ - VGFMG %v1,CONST_R6,%v1 + fpu_vgfmg(1, CONST_R6, 1); /* * The input values to the Barret reduction are the degree-63 polynomial @@ -194,20 +160,15 @@ SYM_FUNC_START(crc32_be_vgfm_16) */ /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ - VUPLLF %v2,%v1 - VGFMG %v2,CONST_RU_POLY,%v2 + fpu_vupllf(2, 1); + fpu_vgfmg(2, CONST_RU_POLY, 2); /* * Compute the GF(2) product of the CRC polynomial in VO with T1(x) in * V2 and XOR the intermediate result, T2(x), with the value in V1. * The final result is in the rightmost word of V2. */ - VUPLLF %v2,%v2 - VGFMAG %v2,CONST_CRC_POLY,%v2,%v1 - -.Ldone: - VLGVF %r2,%v2,3 - BR_EX %r14 -SYM_FUNC_END(crc32_be_vgfm_16) - -.previous + fpu_vupllf(2, 2); + fpu_vgfmag(2, CONST_CRC_POLY, 2, 1); + return fpu_vlgvf(2, 3); +} From 03325e9b64c48313527f0373c4688d393b1edaf3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 3 Feb 2024 11:45:28 +0100 Subject: [PATCH 082/117] s390/crc32le: convert to C Convert CRC-32 LE variants to C. Signed-off-by: Heiko Carstens --- arch/s390/crypto/crc32-vx.c | 4 - arch/s390/crypto/crc32-vx.h | 2 + .../crypto/{crc32le-vx.S => crc32le-vx.c} | 239 ++++++++---------- 3 files changed, 104 insertions(+), 141 deletions(-) rename arch/s390/crypto/{crc32le-vx.S => crc32le-vx.c} (52%) diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c index 0f3e6094174e..74f17c905d12 100644 --- a/arch/s390/crypto/crc32-vx.c +++ b/arch/s390/crypto/crc32-vx.c @@ -31,10 +31,6 @@ struct crc_desc_ctx { u32 crc; }; -/* Prototypes for functions in assembly files */ -u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); -u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); - /* * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension * diff --git a/arch/s390/crypto/crc32-vx.h b/arch/s390/crypto/crc32-vx.h index eba754c2821b..652c96e1a822 100644 --- a/arch/s390/crypto/crc32-vx.h +++ b/arch/s390/crypto/crc32-vx.h @@ -6,5 +6,7 @@ #include u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size); +u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); +u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size); #endif /* _CRC32_VX_S390_H */ diff --git a/arch/s390/crypto/crc32le-vx.S b/arch/s390/crypto/crc32le-vx.c similarity index 52% rename from arch/s390/crypto/crc32le-vx.S rename to arch/s390/crypto/crc32le-vx.c index df19e06ff8bc..2f629f394df7 100644 --- a/arch/s390/crypto/crc32le-vx.S +++ b/arch/s390/crypto/crc32le-vx.c @@ -13,20 +13,17 @@ * Author(s): Hendrik Brueckner */ -#include -#include -#include +#include +#include +#include "crc32-vx.h" /* Vector register range containing CRC-32 constants */ -#define CONST_PERM_LE2BE %v9 -#define CONST_R2R1 %v10 -#define CONST_R4R3 %v11 -#define CONST_R5 %v12 -#define CONST_RU_POLY %v13 -#define CONST_CRC_POLY %v14 - - .data - .balign 8 +#define CONST_PERM_LE2BE 9 +#define CONST_R2R1 10 +#define CONST_R4R3 11 +#define CONST_R5 12 +#define CONST_RU_POLY 13 +#define CONST_CRC_POLY 14 /* * The CRC-32 constant block contains reduction constants to fold and @@ -59,64 +56,43 @@ * P'(x) = 0x82F63B78 */ -SYM_DATA_START_LOCAL(constants_CRC_32_LE) - .octa 0x0F0E0D0C0B0A09080706050403020100 # BE->LE mask - .quad 0x1c6e41596, 0x154442bd4 # R2, R1 - .quad 0x0ccaa009e, 0x1751997d0 # R4, R3 - .octa 0x163cd6124 # R5 - .octa 0x1F7011641 # u' - .octa 0x1DB710641 # P'(x) << 1 -SYM_DATA_END(constants_CRC_32_LE) +static unsigned long constants_CRC_32_LE[] = { + 0x0f0e0d0c0b0a0908, 0x0706050403020100, /* BE->LE mask */ + 0x1c6e41596, 0x154442bd4, /* R2, R1 */ + 0x0ccaa009e, 0x1751997d0, /* R4, R3 */ + 0x0, 0x163cd6124, /* R5 */ + 0x0, 0x1f7011641, /* u' */ + 0x0, 0x1db710641 /* P'(x) << 1 */ +}; -SYM_DATA_START_LOCAL(constants_CRC_32C_LE) - .octa 0x0F0E0D0C0B0A09080706050403020100 # BE->LE mask - .quad 0x09e4addf8, 0x740eef02 # R2, R1 - .quad 0x14cd00bd6, 0xf20c0dfe # R4, R3 - .octa 0x0dd45aab8 # R5 - .octa 0x0dea713f1 # u' - .octa 0x105ec76f0 # P'(x) << 1 -SYM_DATA_END(constants_CRC_32C_LE) +static unsigned long constants_CRC_32C_LE[] = { + 0x0f0e0d0c0b0a0908, 0x0706050403020100, /* BE->LE mask */ + 0x09e4addf8, 0x740eef02, /* R2, R1 */ + 0x14cd00bd6, 0xf20c0dfe, /* R4, R3 */ + 0x0, 0x0dd45aab8, /* R5 */ + 0x0, 0x0dea713f1, /* u' */ + 0x0, 0x105ec76f0 /* P'(x) << 1 */ +}; - .previous - - GEN_BR_THUNK %r14 - - .text - -/* - * The CRC-32 functions use these calling conventions: - * - * Parameters: - * - * %r2: Initial CRC value, typically ~0; and final CRC (return) value. - * %r3: Input buffer pointer, performance might be improved if the - * buffer is on a doubleword boundary. - * %r4: Length of the buffer, must be 64 bytes or greater. +/** + * crc32_le_vgfm_generic - Compute CRC-32 (LE variant) with vector registers + * @crc: Initial CRC value, typically ~0. + * @buf: Input buffer pointer, performance might be improved if the + * buffer is on a doubleword boundary. + * @size: Size of the buffer, must be 64 bytes or greater. + * @constants: CRC-32 constant pool base pointer. * * Register usage: - * - * %r5: CRC-32 constant pool base pointer. - * V0: Initial CRC value and intermediate constants and results. - * V1..V4: Data for CRC computation. - * V5..V8: Next data chunks that are fetched from the input buffer. - * V9: Constant for BE->LE conversion and shift operations - * + * V0: Initial CRC value and intermediate constants and results. + * V1..V4: Data for CRC computation. + * V5..V8: Next data chunks that are fetched from the input buffer. + * V9: Constant for BE->LE conversion and shift operations * V10..V14: CRC-32 constants. */ - -SYM_FUNC_START(crc32_le_vgfm_16) - larl %r5,constants_CRC_32_LE - j crc32_le_vgfm_generic -SYM_FUNC_END(crc32_le_vgfm_16) - -SYM_FUNC_START(crc32c_le_vgfm_16) - larl %r5,constants_CRC_32C_LE - j crc32_le_vgfm_generic -SYM_FUNC_END(crc32c_le_vgfm_16) - -SYM_FUNC_START(crc32_le_vgfm_generic) +static u32 crc32_le_vgfm_generic(u32 crc, unsigned char const *buf, size_t size, unsigned long *constants) +{ /* Load CRC-32 constants */ - VLM CONST_PERM_LE2BE,CONST_CRC_POLY,0,%r5 + fpu_vlm(CONST_PERM_LE2BE, CONST_CRC_POLY, constants); /* * Load the initial CRC value. @@ -125,90 +101,73 @@ SYM_FUNC_START(crc32_le_vgfm_generic) * vector register and is later XORed with the LSB portion * of the loaded input data. */ - VZERO %v0 /* Clear V0 */ - VLVGF %v0,%r2,3 /* Load CRC into rightmost word */ + fpu_vzero(0); /* Clear V0 */ + fpu_vlvgf(0, crc, 3); /* Load CRC into rightmost word */ /* Load a 64-byte data chunk and XOR with CRC */ - VLM %v1,%v4,0,%r3 /* 64-bytes into V1..V4 */ - VPERM %v1,%v1,%v1,CONST_PERM_LE2BE - VPERM %v2,%v2,%v2,CONST_PERM_LE2BE - VPERM %v3,%v3,%v3,CONST_PERM_LE2BE - VPERM %v4,%v4,%v4,CONST_PERM_LE2BE + fpu_vlm(1, 4, buf); + fpu_vperm(1, 1, 1, CONST_PERM_LE2BE); + fpu_vperm(2, 2, 2, CONST_PERM_LE2BE); + fpu_vperm(3, 3, 3, CONST_PERM_LE2BE); + fpu_vperm(4, 4, 4, CONST_PERM_LE2BE); - VX %v1,%v0,%v1 /* V1 ^= CRC */ - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ + fpu_vx(1, 0, 1); /* V1 ^= CRC */ + buf += 64; + size -= 64; - cghi %r4,64 - jl .Lless_than_64bytes + while (size >= 64) { + fpu_vlm(5, 8, buf); + fpu_vperm(5, 5, 5, CONST_PERM_LE2BE); + fpu_vperm(6, 6, 6, CONST_PERM_LE2BE); + fpu_vperm(7, 7, 7, CONST_PERM_LE2BE); + fpu_vperm(8, 8, 8, CONST_PERM_LE2BE); + /* + * Perform a GF(2) multiplication of the doublewords in V1 with + * the R1 and R2 reduction constants in V0. The intermediate + * result is then folded (accumulated) with the next data chunk + * in V5 and stored in V1. Repeat this step for the register + * contents in V2, V3, and V4 respectively. + */ + fpu_vgfmag(1, CONST_R2R1, 1, 5); + fpu_vgfmag(2, CONST_R2R1, 2, 6); + fpu_vgfmag(3, CONST_R2R1, 3, 7); + fpu_vgfmag(4, CONST_R2R1, 4, 8); + buf += 64; + size -= 64; + } -.Lfold_64bytes_loop: - /* Load the next 64-byte data chunk into V5 to V8 */ - VLM %v5,%v8,0,%r3 - VPERM %v5,%v5,%v5,CONST_PERM_LE2BE - VPERM %v6,%v6,%v6,CONST_PERM_LE2BE - VPERM %v7,%v7,%v7,CONST_PERM_LE2BE - VPERM %v8,%v8,%v8,CONST_PERM_LE2BE - - /* - * Perform a GF(2) multiplication of the doublewords in V1 with - * the R1 and R2 reduction constants in V0. The intermediate result - * is then folded (accumulated) with the next data chunk in V5 and - * stored in V1. Repeat this step for the register contents - * in V2, V3, and V4 respectively. - */ - VGFMAG %v1,CONST_R2R1,%v1,%v5 - VGFMAG %v2,CONST_R2R1,%v2,%v6 - VGFMAG %v3,CONST_R2R1,%v3,%v7 - VGFMAG %v4,CONST_R2R1,%v4,%v8 - - aghi %r3,64 /* BUF = BUF + 64 */ - aghi %r4,-64 /* LEN = LEN - 64 */ - - cghi %r4,64 - jnl .Lfold_64bytes_loop - -.Lless_than_64bytes: /* * Fold V1 to V4 into a single 128-bit value in V1. Multiply V1 with R3 * and R4 and accumulating the next 128-bit chunk until a single 128-bit * value remains. */ - VGFMAG %v1,CONST_R4R3,%v1,%v2 - VGFMAG %v1,CONST_R4R3,%v1,%v3 - VGFMAG %v1,CONST_R4R3,%v1,%v4 + fpu_vgfmag(1, CONST_R4R3, 1, 2); + fpu_vgfmag(1, CONST_R4R3, 1, 3); + fpu_vgfmag(1, CONST_R4R3, 1, 4); - cghi %r4,16 - jl .Lfinal_fold + while (size >= 16) { + fpu_vl(2, buf); + fpu_vperm(2, 2, 2, CONST_PERM_LE2BE); + fpu_vgfmag(1, CONST_R4R3, 1, 2); + buf += 16; + size -= 16; + } -.Lfold_16bytes_loop: - - VL %v2,0,,%r3 /* Load next data chunk */ - VPERM %v2,%v2,%v2,CONST_PERM_LE2BE - VGFMAG %v1,CONST_R4R3,%v1,%v2 /* Fold next data chunk */ - - aghi %r3,16 - aghi %r4,-16 - - cghi %r4,16 - jnl .Lfold_16bytes_loop - -.Lfinal_fold: /* * Set up a vector register for byte shifts. The shift value must * be loaded in bits 1-4 in byte element 7 of a vector register. * Shift by 8 bytes: 0x40 * Shift by 4 bytes: 0x20 */ - VLEIB %v9,0x40,7 + fpu_vleib(9, 0x40, 7); /* * Prepare V0 for the next GF(2) multiplication: shift V0 by 8 bytes * to move R4 into the rightmost doubleword and set the leftmost * doubleword to 0x1. */ - VSRLB %v0,CONST_R4R3,%v9 - VLEIG %v0,1,0 + fpu_vsrlb(0, CONST_R4R3, 9); + fpu_vleig(0, 1, 0); /* * Compute GF(2) product of V1 and V0. The rightmost doubleword @@ -216,7 +175,7 @@ SYM_FUNC_START(crc32_le_vgfm_generic) * multiplied by 0x1 and is then XORed with rightmost product. * Implicitly, the intermediate leftmost product becomes padded */ - VGFMG %v1,%v0,%v1 + fpu_vgfmg(1, 0, 1); /* * Now do the final 32-bit fold by multiplying the rightmost word @@ -231,10 +190,10 @@ SYM_FUNC_START(crc32_le_vgfm_generic) * rightmost doubleword and the leftmost doubleword is zero to ignore * the leftmost product of V1. */ - VLEIB %v9,0x20,7 /* Shift by words */ - VSRLB %v2,%v1,%v9 /* Store remaining bits in V2 */ - VUPLLF %v1,%v1 /* Split rightmost doubleword */ - VGFMAG %v1,CONST_R5,%v1,%v2 /* V1 = (V1 * R5) XOR V2 */ + fpu_vleib(9, 0x20, 7); /* Shift by words */ + fpu_vsrlb(2, 1, 9); /* Store remaining bits in V2 */ + fpu_vupllf(1, 1); /* Split rightmost doubleword */ + fpu_vgfmag(1, CONST_R5, 1, 2); /* V1 = (V1 * R5) XOR V2 */ /* * Apply a Barret reduction to compute the final 32-bit CRC value. @@ -256,20 +215,26 @@ SYM_FUNC_START(crc32_le_vgfm_generic) */ /* T1(x) = floor( R(x) / x^32 ) GF2MUL u */ - VUPLLF %v2,%v1 - VGFMG %v2,CONST_RU_POLY,%v2 + fpu_vupllf(2, 1); + fpu_vgfmg(2, CONST_RU_POLY, 2); /* * Compute the GF(2) product of the CRC polynomial with T1(x) in * V2 and XOR the intermediate result, T2(x), with the value in V1. * The final result is stored in word element 2 of V2. */ - VUPLLF %v2,%v2 - VGFMAG %v2,CONST_CRC_POLY,%v2,%v1 + fpu_vupllf(2, 2); + fpu_vgfmag(2, CONST_CRC_POLY, 2, 1); -.Ldone: - VLGVF %r2,%v2,2 - BR_EX %r14 -SYM_FUNC_END(crc32_le_vgfm_generic) + return fpu_vlgvf(2, 2); +} -.previous +u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size) +{ + return crc32_le_vgfm_generic(crc, buf, size, &constants_CRC_32_LE[0]); +} + +u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size) +{ + return crc32_le_vgfm_generic(crc, buf, size, &constants_CRC_32C_LE[0]); +} From d065bdb4d15957fef3e115c65ab3ac34196fd7e3 Mon Sep 17 00:00:00 2001 From: Holger Dengler Date: Thu, 15 Feb 2024 09:34:03 +0100 Subject: [PATCH 083/117] s390/ap: explicitly include ultravisor header The ap_bus is using inline functions of the ultravisor (uv) in-kernel API. The related header file is implicitly included via several other headers. Replace this by an explicit include of the ultravisor header in the ap_bus file. Signed-off-by: Holger Dengler Reviewed-by: Harald Freudenberger Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index fdc0f1036544..a58190e4b745 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "ap_bus.h" #include "ap_debug.h" From fe861b0c8d0693cb65fd45b7e8c756937b5182ae Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Fri, 26 Jan 2024 13:23:35 +0100 Subject: [PATCH 084/117] s390/pai: save PAI counter value page in event structure When the PAI events ALL_CRYPTO or ALL_NNPA are created for system wide sampling, all PAI counters are monitored. On each process schedule out, the values of all PAI counters are investigated. Non-zero values are saved in the event's ring buffer as raw data. This scheme expects the start value of each counter to be reset to zero after each read operation performed by the PAI PMU device driver. This allows for only one active event at any one time as it relies on the start value of counters to be reset to zero. Create a save area for each installed PAI XXXX_ALL event and save all PAI counter values in this save area. Instead of clearing the PAI counter lowcore area to zero after each read operation, copy them from the lowcore area to the event's save area at process schedule out time. The delta of each PAI counter is calculated by subtracting the old counter's value stored in the event's save area from the current value stored in the lowcore area. With this scheme, mulitple events of the PAI counters XXXX_ALL can be handled at the same time. This will be addressed in a follow-on patch. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pai.h | 1 + arch/s390/kernel/perf_pai_crypto.c | 48 +++++++++++++++++++++++------- arch/s390/kernel/perf_pai_ext.c | 27 +++++++++++++---- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 7d1888e3dee6..c60123d3ad74 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -81,4 +81,5 @@ enum paievt_mode { PAI_MODE_COUNTING, }; +#define PAI_SAVE_AREA(x) ((x)->hw.event_base) #endif diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 7e7472aee4e5..8c98bd493707 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -98,6 +98,7 @@ static void paicrypt_event_destroy(struct perf_event *event) event->attr.config, event->cpu, cpump->active_events, cpump->mode, refcount_read(&cpump->refcnt)); + free_page(PAI_SAVE_AREA(event)); if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", __func__, (unsigned long)cpump->page, @@ -260,6 +261,7 @@ static int paicrypt_event_init(struct perf_event *event) { struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump; + int rc = 0; /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) @@ -274,10 +276,21 @@ static int paicrypt_event_init(struct perf_event *event) /* Allow only CRYPTO_ALL for sampling. */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) return -EINVAL; + /* Get a page to store last counter values for sampling */ + if (a->sample_period) { + PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); + if (!PAI_SAVE_AREA(event)) { + rc = -ENOMEM; + goto out; + } + } cpump = paicrypt_busy(event); - if (IS_ERR(cpump)) - return PTR_ERR(cpump); + if (IS_ERR(cpump)) { + free_page(PAI_SAVE_AREA(event)); + rc = PTR_ERR(cpump); + goto out; + } event->destroy = paicrypt_event_destroy; @@ -293,7 +306,8 @@ static int paicrypt_event_init(struct perf_event *event) } static_branch_inc(&pai_key); - return 0; +out: + return rc; } static void paicrypt_read(struct perf_event *event) @@ -367,23 +381,34 @@ static void paicrypt_del(struct perf_event *event, int flags) } } -/* Create raw data and save it in buffer. Returns number of bytes copied. - * Saves only positive counter entries of the form +/* Create raw data and save it in buffer. Calculate the delta for each + * counter between this invocation and the last invocation. + * Returns number of bytes copied. + * Saves only entries with positive counter difference of the form * 2 bytes: Number of counter * 8 bytes: Value of counter */ static size_t paicrypt_copy(struct pai_userdata *userdata, unsigned long *page, - bool exclude_user, bool exclude_kernel) + unsigned long *page_old, bool exclude_user, + bool exclude_kernel) { int i, outidx = 0; for (i = 1; i <= paicrypt_cnt; i++) { - u64 val = 0; + u64 val = 0, val_old = 0; - if (!exclude_kernel) + if (!exclude_kernel) { val += paicrypt_getctr(page, i, true); - if (!exclude_user) + val_old += paicrypt_getctr(page_old, i, true); + } + if (!exclude_user) { val += paicrypt_getctr(page, i, false); + val_old += paicrypt_getctr(page_old, i, false); + } + if (val >= val_old) + val -= val_old; + else + val = (~0ULL - val_old) + val + 1; if (val) { userdata[outidx].num = i; userdata[outidx].value = val; @@ -426,8 +451,8 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); - /* Clear lowcore page after read */ - memset(cpump->page, 0, PAGE_SIZE); + /* Save crypto counter lowcore page after reading event data. */ + memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); return overflow; } @@ -443,6 +468,7 @@ static int paicrypt_have_sample(void) if (!event) /* No event active */ return 0; rawsize = paicrypt_copy(cpump->save, cpump->page, + (unsigned long *)PAI_SAVE_AREA(event), cpump->event->attr.exclude_user, cpump->event->attr.exclude_kernel); if (rawsize) /* No incremented counters */ diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 4d86130bcf14..21a18eafb576 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -120,6 +120,7 @@ static void paiext_event_destroy(struct perf_event *event) struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); struct paiext_map *cpump = mp->mapptr; + free_page(PAI_SAVE_AREA(event)); mutex_lock(&paiext_reserve_mutex); cpump->event = NULL; if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ @@ -256,10 +257,18 @@ static int paiext_event_init(struct perf_event *event) /* Prohibit exclude_user event selection */ if (a->exclude_user) return -EINVAL; + /* Get a page to store last counter values for sampling */ + if (a->sample_period) { + PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL); + if (!PAI_SAVE_AREA(event)) + return -ENOMEM; + } rc = paiext_alloc(a, event); - if (rc) + if (rc) { + free_page(PAI_SAVE_AREA(event)); return rc; + } event->destroy = paiext_event_destroy; if (a->sample_period) { @@ -384,13 +393,19 @@ static void paiext_del(struct perf_event *event, int flags) * 2 bytes: Number of counter * 8 bytes: Value of counter */ -static size_t paiext_copy(struct pai_userdata *userdata, unsigned long *area) +static size_t paiext_copy(struct pai_userdata *userdata, unsigned long *area, + unsigned long *area_old) { int i, outidx = 0; for (i = 1; i <= paiext_cnt; i++) { u64 val = paiext_getctr(area, i); + u64 val_old = paiext_getctr(area_old, i); + if (val >= val_old) + val -= val_old; + else + val = (~0ULL - val_old) + val + 1; if (val) { userdata[outidx].num = i; userdata[outidx].value = val; @@ -446,8 +461,9 @@ static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump, overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); - /* Clear lowcore area after read */ - memset(cpump->area, 0, PAIE1_CTRBLOCK_SZ); + /* Save NNPA lowcore area after read in event */ + memcpy((void *)PAI_SAVE_AREA(event), cpump->area, + PAIE1_CTRBLOCK_SZ); return overflow; } @@ -462,7 +478,8 @@ static int paiext_have_sample(void) if (!event) return 0; - rawsize = paiext_copy(cpump->save, cpump->area); + rawsize = paiext_copy(cpump->save, cpump->area, + (unsigned long *)PAI_SAVE_AREA(event)); if (rawsize) /* Incremented counters */ rc = paiext_push_sample(rawsize, cpump, event); return rc; From 82cb9b618531109824eb8e25b6a3da28d33178ac Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Fri, 26 Jan 2024 13:32:03 +0100 Subject: [PATCH 085/117] s390/pai: simplify event start function for perf stat When an event is started, read the current value of the PAI counter. This value is saved in event::hw.prev_count. When an event is stopped, this value is subtracted from the current value read out at event stop time. The difference is the delta of this counter. Simplify the logic and read the event value every time the event is started. This scheme is identical to other device drivers. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 12 ++---------- arch/s390/kernel/perf_pai_ext.c | 7 ++----- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 8c98bd493707..61fb4e716476 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -326,17 +326,9 @@ static void paicrypt_start(struct perf_event *event, int flags) { u64 sum; - /* Event initialization sets last_tag to 0. When later on the events - * are deleted and re-added, do not reset the event count value to zero. - * Events are added, deleted and re-added when 2 or more events - * are active at the same time. - */ if (!event->attr.sample_period) { /* Counting */ - if (!event->hw.last_tag) { - event->hw.last_tag = 1; - sum = paicrypt_getall(event); /* Get current value */ - local64_set(&event->hw.prev_count, sum); - } + sum = paicrypt_getall(event); /* Get current value */ + local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ perf_sched_cb_inc(event->pmu); } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 21a18eafb576..dd84c4d6fbe9 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -331,11 +331,8 @@ static void paiext_start(struct perf_event *event, int flags) u64 sum; if (!event->attr.sample_period) { /* Counting */ - if (!event->hw.last_tag) { - event->hw.last_tag = 1; - sum = paiext_getall(event); /* Get current value */ - local64_set(&event->hw.prev_count, sum); - } + sum = paiext_getall(event); /* Get current value */ + local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ perf_sched_cb_inc(event->pmu); } From 29f6fe17f3eb14d490ba27f8b9d0cbd766bd9158 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 8 Jan 2024 12:29:24 +0100 Subject: [PATCH 086/117] s390/pai: adjust whitespace indentation Adjust whitespace indentation. No functional change. Signed-off-by: Thomas Richter Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pai.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index c60123d3ad74..3f609565734b 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -16,7 +16,7 @@ struct qpaci_info_block { u64 header; struct { u64 : 8; - u64 num_cc : 8; /* # of supported crypto counters */ + u64 num_cc : 8; /* # of supported crypto counters */ u64 : 9; u64 num_nnpa : 7; /* # of supported NNPA counters */ u64 : 32; From 0d48566d4b58946c8e1b0baac0347616060a81c9 Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Tue, 9 Jan 2024 17:22:39 +0100 Subject: [PATCH 087/117] s390/pci: rename lock member in struct zpci_dev Since this guards only the Function Measurement Block, rename from generic lock to fmb_lock in preparation to introduce another lock that guards the state member Signed-off-by: Gerd Bayer Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pci.h | 2 +- arch/s390/pci/pci.c | 2 +- arch/s390/pci/pci_debug.c | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index e91cd6bbc330..a54c2cc88c47 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -142,7 +142,6 @@ struct zpci_dev { u8 reserved : 2; unsigned int devfn; /* DEVFN part of the RID*/ - struct mutex lock; u8 pfip[CLP_PFIP_NR_SEGMENTS]; /* pci function internal path */ u32 uid; /* user defined id */ u8 util_str[CLP_UTIL_STR_LEN]; /* utility string */ @@ -170,6 +169,7 @@ struct zpci_dev { u64 dma_mask; /* DMA address space mask */ /* Function measurement block */ + struct mutex fmb_lock; struct zpci_fmb *fmb; u16 fmb_update; /* update interval */ u16 fmb_length; diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 676ac74026a8..dff609e8a2a0 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -806,7 +806,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) zdev->state = state; kref_init(&zdev->kref); - mutex_init(&zdev->lock); + mutex_init(&zdev->fmb_lock); mutex_init(&zdev->kzdev_lock); rc = zpci_init_iommu(zdev); diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index 6dde2263c79d..2cb5043a997d 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -91,9 +91,9 @@ static int pci_perf_show(struct seq_file *m, void *v) if (!zdev) return 0; - mutex_lock(&zdev->lock); + mutex_lock(&zdev->fmb_lock); if (!zdev->fmb) { - mutex_unlock(&zdev->lock); + mutex_unlock(&zdev->fmb_lock); seq_puts(m, "FMB statistics disabled\n"); return 0; } @@ -130,7 +130,7 @@ static int pci_perf_show(struct seq_file *m, void *v) } pci_sw_counter_show(m); - mutex_unlock(&zdev->lock); + mutex_unlock(&zdev->fmb_lock); return 0; } @@ -148,7 +148,7 @@ static ssize_t pci_perf_seq_write(struct file *file, const char __user *ubuf, if (rc) return rc; - mutex_lock(&zdev->lock); + mutex_lock(&zdev->fmb_lock); switch (val) { case 0: rc = zpci_fmb_disable_device(zdev); @@ -157,7 +157,7 @@ static ssize_t pci_perf_seq_write(struct file *file, const char __user *ubuf, rc = zpci_fmb_enable_device(zdev); break; } - mutex_unlock(&zdev->lock); + mutex_unlock(&zdev->fmb_lock); return rc ? rc : count; } From bcb5d6c769039c8358a2359e7c3ea5d97ce93108 Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Fri, 10 Nov 2023 16:27:06 +0100 Subject: [PATCH 088/117] s390/pci: introduce lock to synchronize state of zpci_dev's There's a number of tasks that need the state of a zpci device to be stable. Other tasks need to be synchronized as they change the state. State changes could be generated by the system as availability or error events, or be requested by the user through manipulations in sysfs. Some other actions accessible through sysfs - like device resets - need the state to be stable. Unsynchronized state handling could lead to unusable devices. This has been observed in cases of concurrent state changes through systemd udev rules and DPM boot control. Some breakage can be provoked by artificial tests, e.g. through repetitively injecting "recover" on a PCI function through sysfs while running a "hotplug remove/add" in a loop through a PCI slot's "power" attribute in sysfs. After a few iterations this could result in a kernel oops. So introduce a new mutex "state_lock" to guard the state property of the struct zpci_dev. Acquire this lock in all task that modify the state: - hotplug add and remove, through the PCI hotplug slot entry, - avaiability events, as reported by the platform, - error events, as reported by the platform, - during device resets, explicit through sysfs requests or implict through the common PCI layer. Break out an inner _do_recover() routine out of recover_store() to separte the necessary synchronizations from the actual manipulations of the zpci_dev required for the reset. With the following changes I was able to run the inject loops for hours without hitting an error. Signed-off-by: Gerd Bayer Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pci.h | 1 + arch/s390/pci/pci.c | 11 ++++- arch/s390/pci/pci_event.c | 11 ++++- arch/s390/pci/pci_sysfs.c | 70 ++++++++++++++++++------------ drivers/pci/hotplug/s390_pci_hpc.c | 65 +++++++++++++++++---------- 5 files changed, 106 insertions(+), 52 deletions(-) diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index a54c2cc88c47..30820a649e6e 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -122,6 +122,7 @@ struct zpci_dev { struct rcu_head rcu; struct hotplug_slot hotplug_slot; + struct mutex state_lock; /* protect state changes */ enum zpci_state state; u32 fid; /* function ID, used by sclp */ u32 fh; /* function handle, used by insn's */ diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index dff609e8a2a0..17267f659d22 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -730,12 +731,12 @@ EXPORT_SYMBOL_GPL(zpci_disable_device); * equivalent to its state during boot when first probing a driver. * Consequently after reset the PCI function requires re-initialization via the * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors() - * and enabling the function via e.g.pci_enablde_device_flags().The caller + * and enabling the function via e.g. pci_enable_device_flags(). The caller * must guard against concurrent reset attempts. * * In most cases this function should not be called directly but through * pci_reset_function() or pci_reset_bus() which handle the save/restore and - * locking. + * locking - asserted by lockdep. * * Return: 0 on success and an error value otherwise */ @@ -744,6 +745,7 @@ int zpci_hot_reset_device(struct zpci_dev *zdev) u8 status; int rc; + lockdep_assert_held(&zdev->state_lock); zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh); if (zdev_enabled(zdev)) { /* Disables device access, DMAs and IRQs (reset state) */ @@ -806,6 +808,7 @@ struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) zdev->state = state; kref_init(&zdev->kref); + mutex_init(&zdev->state_lock); mutex_init(&zdev->fmb_lock); mutex_init(&zdev->kzdev_lock); @@ -870,6 +873,10 @@ int zpci_deconfigure_device(struct zpci_dev *zdev) { int rc; + lockdep_assert_held(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + return 0; + if (zdev->zbus->bus) zpci_bus_remove_device(zdev, false); diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 4d9773ef9e0a..a17804b0dab4 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -267,6 +267,7 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) zpci_err_hex(ccdf, sizeof(*ccdf)); if (zdev) { + mutex_lock(&zdev->state_lock); zpci_update_fh(zdev, ccdf->fh); if (zdev->zbus->bus) pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); @@ -294,6 +295,8 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) } pci_dev_put(pdev); no_pdev: + if (zdev) + mutex_unlock(&zdev->state_lock); zpci_zdev_put(zdev); } @@ -326,6 +329,10 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", ccdf->fid, ccdf->fh, ccdf->pec); + + if (existing_zdev) + mutex_lock(&zdev->state_lock); + switch (ccdf->pec) { case 0x0301: /* Reserved|Standby -> Configured */ if (!zdev) { @@ -383,8 +390,10 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) default: break; } - if (existing_zdev) + if (existing_zdev) { + mutex_unlock(&zdev->state_lock); zpci_zdev_put(zdev); + } } void zpci_event_availability(void *data) diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c index 8a7abac51816..a0b872b74fe3 100644 --- a/arch/s390/pci/pci_sysfs.c +++ b/arch/s390/pci/pci_sysfs.c @@ -49,6 +49,39 @@ static ssize_t mio_enabled_show(struct device *dev, } static DEVICE_ATTR_RO(mio_enabled); +static int _do_recover(struct pci_dev *pdev, struct zpci_dev *zdev) +{ + u8 status; + int ret; + + pci_stop_and_remove_bus_device(pdev); + if (zdev_enabled(zdev)) { + ret = zpci_disable_device(zdev); + /* + * Due to a z/VM vs LPAR inconsistency in the error + * state the FH may indicate an enabled device but + * disable says the device is already disabled don't + * treat it as an error here. + */ + if (ret == -EINVAL) + ret = 0; + if (ret) + return ret; + } + + ret = zpci_enable_device(zdev); + if (ret) + return ret; + + if (zdev->dma_table) { + ret = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, + virt_to_phys(zdev->dma_table), &status); + if (ret) + zpci_disable_device(zdev); + } + return ret; +} + static ssize_t recover_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -56,7 +89,6 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr, struct pci_dev *pdev = to_pci_dev(dev); struct zpci_dev *zdev = to_zpci(pdev); int ret = 0; - u8 status; /* Can't use device_remove_self() here as that would lead us to lock * the pci_rescan_remove_lock while holding the device' kernfs lock. @@ -70,6 +102,12 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr, */ kn = sysfs_break_active_protection(&dev->kobj, &attr->attr); WARN_ON_ONCE(!kn); + + /* Device needs to be configured and state must not change */ + mutex_lock(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + goto out; + /* device_remove_file() serializes concurrent calls ignoring all but * the first */ @@ -82,35 +120,13 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr, */ pci_lock_rescan_remove(); if (pci_dev_is_added(pdev)) { - pci_stop_and_remove_bus_device(pdev); - if (zdev_enabled(zdev)) { - ret = zpci_disable_device(zdev); - /* - * Due to a z/VM vs LPAR inconsistency in the error - * state the FH may indicate an enabled device but - * disable says the device is already disabled don't - * treat it as an error here. - */ - if (ret == -EINVAL) - ret = 0; - if (ret) - goto out; - } - - ret = zpci_enable_device(zdev); - if (ret) - goto out; - - if (zdev->dma_table) { - ret = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, - virt_to_phys(zdev->dma_table), &status); - if (ret) - zpci_disable_device(zdev); - } + ret = _do_recover(pdev, zdev); } -out: pci_rescan_bus(zdev->zbus->bus); pci_unlock_rescan_remove(); + +out: + mutex_unlock(&zdev->state_lock); if (kn) sysfs_unbreak_active_protection(kn); return ret ? ret : count; diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c index a89b7de72dcf..7333b305f2a5 100644 --- a/drivers/pci/hotplug/s390_pci_hpc.c +++ b/drivers/pci/hotplug/s390_pci_hpc.c @@ -26,58 +26,79 @@ static int enable_slot(struct hotplug_slot *hotplug_slot) hotplug_slot); int rc; - if (zdev->state != ZPCI_FN_STATE_STANDBY) - return -EIO; + mutex_lock(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_STANDBY) { + rc = -EIO; + goto out; + } rc = sclp_pci_configure(zdev->fid); zpci_dbg(3, "conf fid:%x, rc:%d\n", zdev->fid, rc); if (rc) - return rc; + goto out; zdev->state = ZPCI_FN_STATE_CONFIGURED; - return zpci_scan_configured_device(zdev, zdev->fh); + rc = zpci_scan_configured_device(zdev, zdev->fh); +out: + mutex_unlock(&zdev->state_lock); + return rc; } static int disable_slot(struct hotplug_slot *hotplug_slot) { struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev, hotplug_slot); - struct pci_dev *pdev; + struct pci_dev *pdev = NULL; + int rc; - if (zdev->state != ZPCI_FN_STATE_CONFIGURED) - return -EIO; + mutex_lock(&zdev->state_lock); + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) { + rc = -EIO; + goto out; + } pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); if (pdev && pci_num_vf(pdev)) { pci_dev_put(pdev); - return -EBUSY; + rc = -EBUSY; + goto out; } - pci_dev_put(pdev); - return zpci_deconfigure_device(zdev); + rc = zpci_deconfigure_device(zdev); +out: + mutex_unlock(&zdev->state_lock); + if (pdev) + pci_dev_put(pdev); + return rc; } static int reset_slot(struct hotplug_slot *hotplug_slot, bool probe) { struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev, hotplug_slot); + int rc = -EIO; - if (zdev->state != ZPCI_FN_STATE_CONFIGURED) - return -EIO; /* - * We can't take the zdev->lock as reset_slot may be called during - * probing and/or device removal which already happens under the - * zdev->lock. Instead the user should use the higher level - * pci_reset_function() or pci_bus_reset() which hold the PCI device - * lock preventing concurrent removal. If not using these functions - * holding the PCI device lock is required. + * If we can't get the zdev->state_lock the device state is + * currently undergoing a transition and we bail out - just + * the same as if the device's state is not configured at all. */ + if (!mutex_trylock(&zdev->state_lock)) + return rc; - /* As long as the function is configured we can reset */ - if (probe) - return 0; + /* We can reset only if the function is configured */ + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + goto out; - return zpci_hot_reset_device(zdev); + if (probe) { + rc = 0; + goto out; + } + + rc = zpci_hot_reset_device(zdev); +out: + mutex_unlock(&zdev->state_lock); + return rc; } static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value) From 6ee600bfbe0f818ffb7748d99e9b0c89d0d9f02a Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Fri, 10 Nov 2023 12:06:42 +0100 Subject: [PATCH 089/117] s390/pci: remove hotplug slot when releasing the device Centralize the removal so all paths are covered and the hotplug slot will remain active until the device is really destroyed. Signed-off-by: Gerd Bayer Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 17267f659d22..c87b8aff5285 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -906,8 +906,6 @@ int zpci_deconfigure_device(struct zpci_dev *zdev) */ void zpci_device_reserved(struct zpci_dev *zdev) { - if (zdev->has_hp_slot) - zpci_exit_slot(zdev); /* * Remove device from zpci_list as it is going away. This also * makes sure we ignore subsequent zPCI events for this device. @@ -925,6 +923,9 @@ void zpci_release_device(struct kref *kref) struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref); int ret; + if (zdev->has_hp_slot) + zpci_exit_slot(zdev); + if (zdev->zbus->bus) zpci_bus_remove_device(zdev, false); From d0c8fd21006777b3952263973237fcd82e049ec4 Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Tue, 5 Dec 2023 18:39:14 +0100 Subject: [PATCH 090/117] s390/pci: fix three typos in comments Found and fixed these while working on synchronizing the state handling of zpci_dev's. Signed-off-by: Gerd Bayer Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci.c | 2 +- arch/s390/pci/pci_event.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index c87b8aff5285..9e80945fb11d 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -896,7 +896,7 @@ int zpci_deconfigure_device(struct zpci_dev *zdev) } /** - * zpci_device_reserved() - Mark device as resverved + * zpci_device_reserved() - Mark device as reserved * @zdev: the zpci_dev that was reserved * * Handle the case that a given zPCI function was reserved by another system. diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index a17804b0dab4..dbe95ec5917e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -355,7 +355,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) break; case 0x0303: /* Deconfiguration requested */ if (zdev) { - /* The event may have been queued before we confirgured + /* The event may have been queued before we configured * the device. */ if (zdev->state != ZPCI_FN_STATE_CONFIGURED) @@ -366,7 +366,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) break; case 0x0304: /* Configured -> Standby|Reserved */ if (zdev) { - /* The event may have been queued before we confirgured + /* The event may have been queued before we configured * the device.: */ if (zdev->state == ZPCI_FN_STATE_CONFIGURED) From 9ea30fd166e9b869546059f714beab78cb150b11 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Fri, 16 Feb 2024 12:55:53 -0700 Subject: [PATCH 091/117] s390/boot: add 'alloc' to info.bin .vmlinux.info section flags When attempting to boot a kernel compiled with OBJCOPY=llvm-objcopy, there is a crash right at boot: Out of memory allocating 6d7800 bytes 8 aligned in range 0:20000000 Reserved memory ranges: 0000000000000000 a394c3c30d90cdaf DECOMPRESSOR Usable online memory ranges (info source: sclp read info [3]): 0000000000000000 0000000020000000 Usable online memory total: 20000000 Reserved: a394c3c30d90cdaf Free: 0 Call Trace: (sp:0000000000033e90 [<0000000000012fbc>] physmem_alloc_top_down+0x5c/0x104) sp:0000000000033f00 [<0000000000011d56>] startup_kernel+0x3a6/0x77c sp:0000000000033f60 [<00000000000100f4>] startup_normal+0xd4/0xd4 GNU objcopy does not have any issues. Looking at differences between the object files in each build reveals info.bin does not get properly populated with llvm-objcopy, which results in an empty .vmlinux.info section. $ file {gnu,llvm}-objcopy/arch/s390/boot/info.bin gnu-objcopy/arch/s390/boot/info.bin: data llvm-objcopy/arch/s390/boot/info.bin: empty $ llvm-readelf --section-headers {gnu,llvm}-objcopy/arch/s390/boot/vmlinux | rg 'File:|\.vmlinux\.info|\.decompressor\.syms' File: gnu-objcopy/arch/s390/boot/vmlinux [12] .vmlinux.info PROGBITS 0000000000034000 035000 000078 00 WA 0 0 1 [13] .decompressor.syms PROGBITS 0000000000034078 035078 000b00 00 WA 0 0 1 File: llvm-objcopy/arch/s390/boot/vmlinux [12] .vmlinux.info PROGBITS 0000000000034000 035000 000000 00 WA 0 0 1 [13] .decompressor.syms PROGBITS 0000000000034000 035000 000b00 00 WA 0 0 1 Ulrich points out that llvm-objcopy only copies sections marked as alloc with a binary output target, whereas the .vmlinux.info section is only marked as load. Add 'alloc' in addition to 'load', so that both objcopy implementations work properly: $ file {gnu,llvm}-objcopy/arch/s390/boot/info.bin gnu-objcopy/arch/s390/boot/info.bin: data llvm-objcopy/arch/s390/boot/info.bin: data $ llvm-readelf --section-headers {gnu,llvm}-objcopy/arch/s390/boot/vmlinux | rg 'File:|\.vmlinux\.info|\.decompressor\.syms' File: gnu-objcopy/arch/s390/boot/vmlinux [12] .vmlinux.info PROGBITS 0000000000034000 035000 000078 00 WA 0 0 1 [13] .decompressor.syms PROGBITS 0000000000034078 035078 000b00 00 WA 0 0 1 File: llvm-objcopy/arch/s390/boot/vmlinux [12] .vmlinux.info PROGBITS 0000000000034000 035000 000078 00 WA 0 0 1 [13] .decompressor.syms PROGBITS 0000000000034078 035078 000b00 00 WA 0 0 1 Closes: https://github.com/ClangBuiltLinux/linux/issues/1996 Link: https://github.com/llvm/llvm-project/commit/3c02cb7492fc78fb678264cebf57ff88e478e14f Suggested-by: Ulrich Weigand Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240216-s390-fix-boot-with-llvm-objcopy-v1-1-0ac623daf42b@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index ace0bda1ad24..d40135efdec4 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -94,7 +94,7 @@ OBJCOPYFLAGS_syms.o := -I binary -O elf64-s390 -B s390:64-bit --rename-section . $(obj)/syms.o: $(obj)/syms.bin FORCE $(call if_changed,objcopy) -OBJCOPYFLAGS_info.bin := -O binary --only-section=.vmlinux.info --set-section-flags .vmlinux.info=load +OBJCOPYFLAGS_info.bin := -O binary --only-section=.vmlinux.info --set-section-flags .vmlinux.info=alloc,load $(obj)/info.bin: vmlinux FORCE $(call if_changed,objcopy) From 8192a1b3807510d0ed5be1f8988c08f8d41cced9 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Mon, 19 Feb 2024 14:27:31 +0100 Subject: [PATCH 092/117] s390/vdso64: filter out munaligned-symbols flag for vdso Gcc recently implemented an optimization [1] for loading symbols without explicit alignment, aligning with the IBM Z ELF ABI. This ABI mandates symbols to reside on a 2-byte boundary, enabling the use of the larl instruction. However, kernel linker scripts may still generate unaligned symbols. To address this, a new -munaligned-symbols option has been introduced [2] in recent gcc versions. [1] https://gcc.gnu.org/pipermail/gcc-patches/2023-June/622872.html [2] https://gcc.gnu.org/pipermail/gcc-patches/2023-August/625986.html However, when -munaligned-symbols is used in vdso code, it leads to the following compilation error: `.data.rel.ro.local' referenced in section `.text' of arch/s390/kernel/vdso64/vdso64_generic.o: defined in discarded section `.data.rel.ro.local' of arch/s390/kernel/vdso64/vdso64_generic.o vdso linker script discards .data section to make it lightweight. However, -munaligned-symbols in vdso object files references literal pool and accesses _vdso_data. Hence, compile vdso code without -munaligned-symbols. This means in the future, vdso code should deal with alignment of newly introduced unaligned linker symbols. Acked-by: Vasily Gorbik Signed-off-by: Sumanth Korikkar Link: https://lore.kernel.org/r/20240219132734.22881-2-sumanthk@linux.ibm.com Signed-off-by: Heiko Carstens --- arch/s390/kernel/vdso64/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index caa4ebff8a19..ef9832726097 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -25,6 +25,7 @@ KBUILD_AFLAGS_64 += -m64 KBUILD_CFLAGS_64 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_64 := $(filter-out -mno-pic-data-is-text-relative,$(KBUILD_CFLAGS_64)) +KBUILD_CFLAGS_64 := $(filter-out -munaligned-symbols,$(KBUILD_CFLAGS_64)) KBUILD_CFLAGS_64 += -m64 -fPIC -fno-common -fno-builtin ldflags-y := -shared -soname=linux-vdso64.so.1 \ --hash-style=both --build-id=sha1 -T From 55dc65b46023540d5136dcd1f3076661f850dd99 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 19 Feb 2024 14:27:32 +0100 Subject: [PATCH 093/117] s390: add relocs tool This 'relocs' tool is copied from the x86 version, ported for s390, and greatly simplified to remove unnecessary features. It reads vmlinux and outputs assembly to create a .vmlinux.relocs_64 section which contains the offsets of all R_390_64 relocations which apply to allocatable sections. Acked-by: Vasily Gorbik Signed-off-by: Josh Poimboeuf Signed-off-by: Sumanth Korikkar Link: https://lore.kernel.org/r/20240219132734.22881-3-sumanthk@linux.ibm.com Signed-off-by: Heiko Carstens --- arch/s390/tools/.gitignore | 1 + arch/s390/tools/Makefile | 5 + arch/s390/tools/relocs.c | 385 +++++++++++++++++++++++++++++++++++++ 3 files changed, 391 insertions(+) create mode 100644 arch/s390/tools/relocs.c diff --git a/arch/s390/tools/.gitignore b/arch/s390/tools/.gitignore index ea62f37b79ef..e6af51d9d183 100644 --- a/arch/s390/tools/.gitignore +++ b/arch/s390/tools/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only gen_facilities gen_opcode_table +relocs diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index f9dd47ff9ac4..f2862364fb42 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -25,3 +25,8 @@ $(kapi)/facility-defs.h: $(obj)/gen_facilities FORCE $(kapi)/dis-defs.h: $(obj)/gen_opcode_table FORCE $(call filechk,dis-defs.h) + +hostprogs += relocs +PHONY += relocs +relocs: $(obj)/relocs + @: diff --git a/arch/s390/tools/relocs.c b/arch/s390/tools/relocs.c new file mode 100644 index 000000000000..db8bcbf9d8f8 --- /dev/null +++ b/arch/s390/tools/relocs.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define USE_BSD +#include + +#define ELF_BITS 64 + +#define ELF_MACHINE EM_S390 +#define ELF_MACHINE_NAME "IBM S/390" +#define SHT_REL_TYPE SHT_RELA +#define Elf_Rel Elf64_Rela + +#define ELF_CLASS ELFCLASS64 +#define ELF_ENDIAN ELFDATA2MSB +#define ELF_R_SYM(val) ELF64_R_SYM(val) +#define ELF_R_TYPE(val) ELF64_R_TYPE(val) +#define ELF_ST_TYPE(o) ELF64_ST_TYPE(o) +#define ELF_ST_BIND(o) ELF64_ST_BIND(o) +#define ELF_ST_VISIBILITY(o) ELF64_ST_VISIBILITY(o) + +#define ElfW(type) _ElfW(ELF_BITS, type) +#define _ElfW(bits, type) __ElfW(bits, type) +#define __ElfW(bits, type) Elf##bits##_##type + +#define Elf_Addr ElfW(Addr) +#define Elf_Ehdr ElfW(Ehdr) +#define Elf_Phdr ElfW(Phdr) +#define Elf_Shdr ElfW(Shdr) +#define Elf_Sym ElfW(Sym) + +static Elf_Ehdr ehdr; +static unsigned long shnum; +static unsigned int shstrndx; + +struct relocs { + uint32_t *offset; + unsigned long count; + unsigned long size; +}; + +static struct relocs relocs64; +#define FMT PRIu64 + +struct section { + Elf_Shdr shdr; + struct section *link; + Elf_Rel *reltab; +}; + +static struct section *secs; + +#if BYTE_ORDER == LITTLE_ENDIAN +#define le16_to_cpu(val) (val) +#define le32_to_cpu(val) (val) +#define le64_to_cpu(val) (val) +#define be16_to_cpu(val) bswap_16(val) +#define be32_to_cpu(val) bswap_32(val) +#define be64_to_cpu(val) bswap_64(val) +#endif + +#if BYTE_ORDER == BIG_ENDIAN +#define le16_to_cpu(val) bswap_16(val) +#define le32_to_cpu(val) bswap_32(val) +#define le64_to_cpu(val) bswap_64(val) +#define be16_to_cpu(val) (val) +#define be32_to_cpu(val) (val) +#define be64_to_cpu(val) (val) +#endif + +static uint16_t elf16_to_cpu(uint16_t val) +{ + if (ehdr.e_ident[EI_DATA] == ELFDATA2LSB) + return le16_to_cpu(val); + else + return be16_to_cpu(val); +} + +static uint32_t elf32_to_cpu(uint32_t val) +{ + if (ehdr.e_ident[EI_DATA] == ELFDATA2LSB) + return le32_to_cpu(val); + else + return be32_to_cpu(val); +} + +#define elf_half_to_cpu(x) elf16_to_cpu(x) +#define elf_word_to_cpu(x) elf32_to_cpu(x) + +static uint64_t elf64_to_cpu(uint64_t val) +{ + return be64_to_cpu(val); +} + +#define elf_addr_to_cpu(x) elf64_to_cpu(x) +#define elf_off_to_cpu(x) elf64_to_cpu(x) +#define elf_xword_to_cpu(x) elf64_to_cpu(x) + +static void die(char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + exit(1); +} + +static void read_ehdr(FILE *fp) +{ + if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1) + die("Cannot read ELF header: %s\n", strerror(errno)); + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0) + die("No ELF magic\n"); + if (ehdr.e_ident[EI_CLASS] != ELF_CLASS) + die("Not a %d bit executable\n", ELF_BITS); + if (ehdr.e_ident[EI_DATA] != ELF_ENDIAN) + die("ELF endian mismatch\n"); + if (ehdr.e_ident[EI_VERSION] != EV_CURRENT) + die("Unknown ELF version\n"); + + /* Convert the fields to native endian */ + ehdr.e_type = elf_half_to_cpu(ehdr.e_type); + ehdr.e_machine = elf_half_to_cpu(ehdr.e_machine); + ehdr.e_version = elf_word_to_cpu(ehdr.e_version); + ehdr.e_entry = elf_addr_to_cpu(ehdr.e_entry); + ehdr.e_phoff = elf_off_to_cpu(ehdr.e_phoff); + ehdr.e_shoff = elf_off_to_cpu(ehdr.e_shoff); + ehdr.e_flags = elf_word_to_cpu(ehdr.e_flags); + ehdr.e_ehsize = elf_half_to_cpu(ehdr.e_ehsize); + ehdr.e_phentsize = elf_half_to_cpu(ehdr.e_phentsize); + ehdr.e_phnum = elf_half_to_cpu(ehdr.e_phnum); + ehdr.e_shentsize = elf_half_to_cpu(ehdr.e_shentsize); + ehdr.e_shnum = elf_half_to_cpu(ehdr.e_shnum); + ehdr.e_shstrndx = elf_half_to_cpu(ehdr.e_shstrndx); + + shnum = ehdr.e_shnum; + shstrndx = ehdr.e_shstrndx; + + if ((ehdr.e_type != ET_EXEC) && (ehdr.e_type != ET_DYN)) + die("Unsupported ELF header type\n"); + if (ehdr.e_machine != ELF_MACHINE) + die("Not for %s\n", ELF_MACHINE_NAME); + if (ehdr.e_version != EV_CURRENT) + die("Unknown ELF version\n"); + if (ehdr.e_ehsize != sizeof(Elf_Ehdr)) + die("Bad Elf header size\n"); + if (ehdr.e_phentsize != sizeof(Elf_Phdr)) + die("Bad program header entry\n"); + if (ehdr.e_shentsize != sizeof(Elf_Shdr)) + die("Bad section header entry\n"); + + if (shnum == SHN_UNDEF || shstrndx == SHN_XINDEX) { + Elf_Shdr shdr; + + if (fseek(fp, ehdr.e_shoff, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", ehdr.e_shoff, strerror(errno)); + + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot read initial ELF section header: %s\n", strerror(errno)); + + if (shnum == SHN_UNDEF) + shnum = elf_xword_to_cpu(shdr.sh_size); + + if (shstrndx == SHN_XINDEX) + shstrndx = elf_word_to_cpu(shdr.sh_link); + } + + if (shstrndx >= shnum) + die("String table index out of bounds\n"); +} + +static void read_shdrs(FILE *fp) +{ + Elf_Shdr shdr; + int i; + + secs = calloc(shnum, sizeof(struct section)); + if (!secs) + die("Unable to allocate %ld section headers\n", shnum); + + if (fseek(fp, ehdr.e_shoff, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", ehdr.e_shoff, strerror(errno)); + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) { + die("Cannot read ELF section headers %d/%ld: %s\n", + i, shnum, strerror(errno)); + } + + sec->shdr.sh_name = elf_word_to_cpu(shdr.sh_name); + sec->shdr.sh_type = elf_word_to_cpu(shdr.sh_type); + sec->shdr.sh_flags = elf_xword_to_cpu(shdr.sh_flags); + sec->shdr.sh_addr = elf_addr_to_cpu(shdr.sh_addr); + sec->shdr.sh_offset = elf_off_to_cpu(shdr.sh_offset); + sec->shdr.sh_size = elf_xword_to_cpu(shdr.sh_size); + sec->shdr.sh_link = elf_word_to_cpu(shdr.sh_link); + sec->shdr.sh_info = elf_word_to_cpu(shdr.sh_info); + sec->shdr.sh_addralign = elf_xword_to_cpu(shdr.sh_addralign); + sec->shdr.sh_entsize = elf_xword_to_cpu(shdr.sh_entsize); + + if (sec->shdr.sh_link < shnum) + sec->link = &secs[sec->shdr.sh_link]; + } + +} + +static void read_relocs(FILE *fp) +{ + int i, j; + + for (i = 0; i < shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec->reltab = malloc(sec->shdr.sh_size); + if (!sec->reltab) + die("malloc of %" FMT " bytes for relocs failed\n", sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %" FMT " failed: %s\n", sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->reltab, 1, sec->shdr.sh_size, fp) != sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < sec->shdr.sh_size / sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + + rel->r_offset = elf_addr_to_cpu(rel->r_offset); + rel->r_info = elf_xword_to_cpu(rel->r_info); +#if (SHT_REL_TYPE == SHT_RELA) + rel->r_addend = elf_xword_to_cpu(rel->r_addend); +#endif + } + } +} + +static void add_reloc(struct relocs *r, uint32_t offset) +{ + if (r->count == r->size) { + unsigned long newsize = r->size + 50000; + void *mem = realloc(r->offset, newsize * sizeof(r->offset[0])); + + if (!mem) + die("realloc of %ld entries for relocs failed\n", newsize); + + r->offset = mem; + r->size = newsize; + } + r->offset[r->count++] = offset; +} + +static int do_reloc(struct section *sec, Elf_Rel *rel) +{ + unsigned int r_type = ELF64_R_TYPE(rel->r_info); + ElfW(Addr) offset = rel->r_offset; + + switch (r_type) { + case R_390_NONE: + case R_390_PC32: + case R_390_PC64: + case R_390_PC16DBL: + case R_390_PC32DBL: + case R_390_PLT32DBL: + case R_390_GOTENT: + break; + case R_390_64: + add_reloc(&relocs64, offset); + break; + default: + die("Unsupported relocation type: %d\n", r_type); + break; + } + + return 0; +} + +static void walk_relocs(void) +{ + int i; + + /* Walk through the relocations */ + for (i = 0; i < shnum; i++) { + struct section *sec_applies; + int j; + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec_applies = &secs[sec->shdr.sh_info]; + if (!(sec_applies->shdr.sh_flags & SHF_ALLOC)) + continue; + + for (j = 0; j < sec->shdr.sh_size / sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + + do_reloc(sec, rel); + } + } +} + +static int cmp_relocs(const void *va, const void *vb) +{ + const uint32_t *a, *b; + + a = va; b = vb; + return (*a == *b) ? 0 : (*a > *b) ? 1 : -1; +} + +static void sort_relocs(struct relocs *r) +{ + qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs); +} + +static int print_reloc(uint32_t v) +{ + return fprintf(stdout, "\t.long 0x%08"PRIx32"\n", v) > 0 ? 0 : -1; +} + +static void emit_relocs(void) +{ + int i; + + walk_relocs(); + sort_relocs(&relocs64); + + printf(".section \".vmlinux.relocs_64\",\"a\"\n"); + for (i = 0; i < relocs64.count; i++) + print_reloc(relocs64.offset[i]); +} + +static void process(FILE *fp) +{ + read_ehdr(fp); + read_shdrs(fp); + read_relocs(fp); + emit_relocs(); +} + +static void usage(void) +{ + die("relocs vmlinux\n"); +} + +int main(int argc, char **argv) +{ + unsigned char e_ident[EI_NIDENT]; + const char *fname; + FILE *fp; + + fname = NULL; + + if (argc != 2) + usage(); + + fname = argv[1]; + + fp = fopen(fname, "r"); + if (!fp) + die("Cannot open %s: %s\n", fname, strerror(errno)); + + if (fread(&e_ident, 1, EI_NIDENT, fp) != EI_NIDENT) + die("Cannot read %s: %s", fname, strerror(errno)); + + rewind(fp); + + process(fp); + + fclose(fp); + return 0; +} From 778666df60f0d96f215e33e27448de47a2207fb3 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 19 Feb 2024 14:27:33 +0100 Subject: [PATCH 094/117] s390: compile relocatable kernel without -fPIE On s390, currently kernel uses the '-fPIE' compiler flag for compiling vmlinux. This has a few problems: - It uses dynamic symbols (.dynsym), for which the linker refuses to allow more than 64k sections. This can break features which use '-ffunction-sections' and '-fdata-sections', including kpatch-build [1] and Function Granular KASLR. - It unnecessarily uses GOT relocations, adding an extra layer of indirection for many memory accesses. Instead of using '-fPIE', resolve all the relocations at link time and then manually adjust any absolute relocations (R_390_64) during boot. This is done by first telling the linker to preserve all relocations during the vmlinux link. (Note this is harmless: they are later stripped in the vmlinux.bin link.) Then use the 'relocs' tool to find all absolute relocations (R_390_64) which apply to allocatable sections. The offsets of those relocations are saved in a special section which is then used to adjust the relocations during boot. (Note: For some reason, Clang occasionally creates a GOT reference, even without '-fPIE'. So Clang-compiled kernels have a GOT, which needs to be adjusted.) On my mostly-defconfig kernel, this reduces kernel text size by ~1.3%. [1] https://github.com/dynup/kpatch/issues/1284 [2] https://gcc.gnu.org/pipermail/gcc-patches/2023-June/622872.html [3] https://gcc.gnu.org/pipermail/gcc-patches/2023-August/625986.html Compiler consideration: Gcc recently implemented an optimization [2] for loading symbols without explicit alignment, aligning with the IBM Z ELF ABI. This ABI mandates symbols to reside on a 2-byte boundary, enabling the use of the larl instruction. However, kernel linker scripts may still generate unaligned symbols. To address this, a new -munaligned-symbols option has been introduced [3] in recent gcc versions. This option has to be used with future gcc versions. Older Clang lacks support for handling unaligned symbols generated by kernel linker scripts when the kernel is built without -fPIE. However, future versions of Clang will include support for the -munaligned-symbols option. When the support is unavailable, compile the kernel with -fPIE to maintain the existing behavior. In addition to it: move vmlinux.relocs to safe relocation When the kernel is built with CONFIG_KERNEL_UNCOMPRESSED, the entire uncompressed vmlinux.bin is positioned in the bzImage decompressor image at the default kernel LMA of 0x100000, enabling it to be executed in-place. However, the size of .vmlinux.relocs could be large enough to cause an overlap with the uncompressed kernel at the address 0x100000. To address this issue, .vmlinux.relocs is positioned after the .rodata.compressed in the bzImage. Nevertheless, in this configuration, vmlinux.relocs will overlap with the .bss section of vmlinux.bin. To overcome that, move vmlinux.relocs to a safe location before clearing .bss and handling relocs. Compile warning fix from Sumanth Korikkar: When kernel is built with CONFIG_LD_ORPHAN_WARN and -fno-PIE, there are several warnings: ld: warning: orphan section `.rela.iplt' from `arch/s390/kernel/head64.o' being placed in section `.rela.dyn' ld: warning: orphan section `.rela.head.text' from `arch/s390/kernel/head64.o' being placed in section `.rela.dyn' ld: warning: orphan section `.rela.init.text' from `arch/s390/kernel/head64.o' being placed in section `.rela.dyn' ld: warning: orphan section `.rela.rodata.cst8' from `arch/s390/kernel/head64.o' being placed in section `.rela.dyn' Orphan sections are sections that exist in an object file but don't have a corresponding output section in the final executable. ld raises a warning when it identifies such sections. Eliminate the warning by placing all .rela orphan sections in .rela.dyn and raise an error when size of .rela.dyn is greater than zero. i.e. Dont just neglect orphan sections. This is similar to adjustment performed in x86, where kernel is built with -fno-PIE. commit 5354e84598f2 ("x86/build: Add asserts for unwanted sections") [sumanthk@linux.ibm.com: rebased Josh Poimboeuf patches and move vmlinux.relocs to safe location] [hca@linux.ibm.com: merged compile warning fix from Sumanth] Tested-by: Sumanth Korikkar Acked-by: Vasily Gorbik Signed-off-by: Josh Poimboeuf Signed-off-by: Sumanth Korikkar Link: https://lore.kernel.org/r/20240219132734.22881-4-sumanthk@linux.ibm.com Link: https://lore.kernel.org/r/20240219132734.22881-5-sumanthk@linux.ibm.com Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 15 ++++-- arch/s390/Makefile | 8 ++- arch/s390/boot/.gitignore | 1 + arch/s390/boot/Makefile | 14 ++++- arch/s390/boot/boot.h | 6 +++ arch/s390/boot/startup.c | 80 +++++++++++++++++++++++++--- arch/s390/boot/vmlinux.lds.S | 18 +++++++ arch/s390/include/asm/physmem_info.h | 1 + arch/s390/kernel/vmlinux.lds.S | 15 ++++++ 9 files changed, 145 insertions(+), 13 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 39e937309fc2..2dc722fb31b4 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -583,14 +583,23 @@ config RELOCATABLE help This builds a kernel image that retains relocation information so it can be loaded at an arbitrary address. - The kernel is linked as a position-independent executable (PIE) - and contains dynamic relocations which are processed early in the - bootup process. The relocations make the kernel image about 15% larger (compressed 10%), but are discarded at runtime. Note: this option exists only for documentation purposes, please do not remove it. +config PIE_BUILD + def_bool CC_IS_CLANG && !$(cc-option,-munaligned-symbols) + help + If the compiler is unable to generate code that can manage unaligned + symbols, the kernel is linked as a position-independent executable + (PIE) and includes dynamic relocations that are processed early + during bootup. + + For kpatch functionality, it is recommended to build the kernel + without the PIE_BUILD option. PIE_BUILD is only enabled when the + compiler lacks proper support for handling unaligned symbols. + config RANDOMIZE_BASE bool "Randomize the address of the kernel image (KASLR)" default y diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 994f9b3d575f..2a58e1864931 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -14,8 +14,14 @@ KBUILD_AFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC KBUILD_AFLAGS += -m64 KBUILD_CFLAGS += -m64 +ifdef CONFIG_PIE_BUILD KBUILD_CFLAGS += -fPIE LDFLAGS_vmlinux := -pie -z notext +else +KBUILD_CFLAGS += $(call cc-option,-munaligned-symbols,) +LDFLAGS_vmlinux := --emit-relocs --discard-none +extra_tools := relocs +endif aflags_dwarf := -Wa,-gdwarf-2 KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ ifndef CONFIG_AS_IS_LLVM @@ -143,7 +149,7 @@ archheaders: archprepare: $(Q)$(MAKE) $(build)=$(syscalls) kapi - $(Q)$(MAKE) $(build)=$(tools) kapi + $(Q)$(MAKE) $(build)=$(tools) kapi $(extra_tools) ifeq ($(KBUILD_EXTMOD),) # We need to generate vdso-offsets.h before compiling certain files in kernel/. # In order to do that, we should use the archprepare target, but we can't since diff --git a/arch/s390/boot/.gitignore b/arch/s390/boot/.gitignore index f56591bc0897..f5ef099e2fd3 100644 --- a/arch/s390/boot/.gitignore +++ b/arch/s390/boot/.gitignore @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only image bzImage +relocs.S section_cmp.* vmlinux vmlinux.lds diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index d40135efdec4..aecafabc2054 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -37,7 +37,8 @@ CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char obj-y := head.o als.o startup.o physmem_info.o ipl_parm.o ipl_report.o vmem.o obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o -obj-y += version.o pgm_check_info.o ctype.o ipl_data.o machine_kexec_reloc.o +obj-y += version.o pgm_check_info.o ctype.o ipl_data.o +obj-y += $(if $(CONFIG_PIE_BUILD),machine_kexec_reloc.o,relocs.o) obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-y += $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) info.o @@ -48,6 +49,9 @@ targets := bzImage section_cmp.boot.data section_cmp.boot.preserved.data $(obj-y targets += vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 targets += vmlinux.bin.zst info.bin syms.bin vmlinux.syms $(obj-all) +ifndef CONFIG_PIE_BUILD +targets += relocs.S +endif OBJECTS := $(addprefix $(obj)/,$(obj-y)) OBJECTS_ALL := $(addprefix $(obj)/,$(obj-all)) @@ -106,6 +110,14 @@ OBJCOPYFLAGS_vmlinux.bin := -O binary --remove-section=.comment --remove-section $(obj)/vmlinux.bin: vmlinux FORCE $(call if_changed,objcopy) +ifndef CONFIG_PIE_BUILD +CMD_RELOCS=arch/s390/tools/relocs +quiet_cmd_relocs = RELOCS $@ + cmd_relocs = $(CMD_RELOCS) $< > $@ +$(obj)/relocs.S: vmlinux FORCE + $(call if_changed,relocs) +endif + suffix-$(CONFIG_KERNEL_GZIP) := .gz suffix-$(CONFIG_KERNEL_BZIP2) := .bz2 suffix-$(CONFIG_KERNEL_LZ4) := .lz4 diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 222c6886acf6..8943d5be7ca2 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -25,9 +25,14 @@ struct vmlinux_info { unsigned long bootdata_size; unsigned long bootdata_preserved_off; unsigned long bootdata_preserved_size; +#ifdef CONFIG_PIE_BUILD unsigned long dynsym_start; unsigned long rela_dyn_start; unsigned long rela_dyn_end; +#else + unsigned long got_off; + unsigned long got_size; +#endif unsigned long amode31_size; unsigned long init_mm_off; unsigned long swapper_pg_dir_off; @@ -83,6 +88,7 @@ extern unsigned long vmalloc_size; extern int vmalloc_size_set; extern char __boot_data_start[], __boot_data_end[]; extern char __boot_data_preserved_start[], __boot_data_preserved_end[]; +extern int __vmlinux_relocs_64_start[], __vmlinux_relocs_64_end[]; extern char _decompressor_syms_start[], _decompressor_syms_end[]; extern char _stack_start[], _stack_end[]; extern char _end[], _decompressor_end[]; diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 9cc76e631759..cb0d89801c43 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -141,7 +141,8 @@ static void copy_bootdata(void) memcpy((void *)vmlinux.bootdata_preserved_off, __boot_data_preserved_start, vmlinux.bootdata_preserved_size); } -static void handle_relocs(unsigned long offset) +#ifdef CONFIG_PIE_BUILD +static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long offset) { Elf64_Rela *rela_start, *rela_end, *rela; int r_type, r_sym, rc; @@ -172,6 +173,62 @@ static void handle_relocs(unsigned long offset) } } +static void kaslr_adjust_got(unsigned long offset) {} +static void rescue_relocs(void) {} +static void free_relocs(void) {} +#else +int *vmlinux_relocs_64_start; +int *vmlinux_relocs_64_end; + +static void rescue_relocs(void) +{ + unsigned long size, nrelocs; + + nrelocs = __vmlinux_relocs_64_end - __vmlinux_relocs_64_start; + size = nrelocs * sizeof(uint32_t); + vmlinux_relocs_64_start = (void *)physmem_alloc_top_down(RR_RELOC, size, 0); + memmove(vmlinux_relocs_64_start, (void *)__vmlinux_relocs_64_start, size); + vmlinux_relocs_64_end = vmlinux_relocs_64_start + nrelocs; +} + +static void free_relocs(void) +{ + physmem_free(RR_RELOC); +} + +static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long offset) +{ + int *reloc; + unsigned long max_addr = min_addr + vmlinux.image_size; + long loc; + + /* Adjust R_390_64 relocations */ + for (reloc = vmlinux_relocs_64_start; + reloc < vmlinux_relocs_64_end && *reloc; + reloc++) { + loc = (long)*reloc + offset; + if (loc < min_addr || loc > max_addr) + error("64-bit relocation outside of kernel!\n"); + *(u64 *)loc += offset; + } +} + +static void kaslr_adjust_got(unsigned long offset) +{ + u64 *entry; + + /* + * Even without -fPIE, Clang still uses a global offset table for some + * reason. Adjust the GOT entries. + */ + for (entry = (u64 *)vmlinux.got_off; + entry < (u64 *)(vmlinux.got_off + vmlinux.got_size); + entry++) { + *entry += offset; + } +} +#endif + /* * Merge information from several sources into a single ident_map_size value. * "ident_map_size" represents the upper limit of physical memory we may ever @@ -299,14 +356,18 @@ static void setup_vmalloc_size(void) vmalloc_size = max(size, vmalloc_size); } -static void offset_vmlinux_info(unsigned long offset) +static void kaslr_adjust_vmlinux_info(unsigned long offset) { *(unsigned long *)(&vmlinux.entry) += offset; vmlinux.bootdata_off += offset; vmlinux.bootdata_preserved_off += offset; +#ifdef CONFIG_PIE_BUILD vmlinux.rela_dyn_start += offset; vmlinux.rela_dyn_end += offset; vmlinux.dynsym_start += offset; +#else + vmlinux.got_off += offset; +#endif vmlinux.init_mm_off += offset; vmlinux.swapper_pg_dir_off += offset; vmlinux.invalid_pg_dir_off += offset; @@ -361,6 +422,7 @@ void startup_kernel(void) detect_physmem_online_ranges(max_physmem_end); save_ipl_cert_comp_list(); rescue_initrd(safe_addr, ident_map_size); + rescue_relocs(); if (kaslr_enabled()) { vmlinux_lma = randomize_within_range(vmlinux.image_size + vmlinux.bss_size, @@ -368,7 +430,7 @@ void startup_kernel(void) ident_map_size); if (vmlinux_lma) { __kaslr_offset = vmlinux_lma - vmlinux.default_lma; - offset_vmlinux_info(__kaslr_offset); + kaslr_adjust_vmlinux_info(__kaslr_offset); } } vmlinux_lma = vmlinux_lma ?: vmlinux.default_lma; @@ -393,18 +455,20 @@ void startup_kernel(void) /* * The order of the following operations is important: * - * - handle_relocs() must follow clear_bss_section() to establish static + * - kaslr_adjust_relocs() must follow clear_bss_section() to establish static * memory references to data in .bss to be used by setup_vmem() * (i.e init_mm.pgd) * - * - setup_vmem() must follow handle_relocs() to be able using + * - setup_vmem() must follow kaslr_adjust_relocs() to be able using * static memory references to data in .bss (i.e init_mm.pgd) * - * - copy_bootdata() must follow setup_vmem() to propagate changes to - * bootdata made by setup_vmem() + * - copy_bootdata() must follow setup_vmem() to propagate changes + * to bootdata made by setup_vmem() */ clear_bss_section(vmlinux_lma); - handle_relocs(__kaslr_offset); + kaslr_adjust_relocs(vmlinux_lma, __kaslr_offset); + kaslr_adjust_got(__kaslr_offset); + free_relocs(); setup_vmem(asce_limit); copy_bootdata(); diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 4e3fb2b7abb6..e3208893ba6b 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -110,6 +110,24 @@ SECTIONS _compressed_end = .; } +#ifndef CONFIG_PIE_BUILD + /* + * When the kernel is built with CONFIG_KERNEL_UNCOMPRESSED, the entire + * uncompressed vmlinux.bin is positioned in the bzImage decompressor + * image at the default kernel LMA of 0x100000, enabling it to be + * executed in-place. However, the size of .vmlinux.relocs could be + * large enough to cause an overlap with the uncompressed kernel at the + * address 0x100000. To address this issue, .vmlinux.relocs is + * positioned after the .rodata.compressed. + */ + . = ALIGN(4); + .vmlinux.relocs : { + __vmlinux_relocs_64_start = .; + *(.vmlinux.relocs_64) + __vmlinux_relocs_64_end = .; + } +#endif + #define SB_TRAILER_SIZE 32 /* Trailer needed for Secure Boot */ . += SB_TRAILER_SIZE; /* make sure .sb.trailer does not overwrite the previous section */ diff --git a/arch/s390/include/asm/physmem_info.h b/arch/s390/include/asm/physmem_info.h index 9e41a74fce9a..e747b067f8db 100644 --- a/arch/s390/include/asm/physmem_info.h +++ b/arch/s390/include/asm/physmem_info.h @@ -22,6 +22,7 @@ enum reserved_range_type { RR_DECOMPRESSOR, RR_INITRD, RR_VMLINUX, + RR_RELOC, RR_AMODE31, RR_IPLREPORT, RR_CERT_COMP_LIST, diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 63cb403b3344..404883b1b023 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -63,7 +63,9 @@ SECTIONS *(.data.rel.ro .data.rel.ro.*) } .got : { + __got_start = .; *(.got) + __got_end = .; } . = ALIGN(PAGE_SIZE); @@ -190,6 +192,7 @@ SECTIONS PERCPU_SECTION(0x100) +#ifdef CONFIG_PIE_BUILD .dynsym ALIGN(8) : { __dynsym_start = .; *(.dynsym) @@ -206,6 +209,7 @@ SECTIONS .dynstr ALIGN(8) : { *(.dynstr) } +#endif .hash ALIGN(8) : { *(.hash) } @@ -235,9 +239,14 @@ SECTIONS QUAD(__boot_data_preserved_start) /* bootdata_preserved_off */ QUAD(__boot_data_preserved_end - __boot_data_preserved_start) /* bootdata_preserved_size */ +#ifdef CONFIG_PIE_BUILD QUAD(__dynsym_start) /* dynsym_start */ QUAD(__rela_dyn_start) /* rela_dyn_start */ QUAD(__rela_dyn_end) /* rela_dyn_end */ +#else + QUAD(__got_start) /* got_off */ + QUAD(__got_end - __got_start) /* got_size */ +#endif QUAD(_eamode31 - _samode31) /* amode31_size */ QUAD(init_mm) QUAD(swapper_pg_dir) @@ -268,6 +277,12 @@ SECTIONS *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) } ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!") +#ifndef CONFIG_PIE_BUILD + .rela.dyn : { + *(.rela.*) *(.rela_*) + } + ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") +#endif /* Sections to be discarded */ DISCARDS From 4a5993287467d2d0401503256dc9d2690c7f2020 Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Tue, 20 Feb 2024 08:56:34 +0000 Subject: [PATCH 095/117] KVM: s390: introduce kvm_s390_fpu_(store|load) It's a bit nicer than having multiple lines and will help if there's another re-work since we'll only have to change one location. Signed-off-by: Janosch Frank Signed-off-by: Heiko Carstens --- arch/s390/kvm/interrupt.c | 6 +----- arch/s390/kvm/kvm-s390.c | 18 +++--------------- arch/s390/kvm/kvm-s390.h | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index c81708acd1f4..dc721d50a942 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -584,11 +584,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu, mci.val = mchk->mcic; /* take care of lazy register loading */ - fpu_stfpc(&vcpu->run->s.regs.fpc); - if (cpu_has_vx()) - save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); - else - save_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); + kvm_s390_fpu_store(vcpu->run); save_access_regs(vcpu->run->s.regs.acrs); if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 8c222b0dfbf2..6500f80a7086 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4949,11 +4949,7 @@ static void sync_regs(struct kvm_vcpu *vcpu) } save_access_regs(vcpu->arch.host_acrs); restore_access_regs(vcpu->run->s.regs.acrs); - fpu_lfpc_safe(&vcpu->run->s.regs.fpc); - if (cpu_has_vx()) - load_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); - else - load_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); + kvm_s390_fpu_load(vcpu->run); /* Sync fmt2 only data */ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { sync_regs_fmt2(vcpu); @@ -5014,11 +5010,7 @@ static void store_regs(struct kvm_vcpu *vcpu) kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; save_access_regs(vcpu->run->s.regs.acrs); restore_access_regs(vcpu->arch.host_acrs); - fpu_stfpc(&vcpu->run->s.regs.fpc); - if (cpu_has_vx()) - save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); - else - save_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); + kvm_s390_fpu_store(vcpu->run); if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) store_regs_fmt2(vcpu); } @@ -5167,11 +5159,7 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) * switch in the run ioctl. Let's update our copies before we save * it into the save area */ - fpu_stfpc(&vcpu->run->s.regs.fpc); - if (cpu_has_vx()) - save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs); - else - save_fp_regs((freg_t *)&vcpu->run->s.regs.fprs); + kvm_s390_fpu_store(vcpu->run); save_access_regs(vcpu->run->s.regs.acrs); return kvm_s390_store_status_unloaded(vcpu, addr); diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index a7ea80cfa445..111eb5c74784 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -20,6 +20,24 @@ #include #include +static inline void kvm_s390_fpu_store(struct kvm_run *run) +{ + fpu_stfpc(&run->s.regs.fpc); + if (cpu_has_vx()) + save_vx_regs((__vector128 *)&run->s.regs.vrs); + else + save_fp_regs((freg_t *)&run->s.regs.fprs); +} + +static inline void kvm_s390_fpu_load(struct kvm_run *run) +{ + fpu_lfpc_safe(&run->s.regs.fpc); + if (cpu_has_vx()) + load_vx_regs((__vector128 *)&run->s.regs.vrs); + else + load_fp_regs((freg_t *)&run->s.regs.fprs); +} + /* Transactional Memory Execution related macros */ #define IS_TE_ENABLED(vcpu) ((vcpu->arch.sie_block->ecb & ECB_TE)) #define TDB_FORMAT1 1 From 01be7f53dfc29e0a362f3d05e5ab2c8dfcc171c8 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 20 Feb 2024 22:12:10 +0100 Subject: [PATCH 096/117] KVM: s390: fix access register usage in ioctls The routine ar_translation() can be reached by both the instruction intercept path (where the access registers had been loaded with the guest register contents), and the MEM_OP ioctls (which hadn't). Since this routine saves the current registers to vcpu->run, this routine erroneously saves host registers into the guest space. Introduce a boolean in the kvm_vcpu_arch struct to indicate whether the registers contain guest contents. If they do (the instruction intercept path), the save can be performed and the AR translation is done just as it is today. If they don't (the MEM_OP path), the AR can be read from vcpu->run without stashing the current contents. Reviewed-by: Heiko Carstens Reviewed-by: Nina Schoetterl-Glausch Reviewed-by: Christian Borntraeger Signed-off-by: Eric Farman Link: https://lore.kernel.org/r/20240220211211.3102609-2-farman@linux.ibm.com Signed-off-by: Heiko Carstens --- arch/s390/include/asm/kvm_host.h | 2 ++ arch/s390/kvm/gaccess.c | 3 ++- arch/s390/kvm/kvm-s390.c | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 56c2efb41cf2..95990461888f 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -764,6 +764,8 @@ struct kvm_vcpu_arch { __u64 cputm_start; bool gs_enabled; bool skey_enabled; + /* Indicator if the access registers have been loaded from guest */ + bool acrs_loaded; struct kvm_s390_pv_vcpu pv; union diag318_info diag318_info; }; diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index d264aa81aa15..ee863566910b 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -391,7 +391,8 @@ static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, u8 ar, if (ar >= NUM_ACRS) return -EINVAL; - save_access_regs(vcpu->run->s.regs.acrs); + if (vcpu->arch.acrs_loaded) + save_access_regs(vcpu->run->s.regs.acrs); alet.val = vcpu->run->s.regs.acrs[ar]; if (ar == 0 || alet.val == 0) { diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 6500f80a7086..b11bb8e780a1 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3951,6 +3951,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) KVM_SYNC_ARCH0 | KVM_SYNC_PFAULT | KVM_SYNC_DIAG318; + vcpu->arch.acrs_loaded = false; kvm_s390_set_prefix(vcpu, 0); if (test_kvm_facility(vcpu->kvm, 64)) vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB; @@ -4949,6 +4950,7 @@ static void sync_regs(struct kvm_vcpu *vcpu) } save_access_regs(vcpu->arch.host_acrs); restore_access_regs(vcpu->run->s.regs.acrs); + vcpu->arch.acrs_loaded = true; kvm_s390_fpu_load(vcpu->run); /* Sync fmt2 only data */ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) { @@ -5010,6 +5012,7 @@ static void store_regs(struct kvm_vcpu *vcpu) kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; save_access_regs(vcpu->run->s.regs.acrs); restore_access_regs(vcpu->arch.host_acrs); + vcpu->arch.acrs_loaded = false; kvm_s390_fpu_store(vcpu->run); if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) store_regs_fmt2(vcpu); From 559a1462909b893c52b41e2e39c364af8bacae8a Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 20 Feb 2024 22:12:11 +0100 Subject: [PATCH 097/117] KVM: s390: selftests: memop: add a simple AR test There is a selftest that checks for an (expected) error when an invalid AR is specified, but not one that exercises the AR path. Add a simple test that mirrors the vanilla write/read test while providing an AR. An AR that contains zero will direct the CPU to use the primary address space normally used anyway. AR[1] is selected for this test because the host AR[1] is usually non-zero, and KVM needs to correctly swap those values. Reviewed-by: Nina Schoetterl-Glausch Acked-by: Christian Borntraeger Signed-off-by: Eric Farman Link: https://lore.kernel.org/r/20240220211211.3102609-3-farman@linux.ibm.com Signed-off-by: Heiko Carstens --- tools/testing/selftests/kvm/s390x/memop.c | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/testing/selftests/kvm/s390x/memop.c b/tools/testing/selftests/kvm/s390x/memop.c index bb3ca9a5d731..b6da8f71ea19 100644 --- a/tools/testing/selftests/kvm/s390x/memop.c +++ b/tools/testing/selftests/kvm/s390x/memop.c @@ -375,6 +375,32 @@ static void test_copy(void) kvm_vm_free(t.kvm_vm); } +static void test_copy_access_register(void) +{ + struct test_default t = test_default_init(guest_copy); + + HOST_SYNC(t.vcpu, STAGE_INITED); + + prepare_mem12(); + t.run->psw_mask &= ~(3UL << (63 - 17)); + t.run->psw_mask |= 1UL << (63 - 17); /* Enable AR mode */ + + /* + * Primary address space gets used if an access register + * contains zero. The host makes use of AR[1] so is a good + * candidate to ensure the guest AR (of zero) is used. + */ + CHECK_N_DO(MOP, t.vcpu, LOGICAL, WRITE, mem1, t.size, + GADDR_V(mem1), AR(1)); + HOST_SYNC(t.vcpu, STAGE_COPIED); + + CHECK_N_DO(MOP, t.vcpu, LOGICAL, READ, mem2, t.size, + GADDR_V(mem2), AR(1)); + ASSERT_MEM_EQ(mem1, mem2, t.size); + + kvm_vm_free(t.kvm_vm); +} + static void set_storage_key_range(void *addr, size_t len, uint8_t key) { uintptr_t _addr, abs, i; @@ -1101,6 +1127,11 @@ int main(int argc, char *argv[]) .test = test_copy_key_fetch_prot_override, .requirements_met = extension_cap > 0, }, + { + .name = "copy with access register mode", + .test = test_copy_access_register, + .requirements_met = true, + }, { .name = "error checks with key", .test = test_errors_key, From 7f115ff4fc20698452ff4a1cbb9c790be10b0066 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 20 Feb 2024 13:44:48 -0700 Subject: [PATCH 098/117] s390/boot: workaround current 'llvm-objdump -t -j ...' behavior When building with OBJDUMP=llvm-objdump, there are a series of warnings from the section comparisons that arch/s390/boot/Makefile performs between vmlinux and arch/s390/boot/vmlinux: llvm-objdump: warning: section '.boot.preserved.data' mentioned in a -j/--section option, but not found in any input file llvm-objdump: warning: section '.boot.data' mentioned in a -j/--section option, but not found in any input file llvm-objdump: warning: section '.boot.preserved.data' mentioned in a -j/--section option, but not found in any input file llvm-objdump: warning: section '.boot.data' mentioned in a -j/--section option, but not found in any input file The warning is a little misleading, as these sections do exist in the input files. It is really pointing out that llvm-objdump does not match GNU objdump's behavior of respecting '-j' / '--section' in combination with '-t' / '--syms': $ s390x-linux-gnu-objdump -t -j .boot.data vmlinux.full vmlinux.full: file format elf64-s390 SYMBOL TABLE: 0000000001951000 l O .boot.data 0000000000003000 sclp_info_sccb 00000000019550e0 l O .boot.data 0000000000000001 sclp_info_sccb_valid 00000000019550e2 g O .boot.data 0000000000001000 early_command_line ... $ llvm-objdump -t -j .boot.data vmlinux.full vmlinux.full: file format elf64-s390 SYMBOL TABLE: 0000000000100040 l O .text 0000000000000010 dw_psw 0000000000000000 l df *ABS* 0000000000000000 main.c 00000000001001b0 l F .text 00000000000000c6 trace_event_raw_event_initcall_level 0000000000100280 l F .text 0000000000000100 perf_trace_initcall_level ... It may be possible to change llvm-objdump's behavior to match GNU objdump's behavior but the difficulty of that task has not yet been explored. The combination of '$(OBJDUMP) -t -j' is not common in the kernel tree on a whole, so workaround this tool difference by grepping for the sections in the full symbol table output in a similar manner to the sed invocation. This results in no visible change for GNU objdump users while fixing the warnings for OBJDUMP=llvm-objdump, further enabling use of LLVM=1 for ARCH=s390 with versions of LLVM that have support for s390 in ld.lld and llvm-objcopy. Reported-by: Heiko Carstens Closes: https://lore.kernel.org/20240219113248.16287-C-hca@linux.ibm.com/ Link: https://github.com/ClangBuiltLinux/linux/issues/859 Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20240220-s390-work-around-llvm-objdump-t-j-v1-1-47bb0366a831@kernel.org Signed-off-by: Heiko Carstens --- arch/s390/boot/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index aecafabc2054..294f08a8811a 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -60,9 +60,9 @@ clean-files += vmlinux.map quiet_cmd_section_cmp = SECTCMP $* define cmd_section_cmp - s1=`$(OBJDUMP) -t -j "$*" "$<" | sort | \ + s1=`$(OBJDUMP) -t "$<" | grep "\s$*\s\+" | sort | \ sed -n "/0000000000000000/! s/.*\s$*\s\+//p" | sha256sum`; \ - s2=`$(OBJDUMP) -t -j "$*" "$(word 2,$^)" | sort | \ + s2=`$(OBJDUMP) -t "$(word 2,$^)" | grep "\s$*\s\+" | sort | \ sed -n "/0000000000000000/! s/.*\s$*\s\+//p" | sha256sum`; \ if [ "$$s1" != "$$s2" ]; then \ echo "error: section $* differs between $< and $(word 2,$^)" >&2; \ From a795e5d2347def129734a7f247ac70339d50d8c2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 25 Feb 2024 17:39:12 +0100 Subject: [PATCH 099/117] s390: vmlinux.lds.S: fix .got.plt assertion Naresh reported this build error on linux-next: s390x-linux-gnu-ld: Unexpected GOT/PLT entries detected! make[3]: *** [/builds/linux/arch/s390/boot/Makefile:87: arch/s390/boot/vmlinux.syms] Error 1 make[3]: Target 'arch/s390/boot/bzImage' not remade because of errors. The reason for the build error is an incorrect/incomplete assertion which checks the size of the .got.plt section. Similar to x86 the size is either zero or 24 bytes (three entries). See commit 262b5cae67a6 ("x86/boot/compressed: Move .got.plt entries out of the .got section") for more details. The three reserved/additional entries for s390 are described in chapter 3.2.2 of the s390x ABI [1] (thanks to Andreas Krebbel for pointing this out!). [1] https://github.com/IBM/s390x-abi/releases/download/v1.6.1/lzsabi_s390x.pdf Reported-by: Naresh Kamboju Reported-by: Linux Kernel Functional Testing Closes: https://lore.kernel.org/all/CA+G9fYvWp8TY-fMEvc3UhoVtoR_eM5VsfHj3+n+kexcfJJ+Cvw@mail.gmail.com Fixes: 30226853d6ec ("s390: vmlinux.lds.S: explicitly handle '.got' and '.plt' sections") Signed-off-by: Heiko Carstens --- arch/s390/boot/vmlinux.lds.S | 11 ++++++++--- arch/s390/kernel/vmlinux.lds.S | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index e3208893ba6b..3d7ea585ab99 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -144,13 +144,18 @@ SECTIONS ELF_DETAILS /* - * Sections that should stay zero sized, which is safer to - * explicitly check instead of blindly discarding. + * Make sure that the .got.plt is either completely empty or it + * contains only the three reserved double words. */ .got.plt : { *(.got.plt) } - ASSERT(SIZEOF(.got.plt) == 0, "Unexpected GOT/PLT entries detected!") + ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18, "Unexpected GOT/PLT entries detected!") + + /* + * Sections that should stay zero sized, which is safer to + * explicitly check instead of blindly discarding. + */ .plt : { *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) } diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 404883b1b023..9c59715d1745 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -266,13 +266,18 @@ SECTIONS ELF_DETAILS /* - * Sections that should stay zero sized, which is safer to - * explicitly check instead of blindly discarding. + * Make sure that the .got.plt is either completely empty or it + * contains only the three reserved double words. */ .got.plt : { *(.got.plt) } - ASSERT(SIZEOF(.got.plt) == 0, "Unexpected GOT/PLT entries detected!") + ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18, "Unexpected GOT/PLT entries detected!") + + /* + * Sections that should stay zero sized, which is safer to + * explicitly check instead of blindly discarding. + */ .plt : { *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt) } From 3334fda639cfa7bb27a457f85f4ce06d622e0511 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 21 Feb 2024 11:51:55 +0100 Subject: [PATCH 100/117] s390/boot: simplify GOT handling The end of GOT is calculated dynamically on boot. The size of GOT is calculated on build from the start and end of GOT. Avoid both calculations and use the end of GOT directly. Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/boot/boot.h | 4 ++-- arch/s390/boot/startup.c | 8 +++----- arch/s390/kernel/vmlinux.lds.S | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 8943d5be7ca2..cebf9feba6b1 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -30,8 +30,8 @@ struct vmlinux_info { unsigned long rela_dyn_start; unsigned long rela_dyn_end; #else - unsigned long got_off; - unsigned long got_size; + unsigned long got_start; + unsigned long got_end; #endif unsigned long amode31_size; unsigned long init_mm_off; diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index cb0d89801c43..c1a758921b7a 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -221,11 +221,8 @@ static void kaslr_adjust_got(unsigned long offset) * Even without -fPIE, Clang still uses a global offset table for some * reason. Adjust the GOT entries. */ - for (entry = (u64 *)vmlinux.got_off; - entry < (u64 *)(vmlinux.got_off + vmlinux.got_size); - entry++) { + for (entry = (u64 *)vmlinux.got_start; entry < (u64 *)vmlinux.got_end; entry++) *entry += offset; - } } #endif @@ -366,7 +363,8 @@ static void kaslr_adjust_vmlinux_info(unsigned long offset) vmlinux.rela_dyn_end += offset; vmlinux.dynsym_start += offset; #else - vmlinux.got_off += offset; + vmlinux.got_start += offset; + vmlinux.got_end += offset; #endif vmlinux.init_mm_off += offset; vmlinux.swapper_pg_dir_off += offset; diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 9c59715d1745..48de296e8905 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -244,8 +244,8 @@ SECTIONS QUAD(__rela_dyn_start) /* rela_dyn_start */ QUAD(__rela_dyn_end) /* rela_dyn_end */ #else - QUAD(__got_start) /* got_off */ - QUAD(__got_end - __got_start) /* got_size */ + QUAD(__got_start) /* got_start */ + QUAD(__got_end) /* got_end */ #endif QUAD(_eamode31 - _samode31) /* amode31_size */ QUAD(init_mm) From 8495fd4dfee89b9b07353e8d01f0569c0330ff12 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 21 Feb 2024 13:25:06 +0100 Subject: [PATCH 101/117] s390/boot: sanitize kaslr_adjust_relocs() function prototype Do not use vmlinux.image_size within kaslr_adjust_relocs() function to calculate the upper relocation table boundary. Instead, make both lower and upper boundaries the function input parameters. Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/boot/startup.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index c1a758921b7a..09490ddd6973 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -142,7 +142,7 @@ static void copy_bootdata(void) } #ifdef CONFIG_PIE_BUILD -static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long offset) +static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long max_addr, unsigned long offset) { Elf64_Rela *rela_start, *rela_end, *rela; int r_type, r_sym, rc; @@ -196,10 +196,9 @@ static void free_relocs(void) physmem_free(RR_RELOC); } -static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long offset) +static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long max_addr, unsigned long offset) { int *reloc; - unsigned long max_addr = min_addr + vmlinux.image_size; long loc; /* Adjust R_390_64 relocations */ @@ -464,7 +463,7 @@ void startup_kernel(void) * to bootdata made by setup_vmem() */ clear_bss_section(vmlinux_lma); - kaslr_adjust_relocs(vmlinux_lma, __kaslr_offset); + kaslr_adjust_relocs(vmlinux_lma, vmlinux_lma + vmlinux.image_size, __kaslr_offset); kaslr_adjust_got(__kaslr_offset); free_relocs(); setup_vmem(asce_limit); From 4394a50792346ec291d1bbde6da7571f799a6f12 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 21 Feb 2024 14:32:24 +0100 Subject: [PATCH 102/117] s390/boot: make type of __vmlinux_relocs_64_start|end consistent Make the type of __vmlinux_relocs_64_start|end symbols as char array, just like it is done for all other sections. Function rescue_relocs() is simplified as result. Suggested-by: Heiko Carstens Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/boot/boot.h | 2 +- arch/s390/boot/startup.c | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index cebf9feba6b1..567d60f78bbc 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -88,7 +88,7 @@ extern unsigned long vmalloc_size; extern int vmalloc_size_set; extern char __boot_data_start[], __boot_data_end[]; extern char __boot_data_preserved_start[], __boot_data_preserved_end[]; -extern int __vmlinux_relocs_64_start[], __vmlinux_relocs_64_end[]; +extern char __vmlinux_relocs_64_start[], __vmlinux_relocs_64_end[]; extern char _decompressor_syms_start[], _decompressor_syms_end[]; extern char _stack_start[], _stack_end[]; extern char _end[], _decompressor_end[]; diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 09490ddd6973..d6709fd80fef 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -177,18 +177,16 @@ static void kaslr_adjust_got(unsigned long offset) {} static void rescue_relocs(void) {} static void free_relocs(void) {} #else -int *vmlinux_relocs_64_start; -int *vmlinux_relocs_64_end; +static int *vmlinux_relocs_64_start; +static int *vmlinux_relocs_64_end; static void rescue_relocs(void) { - unsigned long size, nrelocs; + unsigned long size = __vmlinux_relocs_64_end - __vmlinux_relocs_64_start; - nrelocs = __vmlinux_relocs_64_end - __vmlinux_relocs_64_start; - size = nrelocs * sizeof(uint32_t); vmlinux_relocs_64_start = (void *)physmem_alloc_top_down(RR_RELOC, size, 0); - memmove(vmlinux_relocs_64_start, (void *)__vmlinux_relocs_64_start, size); - vmlinux_relocs_64_end = vmlinux_relocs_64_start + nrelocs; + vmlinux_relocs_64_end = (void *)vmlinux_relocs_64_start + size; + memmove(vmlinux_relocs_64_start, __vmlinux_relocs_64_start, size); } static void free_relocs(void) From 923d48e48033b37ef77323d2cd48771d68160c96 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 21 Feb 2024 13:41:45 +0100 Subject: [PATCH 103/117] s390/boot: do not check for zero-termination relocation entry The relocation table is not expected to contain a zero-termination entry. The existing check is likely a left-over from similar x86 code that uses zero-entries as delimiters. s390 does not have ones and therefore the check could be avoided. Suggested-by: Ilya Leoshkevich Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/boot/startup.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index d6709fd80fef..c09797ac820f 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -200,9 +200,7 @@ static void kaslr_adjust_relocs(unsigned long min_addr, unsigned long max_addr, long loc; /* Adjust R_390_64 relocations */ - for (reloc = vmlinux_relocs_64_start; - reloc < vmlinux_relocs_64_end && *reloc; - reloc++) { + for (reloc = vmlinux_relocs_64_start; reloc < vmlinux_relocs_64_end; reloc++) { loc = (long)*reloc + offset; if (loc < min_addr || loc > max_addr) error("64-bit relocation outside of kernel!\n"); From 13ff094d32e7cc42e89b7944de9354ae6efb519e Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 21 Feb 2024 14:46:53 +0100 Subject: [PATCH 104/117] s390/boot: fix minor comment style damages Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/boot/startup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index c09797ac820f..6cf89314209a 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -214,7 +214,7 @@ static void kaslr_adjust_got(unsigned long offset) /* * Even without -fPIE, Clang still uses a global offset table for some - * reason. Adjust the GOT entries. + * reason. Adjust the GOT entries. */ for (entry = (u64 *)vmlinux.got_start; entry < (u64 *)vmlinux.got_end; entry++) *entry += offset; @@ -448,8 +448,8 @@ void startup_kernel(void) /* * The order of the following operations is important: * - * - kaslr_adjust_relocs() must follow clear_bss_section() to establish static - * memory references to data in .bss to be used by setup_vmem() + * - kaslr_adjust_relocs() must follow clear_bss_section() to establish + * static memory references to data in .bss to be used by setup_vmem() * (i.e init_mm.pgd) * * - setup_vmem() must follow kaslr_adjust_relocs() to be able using From e22033fddd453a81185557fb77568651819bc8f1 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 19 Feb 2024 15:40:01 +0100 Subject: [PATCH 105/117] s390/pai: change sampling event assignment for PMU device driver Currently only one PAI sampling event can be created and active at any one time. The PMU device drivers store a pointer to this event in their data structures even when the event is created for counting and the PMU device driver reference to this counting event is never needed. Change this and assign the pointer to the PMU device driver only when a sampling event is created. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_pai_crypto.c | 4 +++- arch/s390/kernel/perf_pai_ext.c | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 61fb4e716476..823d652e3917 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -324,12 +324,15 @@ static void paicrypt_read(struct perf_event *event) static void paicrypt_start(struct perf_event *event, int flags) { + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; u64 sum; if (!event->attr.sample_period) { /* Counting */ sum = paicrypt_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ + cpump->event = event; perf_sched_cb_inc(event->pmu); } } @@ -345,7 +348,6 @@ static int paicrypt_add(struct perf_event *event, int flags) WRITE_ONCE(S390_lowcore.ccd, ccd); local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); } - cpump->event = event; if (flags & PERF_EF_START) paicrypt_start(event, PERF_EF_RELOAD); event->hw.state = 0; diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index dd84c4d6fbe9..616a25606cd6 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -203,7 +203,6 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) } rc = 0; - cpump->event = event; undo: if (rc) { @@ -328,12 +327,15 @@ static void paiext_read(struct perf_event *event) static void paiext_start(struct perf_event *event, int flags) { + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; u64 sum; if (!event->attr.sample_period) { /* Counting */ sum = paiext_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ + cpump->event = event; perf_sched_cb_inc(event->pmu); } } @@ -352,7 +354,6 @@ static int paiext_add(struct perf_event *event, int flags) debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", __func__, S390_lowcore.aicd, pcb->acc); } - cpump->event = event; if (flags & PERF_EF_START) paiext_start(event, PERF_EF_RELOAD); event->hw.state = 0; From a681226c675cf4ff4f1ece8f44a7f698c4a6ca6a Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Thu, 15 Feb 2024 10:31:44 -0500 Subject: [PATCH 106/117] s390/vfio-ap: handle hardware checkstop state on queue reset operation Update vfio_ap_mdev_reset_queue() to handle an unexpected checkstop (hardware error) the same as the deconfigured case. This prevents unexpected and unhelpful warnings in the event of a hardware error. We also stop lying about a queue's reset response code. This was originally done so we could force vfio_ap_mdev_filter_matrix to pass a deconfigured device through to the guest for the hotplug scenario. vfio_ap_mdev_filter_matrix is instead modified to allow passthrough for all queues with reset state normal, deconfigured, or checkstopped. In the checkstopped case we choose to pass the device through and let the error state be reflected at the guest level. Signed-off-by: "Jason J. Herne" Reviewed-by: Anthony Krowiak Link: https://lore.kernel.org/r/20240215153144.14747-1-jjherne@linux.ibm.com Signed-off-by: Heiko Carstens --- drivers/s390/crypto/vfio_ap_ops.c | 35 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 983b3b16196c..fc169bc61593 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -659,6 +659,21 @@ static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev) AP_DOMAINS); } +static bool _queue_passable(struct vfio_ap_queue *q) +{ + if (!q) + return false; + + switch (q->reset_status.response_code) { + case AP_RESPONSE_NORMAL: + case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: + return true; + default: + return false; + } +} + /* * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev * to ensure no queue devices are passed through to @@ -687,7 +702,6 @@ static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev, unsigned long apid, apqi, apqn; DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES); DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS); - struct vfio_ap_queue *q; bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES); bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS); @@ -716,8 +730,7 @@ static bool vfio_ap_mdev_filter_matrix(struct ap_matrix_mdev *matrix_mdev, * hardware device. */ apqn = AP_MKQID(apid, apqi); - q = vfio_ap_mdev_get_queue(matrix_mdev, apqn); - if (!q || q->reset_status.response_code) { + if (!_queue_passable(vfio_ap_mdev_get_queue(matrix_mdev, apqn))) { clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm); /* @@ -1691,6 +1704,7 @@ static int apq_status_check(int apqn, struct ap_queue_status *status) switch (status->response_code) { case AP_RESPONSE_NORMAL: case AP_RESPONSE_DECONFIGURED: + case AP_RESPONSE_CHECKSTOPPED: return 0; case AP_RESPONSE_RESET_IN_PROGRESS: case AP_RESPONSE_BUSY: @@ -1747,14 +1761,6 @@ static void apq_reset_check(struct work_struct *reset_work) memcpy(&q->reset_status, &status, sizeof(status)); continue; } - /* - * When an AP adapter is deconfigured, the - * associated queues are reset, so let's set the - * status response code to 0 so the queue may be - * passed through (i.e., not filtered) - */ - if (status.response_code == AP_RESPONSE_DECONFIGURED) - q->reset_status.response_code = 0; if (q->saved_isc != VFIO_AP_ISC_INVALID) vfio_ap_free_aqic_resources(q); break; @@ -1781,12 +1787,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) queue_work(system_long_wq, &q->reset_work); break; case AP_RESPONSE_DECONFIGURED: - /* - * When an AP adapter is deconfigured, the associated - * queues are reset, so let's set the status response code to 0 - * so the queue may be passed through (i.e., not filtered). - */ - q->reset_status.response_code = 0; + case AP_RESPONSE_CHECKSTOPPED: vfio_ap_free_aqic_resources(q); break; default: From bbe37e3e351bce348b14d50240cd5be79542e203 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 20 Feb 2024 14:47:38 +0100 Subject: [PATCH 107/117] s390/configs: increase number of LOCKDEP_BITS Set LOCKDEP_BITS to 16 and LOCKDEP_CHAINS_BITS to 17, since test systems frequently run out of lockdep entries and lockdep chains. Signed-off-by: Heiko Carstens --- arch/s390/configs/debug_defconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index c924be0d7ed8..d0c44973dd09 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -824,6 +824,8 @@ CONFIG_TEST_LOCKUP=m CONFIG_DEBUG_PREEMPT=y CONFIG_PROVE_LOCKING=y CONFIG_LOCK_STAT=y +CONFIG_LOCKDEP_BITS=16 +CONFIG_LOCKDEP_CHAINS_BITS=17 CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_DEBUG_LOCKING_API_SELFTESTS=y CONFIG_DEBUG_IRQFLAGS=y From 778412ab915d6d257300a645e25d219a6344a58e Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 15:30:22 +0100 Subject: [PATCH 108/117] s390/ap: rearm APQNs bindings complete completion The APQN bindings complete completion was used to reflect that 1st the AP bus initial scan is done and 2nd all the detected APQNs have been bound to a device driver. This was a single-shot action. However, as the AP bus supports hot-plug it may be that new APQNs appear reflected as new AP queue and card devices which need to be bound to appropriate device drivers. So the condition that all existing AP queue devices are bound to device drivers may go away for a certain time. This patch now checks during AP bus scan for maybe new AP devices appearing and does a re-init of the internal completion variable. So the AP bus function ap_wait_apqn_bindings_complete() now may block on this condition variable even later after initial scan is through when new APQNs appear which need to get bound. This patch also moves the check for binding complete invocation from the probe function to the end of the AP bus scan function. This change also covers some weird scenarios where during a card hotplug the binding of the card device was sufficient for binding complete but the queue devices where still in the process of being discovered. As of now this change has no impact on existing code. The behavior change in the now later bindings complete should not impact any code (and has been tested so far). The only exploiter is the zcrypt function zcrypt_wait_api_operational() which only initial calls ap_wait_apqn_bindings_complete(). However, this new behavior of the AP bus wait for APQNs bindings complete function will be used in a later patch exploiting this for the zcrypt API layer. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 93 +++++++++++++++++++++++++------- drivers/s390/crypto/ap_bus.h | 6 ++- drivers/s390/crypto/zcrypt_api.c | 2 +- 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index a58190e4b745..56f32f732ddf 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -90,8 +90,8 @@ static atomic64_t ap_scan_bus_count; /* # of bindings complete since init */ static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0); -/* completion for initial APQN bindings complete */ -static DECLARE_COMPLETION(ap_init_apqn_bindings_complete); +/* completion for APQN bindings complete */ +static DECLARE_COMPLETION(ap_apqn_bindings_complete); static struct ap_config_info *ap_qci_info; static struct ap_config_info *ap_qci_info_old; @@ -754,7 +754,7 @@ static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound) } /* - * After initial ap bus scan do check if all existing APQNs are + * After ap bus scan do check if all existing APQNs are * bound to device drivers. */ static void ap_check_bindings_complete(void) @@ -764,9 +764,9 @@ static void ap_check_bindings_complete(void) if (atomic64_read(&ap_scan_bus_count) >= 1) { ap_calc_bound_apqns(&apqns, &bound); if (bound == apqns) { - if (!completion_done(&ap_init_apqn_bindings_complete)) { - complete_all(&ap_init_apqn_bindings_complete); - AP_DBF_INFO("%s complete\n", __func__); + if (!completion_done(&ap_apqn_bindings_complete)) { + complete_all(&ap_apqn_bindings_complete); + pr_debug("%s all apqn bindings complete\n", __func__); } ap_send_bindings_complete_uevent(); } @@ -783,27 +783,29 @@ static void ap_check_bindings_complete(void) * -ETIME is returned. On failures negative return values are * returned to the caller. */ -int ap_wait_init_apqn_bindings_complete(unsigned long timeout) +int ap_wait_apqn_bindings_complete(unsigned long timeout) { + int rc = 0; long l; - if (completion_done(&ap_init_apqn_bindings_complete)) + if (completion_done(&ap_apqn_bindings_complete)) return 0; if (timeout) l = wait_for_completion_interruptible_timeout( - &ap_init_apqn_bindings_complete, timeout); + &ap_apqn_bindings_complete, timeout); else l = wait_for_completion_interruptible( - &ap_init_apqn_bindings_complete); + &ap_apqn_bindings_complete); if (l < 0) - return l == -ERESTARTSYS ? -EINTR : l; + rc = l == -ERESTARTSYS ? -EINTR : l; else if (l == 0 && timeout) - return -ETIME; + rc = -ETIME; - return 0; + pr_debug("%s rc=%d\n", __func__, rc); + return rc; } -EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete); +EXPORT_SYMBOL(ap_wait_apqn_bindings_complete); static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) { @@ -940,8 +942,6 @@ static int ap_device_probe(struct device *dev) if (is_queue_dev(dev)) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); - } else { - ap_check_bindings_complete(); } out: @@ -2136,6 +2136,49 @@ static bool ap_get_configuration(void) sizeof(struct ap_config_info)) != 0; } +/* + * ap_config_has_new_aps - Check current against old qci info if + * new adapters have appeared. Returns true if at least one new + * adapter in the apm mask is showing up. Existing adapters or + * receding adapters are not counted. + */ +static bool ap_config_has_new_aps(void) +{ + + unsigned long m[BITS_TO_LONGS(AP_DEVICES)]; + + if (!ap_qci_info) + return false; + + bitmap_andnot(m, (unsigned long *)ap_qci_info->apm, + (unsigned long *)ap_qci_info_old->apm, AP_DEVICES); + if (!bitmap_empty(m, AP_DEVICES)) + return true; + + return false; +} + +/* + * ap_config_has_new_doms - Check current against old qci info if + * new (usage) domains have appeared. Returns true if at least one + * new domain in the aqm mask is showing up. Existing domains or + * receding domains are not counted. + */ +static bool ap_config_has_new_doms(void) +{ + unsigned long m[BITS_TO_LONGS(AP_DOMAINS)]; + + if (!ap_qci_info) + return false; + + bitmap_andnot(m, (unsigned long *)ap_qci_info->aqm, + (unsigned long *)ap_qci_info_old->aqm, AP_DOMAINS); + if (!bitmap_empty(m, AP_DOMAINS)) + return true; + + return false; +} + /** * ap_scan_bus(): Scan the AP bus for new devices * Runs periodically, workqueue timer (ap_config_time) @@ -2147,10 +2190,21 @@ static void ap_scan_bus(struct work_struct *unused) pr_debug(">%s\n", __func__); - /* config change notify */ + /* (re-)fetch configuration via QCI */ config_changed = ap_get_configuration(); - if (config_changed) + if (config_changed) { + if (ap_config_has_new_aps() || ap_config_has_new_doms()) { + /* + * Appearance of new adapters and/or domains need to + * build new ap devices which need to get bound to an + * device driver. Thus reset the APQN bindings complete + * completion. + */ + reinit_completion(&ap_apqn_bindings_complete); + } + /* post a config change notify */ notify_config_changed(); + } ap_select_domain(); /* loop over all possible adapters */ @@ -2177,9 +2231,10 @@ static void ap_scan_bus(struct work_struct *unused) if (atomic64_inc_return(&ap_scan_bus_count) == 1) { pr_debug("%s init scan complete\n", __func__); ap_send_init_scan_done_uevent(); - ap_check_bindings_complete(); } + ap_check_bindings_complete(); + mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); pr_debug("<%s\n", __func__); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 98814839ef30..be320bd46f65 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -352,8 +352,12 @@ int ap_parse_mask_str(const char *str, * the return value is 0. If the timeout (in jiffies) hits instead * -ETIME is returned. On failures negative return values are * returned to the caller. + * It may be that the AP bus scan finds new devices. Then the + * condition that all APQNs are bound to their device drivers + * is reset to false and this call again blocks until either all + * APQNs are bound to a device driver or the timeout hits again. */ -int ap_wait_init_apqn_bindings_complete(unsigned long timeout); +int ap_wait_apqn_bindings_complete(unsigned long timeout); void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg); void ap_send_online_uevent(struct ap_device *ap_dev, int online); diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index e8742757085b..52990a8553e0 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -2008,7 +2008,7 @@ int zcrypt_wait_api_operational(void) switch (zcrypt_wait_api_state) { case 0: /* initial state, invoke wait for the ap bus complete */ - rc = ap_wait_init_apqn_bindings_complete( + rc = ap_wait_apqn_bindings_complete( msecs_to_jiffies(60 * 1000)); switch (rc) { case 0: From 99b3126e46ef8c5c40291b61958f7b9e78b9f162 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 16:50:05 +0100 Subject: [PATCH 109/117] s390/ap: clarify AP scan bus related functions and variables This patch tries to clarify the functions and variables around the AP scan bus job. All these variables and functions start with ap_scan_bus and are declared in one place now. No functional changes in this patch - only renaming and move of code or declarations. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 43 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 56f32f732ddf..e04369a12d45 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -84,9 +84,6 @@ EXPORT_SYMBOL(ap_perms); DEFINE_MUTEX(ap_perms_mutex); EXPORT_SYMBOL(ap_perms_mutex); -/* # of bus scans since init */ -static atomic64_t ap_scan_bus_count; - /* # of bindings complete since init */ static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0); @@ -102,12 +99,13 @@ static struct ap_config_info *ap_qci_info_old; debug_info_t *ap_dbf_info; /* - * Workqueue timer for bus rescan. + * AP bus rescan related things. */ -static struct timer_list ap_config_timer; -static int ap_config_time = AP_CONFIG_TIME; +static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */ +static int ap_scan_bus_time = AP_CONFIG_TIME; +static struct timer_list ap_scan_bus_timer; static void ap_scan_bus(struct work_struct *); -static DECLARE_WORK(ap_scan_work, ap_scan_bus); +static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus); /* * Tasklet & timer for AP request polling and interrupts @@ -1020,9 +1018,9 @@ void ap_bus_force_rescan(void) return; /* processing a asynchronous bus rescan */ - del_timer(&ap_config_timer); - queue_work(system_long_wq, &ap_scan_work); - flush_work(&ap_scan_work); + del_timer(&ap_scan_bus_timer); + queue_work(system_long_wq, &ap_scan_bus_work); + flush_work(&ap_scan_bus_work); } EXPORT_SYMBOL(ap_bus_force_rescan); @@ -1251,7 +1249,7 @@ static BUS_ATTR_RO(ap_interrupts); static ssize_t config_time_show(const struct bus_type *bus, char *buf) { - return sysfs_emit(buf, "%d\n", ap_config_time); + return sysfs_emit(buf, "%d\n", ap_scan_bus_time); } static ssize_t config_time_store(const struct bus_type *bus, @@ -1261,8 +1259,8 @@ static ssize_t config_time_store(const struct bus_type *bus, if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120) return -EINVAL; - ap_config_time = time; - mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + ap_scan_bus_time = time; + mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); return count; } @@ -2181,7 +2179,7 @@ static bool ap_config_has_new_doms(void) /** * ap_scan_bus(): Scan the AP bus for new devices - * Runs periodically, workqueue timer (ap_config_time) + * Runs periodically, workqueue timer (ap_scan_bus_time) * @unused: Unused pointer. */ static void ap_scan_bus(struct work_struct *unused) @@ -2235,14 +2233,21 @@ static void ap_scan_bus(struct work_struct *unused) ap_check_bindings_complete(); - mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); + mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); pr_debug("<%s\n", __func__); } -static void ap_config_timeout(struct timer_list *unused) +/* + * Callback for the ap_scan_bus_timer + */ +static void ap_scan_bus_timer_callback(struct timer_list *unused) { - queue_work(system_long_wq, &ap_scan_work); + /* + * schedule work into the system long wq which when + * the work is finally executed, calls the AP bus scan. + */ + queue_work(system_long_wq, &ap_scan_bus_work); } static int __init ap_debug_init(void) @@ -2332,7 +2337,7 @@ static int __init ap_module_init(void) ap_root_device->bus = &ap_bus_type; /* Setup the AP bus rescan timer. */ - timer_setup(&ap_config_timer, ap_config_timeout, 0); + timer_setup(&ap_scan_bus_timer, ap_scan_bus_timer_callback, 0); /* * Setup the high resolution poll timer. @@ -2350,7 +2355,7 @@ static int __init ap_module_init(void) goto out_work; } - queue_work(system_long_wq, &ap_scan_work); + queue_work(system_long_wq, &ap_scan_bus_work); return 0; From b5caf05ee8795a628992fe7b5ac3e7b9bbd735c5 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 17:15:47 +0100 Subject: [PATCH 110/117] s390/ap: rework ap_scan_bus() to return true on config change The AP scan bus function now returns true if there have been any config changes detected. This will become important in a follow up patch which will exploit this hint for further actions. This also required to have the AP scan bus timer callback reworked as the function signature has changed to bool ap_scan_bus(void). Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index e04369a12d45..2027baec995c 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -104,8 +104,8 @@ debug_info_t *ap_dbf_info; static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */ static int ap_scan_bus_time = AP_CONFIG_TIME; static struct timer_list ap_scan_bus_timer; -static void ap_scan_bus(struct work_struct *); -static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus); +static void ap_scan_bus_wq_callback(struct work_struct *); +static DECLARE_WORK(ap_scan_bus_work, ap_scan_bus_wq_callback); /* * Tasklet & timer for AP request polling and interrupts @@ -2179,12 +2179,13 @@ static bool ap_config_has_new_doms(void) /** * ap_scan_bus(): Scan the AP bus for new devices - * Runs periodically, workqueue timer (ap_scan_bus_time) - * @unused: Unused pointer. + * Returns true if any config change has been detected + * otherwise false. */ -static void ap_scan_bus(struct work_struct *unused) +static bool ap_scan_bus(void) { - int ap, config_changed = 0; + bool config_changed; + int ap; pr_debug(">%s\n", __func__); @@ -2235,11 +2236,14 @@ static void ap_scan_bus(struct work_struct *unused) mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ); - pr_debug("<%s\n", __func__); + pr_debug("<%s config_changed=%d\n", __func__, config_changed); + + return config_changed; } /* * Callback for the ap_scan_bus_timer + * Runs periodically, workqueue timer (ap_scan_bus_time) */ static void ap_scan_bus_timer_callback(struct timer_list *unused) { @@ -2250,6 +2254,15 @@ static void ap_scan_bus_timer_callback(struct timer_list *unused) queue_work(system_long_wq, &ap_scan_bus_work); } +/* + * Callback for the ap_scan_bus_work + */ +static void ap_scan_bus_wq_callback(struct work_struct *unused) +{ + /* now finally do the AP bus scan */ + ap_scan_bus(); +} + static int __init ap_debug_init(void) { ap_dbf_info = debug_register("ap", 2, 1, From eacf5b3651c530e0666efbd64e1d1115258c5903 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 17:42:50 +0100 Subject: [PATCH 111/117] s390/ap: introduce mutex to lock the AP bus scan Rework the invocations around ap_scan_bus(): - Protect ap_scan_bus() with a mutex to make sure only one scan at a time is running. - The workqueue invocation which is triggered by either the module init or via AP bus scan timer expiration uses this mutex and if there is already a scan running, the work is simple aborted (as the job is done by another task). - The ap_bus_force_rescan() which is invoked by higher level layers mostly on failures which indicate a bus scan may help is reworked to call ap_scan_bus() direct instead of enqueuing work into a system workqueue and waiting for that to finish. Of course the mutex is respected and in case of another task already running a bus scan the shortcut of waiting for this scan to finish and reusing the scan result is taken. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 69 ++++++++++++++++++++++++++++++------ drivers/s390/crypto/ap_bus.h | 2 +- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 2027baec995c..cce0bafd4c92 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -101,6 +101,9 @@ debug_info_t *ap_dbf_info; /* * AP bus rescan related things. */ +static bool ap_scan_bus(void); +static bool ap_scan_bus_result; /* result of last ap_scan_bus() */ +static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */ static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */ static int ap_scan_bus_time = AP_CONFIG_TIME; static struct timer_list ap_scan_bus_timer; @@ -1011,16 +1014,47 @@ void ap_driver_unregister(struct ap_driver *ap_drv) } EXPORT_SYMBOL(ap_driver_unregister); -void ap_bus_force_rescan(void) +/* + * Enforce a synchronous AP bus rescan. + * Returns true if the bus scan finds a change in the AP configuration + * and AP devices have been added or deleted when this function returns. + */ +bool ap_bus_force_rescan(void) { - /* Only trigger AP bus scans after the initial scan is done */ - if (atomic64_read(&ap_scan_bus_count) <= 0) - return; + unsigned long scan_counter = atomic64_read(&ap_scan_bus_count); + bool rc = false; - /* processing a asynchronous bus rescan */ - del_timer(&ap_scan_bus_timer); - queue_work(system_long_wq, &ap_scan_bus_work); - flush_work(&ap_scan_bus_work); + pr_debug(">%s scan counter=%lu\n", __func__, scan_counter); + + /* Only trigger AP bus scans after the initial scan is done */ + if (scan_counter <= 0) + goto out; + + /* Try to acquire the AP scan bus mutex */ + if (mutex_trylock(&ap_scan_bus_mutex)) { + /* mutex acquired, run the AP bus scan */ + ap_scan_bus_result = ap_scan_bus(); + rc = ap_scan_bus_result; + mutex_unlock(&ap_scan_bus_mutex); + goto out; + } + + /* + * Mutex acquire failed. So there is currently another task + * already running the AP bus scan. Then let's simple wait + * for the lock which means the other task has finished and + * stored the result in ap_scan_bus_result. + */ + if (mutex_lock_interruptible(&ap_scan_bus_mutex)) { + /* some error occurred, ignore and go out */ + goto out; + } + rc = ap_scan_bus_result; + mutex_unlock(&ap_scan_bus_mutex); + +out: + pr_debug("%s rc=%d\n", __func__, rc); + return rc; } EXPORT_SYMBOL(ap_bus_force_rescan); @@ -2179,8 +2213,10 @@ static bool ap_config_has_new_doms(void) /** * ap_scan_bus(): Scan the AP bus for new devices + * Always run under mutex ap_scan_bus_mutex protection + * which needs to get locked/unlocked by the caller! * Returns true if any config change has been detected - * otherwise false. + * during the scan, otherwise false. */ static bool ap_scan_bus(void) { @@ -2259,8 +2295,19 @@ static void ap_scan_bus_timer_callback(struct timer_list *unused) */ static void ap_scan_bus_wq_callback(struct work_struct *unused) { - /* now finally do the AP bus scan */ - ap_scan_bus(); + /* + * Try to invoke an ap_scan_bus(). If the mutex acquisition + * fails there is currently another task already running the + * AP scan bus and there is no need to wait and re-trigger the + * scan again. Please note at the end of the scan bus function + * the AP scan bus timer is re-armed which triggers then the + * ap_scan_bus_timer_callback which enqueues a work into the + * system_long_wq which invokes this function here again. + */ + if (mutex_trylock(&ap_scan_bus_mutex)) { + ap_scan_bus_result = ap_scan_bus(); + mutex_unlock(&ap_scan_bus_mutex); + } } static int __init ap_debug_init(void) diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index be320bd46f65..59c7ed49aa02 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -266,7 +266,7 @@ int ap_sb_available(void); bool ap_is_se_guest(void); void ap_wait(enum ap_sm_wait wait); void ap_request_timeout(struct timer_list *t); -void ap_bus_force_rescan(void); +bool ap_bus_force_rescan(void); int ap_test_config_usage_domain(unsigned int domain); int ap_test_config_ctrl_domain(unsigned int domain); From 77c51fc6fba7af918db58808d38513f21e91493d Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Fri, 26 Jan 2024 15:43:10 +0100 Subject: [PATCH 112/117] s390/zcrypt: introduce retries on in-kernel send CPRB functions The both functions zcrypt_send_cprb() and zcrypt_send_ep11_cprb() are used to send CPRBs in-kernel from different sources. For example the pkey module may call one of the functions in zcrypt_ep11misc.c to trigger a derive of a protected key from a secure key blob via an existing crypto card. These both functions are then the internal API to send the CPRB and receive the response. All the ioctl functions to send an CPRB down to the addressed crypto card use some kind of retry mechanism. When the first attempt fails with ENODEV, a bus rescan is triggered and a loop with retries is carried out. For the both named internal functions there was never any retry attempt made. This patch now introduces the retry code even for this both internal functions to have effectively same behavior on sending an CPRB from an in-kernel source and sending an CPRB from userspace via ioctl. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/zcrypt_api.c | 42 ++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 52990a8553e0..56efaa63841f 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -977,7 +977,26 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, long zcrypt_send_cprb(struct ica_xcRB *xcrb) { - return _zcrypt_send_cprb(false, &ap_perms, NULL, xcrb); + struct zcrypt_track tr; + int rc; + + memset(&tr, 0, sizeof(tr)); + + do { + rc = _zcrypt_send_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) + do { + rc = _zcrypt_send_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) + rc = -EIO; + if (rc) + pr_debug("%s rc=%d\n", __func__, rc); + + return rc; } EXPORT_SYMBOL(zcrypt_send_cprb); @@ -1162,7 +1181,26 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb) { - return _zcrypt_send_ep11_cprb(false, &ap_perms, NULL, xcrb); + struct zcrypt_track tr; + int rc; + + memset(&tr, 0, sizeof(tr)); + + do { + rc = _zcrypt_send_ep11_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) + do { + rc = _zcrypt_send_ep11_cprb(false, &ap_perms, &tr, xcrb); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) + rc = -EIO; + if (rc) + pr_debug("%s rc=%d\n", __func__, rc); + + return rc; } EXPORT_SYMBOL(zcrypt_send_ep11_cprb); From c3384369bc530e95958985918771af6d7b74d014 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 30 Jan 2024 18:35:51 +0100 Subject: [PATCH 113/117] s390/zcrypt: improve zcrypt retry behavior This patch reworks and improves the zcrypt retry behavior: - The zcrypt_rescan_req counter has been removed. This counter variable has been increased on some transport errors and was used as a gatekeeper for AP bus rescans. - Rework of the zcrypt_process_rescan() function to not use the above counter variable any more. Instead now always the ap_bus_force_rescan() function is called (as this has been improved with a previous patch). - As the zcrpyt_process_rescan() function is called in all cprb send functions in case of the first attempt to send failed with ENODEV now before the next attempt to send an cprb is started. - Introduce a define ZCRYPT_WAIT_BINDINGS_COMPLETE_MS for the amount of milliseconds to have the zcrypt API wait for AP bindings complete. This amount has been reduced to 30s (was 60s). Some playing around showed that 30s is a really fair limit. The result of the above together with the patches to improve the AP scan bus functions is that after the first loop of cprb send retries when the result is a ENODEV the AP bus scan is always triggered (synchronous). If the AP bus scan detects changes in the configuration, all the send functions now retry when the first attempt was failing with ENODEV in the hope that now a suitable device has appeared. About concurrency: The ap_bus_force_rescan() uses a mutex to ensure only one active AP bus scan is running. Another caller of this function is blocked as long as the scan is running but does not cause yet another scan. Instead the result of the 'other' scan is used. This affects only tasks which run into an initial ENODEV. Tasks with successful delivery of cprbs will never invoke the bus scan and thus never get blocked by the mutex. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/zcrypt_api.c | 122 +++++++++++------------------ drivers/s390/crypto/zcrypt_api.h | 9 +++ drivers/s390/crypto/zcrypt_error.h | 5 +- 3 files changed, 57 insertions(+), 79 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 56efaa63841f..02c503f16bc2 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -60,10 +60,6 @@ DEFINE_SPINLOCK(zcrypt_list_lock); LIST_HEAD(zcrypt_card_list); static atomic_t zcrypt_open_count = ATOMIC_INIT(0); -static atomic_t zcrypt_rescan_count = ATOMIC_INIT(0); - -atomic_t zcrypt_rescan_req = ATOMIC_INIT(0); -EXPORT_SYMBOL(zcrypt_rescan_req); static LIST_HEAD(zcrypt_ops_list); @@ -72,20 +68,15 @@ debug_info_t *zcrypt_dbf_info; /* * Process a rescan of the transport layer. - * - * Returns 1, if the rescan has been processed, otherwise 0. + * Runs a synchronous AP bus rescan. + * Returns true if something has changed (for example the + * bus scan has found and build up new devices) and it is + * worth to do a retry. Otherwise false is returned meaning + * no changes on the AP bus level. */ -static inline int zcrypt_process_rescan(void) +static inline bool zcrypt_process_rescan(void) { - if (atomic_read(&zcrypt_rescan_req)) { - atomic_set(&zcrypt_rescan_req, 0); - atomic_inc(&zcrypt_rescan_count); - ap_bus_force_rescan(); - ZCRYPT_DBF_INFO("%s rescan count=%07d\n", __func__, - atomic_inc_return(&zcrypt_rescan_count)); - return 1; - } - return 0; + return ap_bus_force_rescan(); } void zcrypt_msgtype_register(struct zcrypt_ops *zops) @@ -1481,16 +1472,13 @@ static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { @@ -1513,16 +1501,13 @@ static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = zcrypt_rsa_crt(perms, &tr, &crt); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_crt(perms, &tr, &crt); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) { @@ -1545,16 +1530,13 @@ static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) @@ -1578,16 +1560,13 @@ static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg) do { rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) @@ -1758,16 +1737,13 @@ static long trans_modexpo32(struct ap_perms *perms, struct file *filp, mex64.n_modulus = compat_ptr(mex32.n_modulus); do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) @@ -1811,16 +1787,13 @@ static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp, crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv); do { rc = zcrypt_rsa_crt(perms, &tr, &crt64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = zcrypt_rsa_crt(perms, &tr, &crt64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; if (rc) @@ -1883,16 +1856,13 @@ static long trans_xcrb32(struct ap_perms *perms, struct file *filp, xcrb64.status = xcrb32.status; do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); + + /* on ENODEV failure: retry once again after a requested rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) do { rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64); - if (rc == -EAGAIN) - tr.again_counter++; - } while (rc == -EAGAIN && tr.again_counter < TRACK_AGAIN_MAX); + } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) rc = -EIO; xcrb32.reply_control_blk_length = xcrb64.reply_control_blk_length; @@ -1964,8 +1934,8 @@ static int zcrypt_rng_data_read(struct hwrng *rng, u32 *data) */ if (zcrypt_rng_buffer_index == 0) { rc = zcrypt_rng((char *)zcrypt_rng_buffer); - /* on failure: retry once again after a requested rescan */ - if ((rc == -ENODEV) && (zcrypt_process_rescan())) + /* on ENODEV failure: retry once again after an AP bus rescan */ + if (rc == -ENODEV && zcrypt_process_rescan()) rc = zcrypt_rng((char *)zcrypt_rng_buffer); if (rc < 0) return -EIO; @@ -2027,7 +1997,7 @@ void zcrypt_rng_device_remove(void) * an asynchronous job. This function waits until these initial jobs * are done and so the zcrypt api should be ready to serve crypto * requests - if there are resources available. The function uses an - * internal timeout of 60s. The very first caller will either wait for + * internal timeout of 30s. The very first caller will either wait for * ap bus bindings complete or the timeout happens. This state will be * remembered for further callers which will only be blocked until a * decision is made (timeout or bindings complete). @@ -2047,7 +2017,7 @@ int zcrypt_wait_api_operational(void) case 0: /* initial state, invoke wait for the ap bus complete */ rc = ap_wait_apqn_bindings_complete( - msecs_to_jiffies(60 * 1000)); + msecs_to_jiffies(ZCRYPT_WAIT_BINDINGS_COMPLETE_MS)); switch (rc) { case 0: /* ap bus bindings are complete */ diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index de659954c8f7..4ed481df57ca 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -38,6 +38,15 @@ */ #define ZCRYPT_RNG_BUFFER_SIZE 4096 +/** + * The zcrypt_wait_api_operational() function waits this + * amount in milliseconds for ap_wait_aqpn_bindings_complete(). + * Also on a cprb send failure with ENODEV the send functions + * trigger an ap bus rescan and wait this time in milliseconds + * for ap_wait_aqpn_bindings_complete() before resending. + */ +#define ZCRYPT_WAIT_BINDINGS_COMPLETE_MS 30000 + /* * Identifier for Crypto Request Performance Index */ diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h index a44fcfcec938..46e27b43a8af 100644 --- a/drivers/s390/crypto/zcrypt_error.h +++ b/drivers/s390/crypto/zcrypt_error.h @@ -119,10 +119,9 @@ static inline int convert_error(struct zcrypt_queue *zq, case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */ case REP82_ERROR_TRANSPORT_FAIL: /* 0x90 */ /* - * Msg to wrong type or card/infrastructure failure. - * Trigger rescan of the ap bus, trigger retry request. + * Msg to wrong type or card/infrastructure failure. Return + * EAGAIN, the upper layer may do a retry on the request. */ - atomic_set(&zcrypt_rescan_req, 1); /* For type 86 response show the apfs value (failure reason) */ if (ehdr->reply_code == REP82_ERROR_TRANSPORT_FAIL && ehdr->type == TYPE86_RSP_CODE) { From 5dabfecad4a0868201af2ffb69dcd3223f9ca630 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 8 Feb 2024 10:33:35 +0100 Subject: [PATCH 114/117] s390/pkey: improve pkey retry behavior This patch reworks and improves the pkey retry behavior for the pkey_ep11key2pkey() function. In contrast to the pkey_skey2pkey() function which is used to trigger a protected key derivation from an CCA secure data or cipher key the EP11 counterpart function had no proper retry loop implemented. This patch now introduces code which acts similar to the retry already done for CCA keys for this function used for EP11 keys. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/pkey_api.c | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 4c60b5acbae1..dccf664a3d95 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -293,33 +293,36 @@ static int pkey_ep11key2pkey(const u8 *key, size_t keylen, u8 *protkey, u32 *protkeylen, u32 *protkeytype) { u32 nr_apqns, *apqns = NULL; + int i, j, rc = -ENODEV; u16 card, dom; - int i, rc; zcrypt_wait_api_operational(); - /* build a list of apqns suitable for this key */ - rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, - ZCRYPT_CEX7, - ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, - ep11_kb_wkvp(key, keylen)); - if (rc) - goto out; + /* try two times in case of failure */ + for (i = 0; i < 2 && rc; i++) { - /* go through the list of apqns and try to derive an pkey */ - for (rc = -ENODEV, i = 0; i < nr_apqns; i++) { - card = apqns[i] >> 16; - dom = apqns[i] & 0xFFFF; - rc = ep11_kblob2protkey(card, dom, key, keylen, - protkey, protkeylen, protkeytype); - if (rc == 0) - break; + /* build a list of apqns suitable for this key */ + rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, + ZCRYPT_CEX7, + ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4, + ep11_kb_wkvp(key, keylen)); + if (rc) + continue; /* retry findcard on failure */ + + /* go through the list of apqns and try to derive an pkey */ + for (rc = -ENODEV, j = 0; j < nr_apqns && rc; j++) { + card = apqns[j] >> 16; + dom = apqns[j] & 0xFFFF; + rc = ep11_kblob2protkey(card, dom, key, keylen, + protkey, protkeylen, protkeytype); + } + + kfree(apqns); } -out: - kfree(apqns); if (rc) pr_debug("%s failed rc=%d\n", __func__, rc); + return rc; } From ed6776c96c60ffe92e79e8f18bf5afd2f6e0eb79 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 8 Feb 2024 08:34:25 +0100 Subject: [PATCH 115/117] s390/crypto: remove retry loop with sleep from PAES pkey invocation Upon calling the pkey module to (re-)derive an protected key from a secure key the PAES implementation did a retry 3 times with an 1000 ms sleep after each failure. This patch removes this retry loop - retries should be done if needed in a lower layer but the consumer of the pkey module functions should not be bothered with retries. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- arch/s390/crypto/paes_s390.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 55ee5567a5ea..99f7e1f2b70a 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -125,20 +125,8 @@ struct s390_pxts_ctx { static inline int __paes_keyblob2pkey(struct key_blob *kb, struct pkey_protkey *pk) { - int i, ret; - - /* try three times in case of failure */ - for (i = 0; i < 3; i++) { - if (i > 0 && ret == -EAGAIN && in_task()) - if (msleep_interruptible(1000)) - return -EINTR; - ret = pkey_keyblob2pkey(kb->key, kb->keylen, - pk->protkey, &pk->len, &pk->type); - if (ret == 0) - break; - } - - return ret; + return pkey_keyblob2pkey(kb->key, kb->keylen, + pk->protkey, &pk->len, &pk->type); } static inline int __paes_convert_key(struct s390_paes_ctx *ctx) From cb0cd4ee11142339f2d47eef6db274290b7a482d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sat, 2 Mar 2024 20:22:09 +0100 Subject: [PATCH 116/117] s390/cache: prevent rebuild of shared_cpu_list With commit 36bbc5b4ffab ("cacheinfo: Allow early detection and population of cache attributes") the shared cpu list for each cache level higher than L1 is rebuilt even if the list already has been set up. This is caused by the removal of the cpumask_empty() check within cache_shared_cpu_map_setup(). However architectures can enforce that the shared cpu list is not rebuilt by simply setting cpu_map_populated of the per cpu cache info structure to true, which is also the fix for this problem. Before: $ cat /sys/devices/system/cpu/cpu1/cache/index2/shared_cpu_list 0-7 After: $ cat /sys/devices/system/cpu/cpu1/cache/index2/shared_cpu_list 1 Fixes: 36bbc5b4ffab ("cacheinfo: Allow early detection and population of cache attributes") Signed-off-by: Heiko Carstens --- arch/s390/kernel/cache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 56254fa06f99..4f2669030220 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -166,5 +166,6 @@ int populate_cache_leaves(unsigned int cpu) ci_leaf_init(this_leaf++, pvt, ctype, level, cpu); } } + this_cpu_ci->cpu_map_populated = true; return 0; } From fa9e3139e6c5ac756e1ab2a6c7eca99eb684a2fe Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Thu, 7 Mar 2024 09:46:11 +0100 Subject: [PATCH 117/117] s390/tools: handle rela R_390_GOTPCDBL/R_390_GOTOFF64 lkp test robot reported unhandled relocation type: R_390_GOTPCDBL, when kernel is built with -fno-PIE. relocs tool reads vmlinux and handles absolute relocations. PC relative relocs doesn't need adjustment. Also, the R_390_GOTPCDBL/R_390_GOTOFF64 relocations are present currently only when KASAN is enabled. The following program can create a R_390_GOTPCDBL/R_390_GOTOFF64 reloc (with fPIE/fPIC). void funcb(int *b) { *b = *b + 100; } void gen_gotoff(void) { int b = 10; funcb (&b); } gcc -c sample.c -fPIC -fsanitize=kernel-address --param asan-stack=1 The above example (built with -fPIC) was linked to one of the built-in.a (built with -fno-PIE) and checked for correctness with kaslr enabled. Both the relocs turns out relative and can be skipped. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202402221404.T2TGs8El-lkp@intel.com/ Fixes: 55dc65b46023 ("s390: add relocs tool") Signed-off-by: Sumanth Korikkar Acked-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/tools/relocs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/tools/relocs.c b/arch/s390/tools/relocs.c index db8bcbf9d8f8..30a732c808f3 100644 --- a/arch/s390/tools/relocs.c +++ b/arch/s390/tools/relocs.c @@ -276,6 +276,8 @@ static int do_reloc(struct section *sec, Elf_Rel *rel) case R_390_PC32DBL: case R_390_PLT32DBL: case R_390_GOTENT: + case R_390_GOTPCDBL: + case R_390_GOTOFF64: break; case R_390_64: add_reloc(&relocs64, offset);