s390x/kvm/migration/cpumodel: fixes, enhancements and cleanups

- add a network boot rom for s390 (Thomas Huth)
 - migration of storage attributes like the CMMA used/unused state
 - PCI related enhancements - full support for aen, ais and zpci
 - migration support for css with vmstates (Halil Pasic)
 - cpu model enhancements for cpu features
 - guarded storage support
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2.0.14 (GNU/Linux)
 
 iQIcBAABAgAGBQJZaJ3gAAoJEBF7vIC1phx8VSAP/1zKh7ti4Y2dIVb94c1tvECE
 LRNdCdAPhEqL6zybty85aG04sjAmSu50NGfo5t8AGq1U9WBWrCy7/wWSFdK2GI63
 Umc1fR7aBF9FiFayKONhExaREh6gSWVHZF1RyaPIWnnjRIeX8nqgPEnpdZNiVVrG
 5cKHV2SUd5pMDJUiQdZGZgbgG1c+MWJx2BHoduM+K0UnmFjpyLCL4Rq58Q2Q87Nj
 /+yPSVApFFeMsDpem6DNttE6Msa+V+K+EmRhRKqZNOWrdRKH5vvj6Fl/LSxVtd9c
 CEG+aZGjFd693uP9ge0WmjeUJtVHIGt9xKdeU0d7FijZWehjsIqalLoqapzK8ddF
 h6HJuNsmk/SZF7O9JsbHT3Epyr+7Hk0dx78Ku1GNQuUxtFL93eyIJmRdgz7Zo3Lj
 ZTPJvCA13GjPWtgzG5dG3JH1hiAS+Yai18BgdzGbs+qfMCwPdbWkoqg7sARwAJNe
 50fo/ayJvcmHJnSNO6hErFoU38WctGgO8fWp+oVvD8Um1ny1aBFFuJgJIMf47nhu
 x1IdA6UGrNN0yNC4/UgyYBDV1hfvo/phMdoHqle9AcMmPYOD1DBr0genK/bYbICk
 Dio7og9nKgheLRBHz2u5TuYcCsfE/7rtwZX+iXMvoC7VE7Dqs+Q7Zjwwwtwj4x9F
 FwWuf/Bv1s6IkVLlP8Ow
 =2bOV
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20170714' into staging

s390x/kvm/migration/cpumodel: fixes, enhancements and cleanups

- add a network boot rom for s390 (Thomas Huth)
- migration of storage attributes like the CMMA used/unused state
- PCI related enhancements - full support for aen, ais and zpci
- migration support for css with vmstates (Halil Pasic)
- cpu model enhancements for cpu features
- guarded storage support

# gpg: Signature made Fri 14 Jul 2017 11:33:04 BST
# gpg:                using RSA key 0x117BBC80B5A61C7C
# gpg: Good signature from "Christian Borntraeger (IBM) <borntraeger@de.ibm.com>"
# Primary key fingerprint: F922 9381 A334 08F9 DBAB  FBCA 117B BC80 B5A6 1C7C

* remotes/borntraeger/tags/s390x-20170714: (40 commits)
  s390x/gdb: add gs registers
  s390x/arch_dump: also dump guarded storage control block
  s390x/kvm: enable guarded storage
  s390x/kvm: Enable KSS facility for nested virtualization
  s390x/cpumodel: add esop/esop2 to z12 model
  s390x/cpumodel: we are always in zarchitecture mode
  s390x/cpumodel: wire up new hardware features
  s390x/flic: migrate ais states
  s390x/cpumodel: add zpci, aen and ais facilities
  s390x: initialize cpu firstly
  pc-bios/s390: rebuild s390-ccw.img
  pc-bios/s390: add s390-netboot.img
  pc-bios/s390-ccw: Link libnet into the netboot image and do the TFTP load
  pc-bios/s390-ccw: Add virtio-net driver code
  pc-bios/s390-ccw: Add core files for the network bootloading program
  roms/SLOF: Update submodule to latest status
  pc-bios/s390-ccw: Add code for virtio feature negotiation
  pc-bios/s390-ccw: Remove unused structs from virtio.h
  pc-bios/s390-ccw: Move byteswap functions to a separate header
  pc-bios/s390-ccw: Add a write() function for stdio
  ...

Conflicts:
	target/s390x/kvm.c

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

Makefile

@@ -553,7 +553,7 @@ efi-e1000e.rom efi-vmxnet3.rom \
 qemu-icon.bmp qemu_logo_no_text.svg \
 bamboo.dtb petalogix-s3adsp1800.dtb petalogix-ml605.dtb \
 multiboot.bin linuxboot.bin linuxboot_dma.bin kvmvapic.bin \
-s390-ccw.img \
+s390-ccw.img s390-netboot.img \
 spapr-rtas.bin slof.bin skiboot.lid \
 palcode-clipper \
 u-boot.e500 \

configure

@@ -6231,7 +6231,7 @@ case "$target_name" in
     echo "TARGET_ABI32=y" >> $config_target_mak
   ;;
   s390x)
-    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml s390-cr.xml s390-virt.xml"
+    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml s390-cr.xml s390-virt.xml s390-gs.xml"
   ;;
   tilegx)
   ;;

gdb-xml/s390-gs.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.gs">
+  <reg name="gs_reserved" bitsize="64" type="uint64" group="system"/>
+  <reg name="gsd" bitsize="64" type="uint64" group="system"/>
+  <reg name="gssm" bitsize="64" type="uint64" group="system"/>
+  <reg name="gsepla" bitsize="64" type="data_ptr" group="system"/>
+</feature>

hmp-commands-info.hx

@@ -775,6 +775,22 @@ STEXI
 @item info skeys @var{address}
 @findex skeys
 Display the value of a storage key (s390 only)
+ETEXI
+
+#if defined(TARGET_S390X)
+    {
+        .name       = "cmma",
+        .args_type  = "addr:l,count:l?",
+        .params     = "address [count]",
+        .help       = "Display the values of the CMMA storage attributes for a range of pages",
+        .cmd        = hmp_info_cmma,
+    },
+#endif
+
+STEXI
+@item info cmma @var{address}
+@findex cmma
+Display the values of the CMMA storage attributes for a range of pages (s390 only)
 ETEXI
 
     {

hmp-commands.hx

@@ -1151,6 +1151,22 @@ STEXI
 @item dump-skeys @var{filename}
 @findex dump-skeys
 Save guest storage keys to a file.
+ETEXI
+
+#if defined(TARGET_S390X)
+    {
+        .name       = "migration_mode",
+        .args_type  = "mode:i",
+        .params     = "mode",
+        .help       = "Enables or disables migration mode\n",
+        .cmd        = hmp_migrationmode,
+    },
+#endif
+
+STEXI
+@item migration_mode @var{mode}
+@findex migration_mode
+Enables or disables migration mode.
 ETEXI
 
     {

hw/intc/s390_flic.c

@@ -13,8 +13,11 @@
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
+#include "hw/s390x/ioinst.h"
 #include "hw/s390x/s390_flic.h"
+#include "hw/s390x/css.h"
 #include "trace.h"
+#include "cpu.h"
 #include "hw/qdev.h"
 #include "qapi/error.h"
 #include "hw/s390x/s390-virtio-ccw.h"
@@ -48,7 +51,7 @@ void s390_flic_init(void)
 
 static int qemu_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
                                          uint8_t isc, bool swap,
-                                         bool is_maskable)
+                                         bool is_maskable, uint8_t flags)
 {
     /* nothing to do */
     return 0;
@@ -79,15 +82,91 @@ static int qemu_s390_clear_io_flic(S390FLICState *fs, uint16_t subchannel_id,
     return -ENOSYS;
 }
 
+static int qemu_s390_modify_ais_mode(S390FLICState *fs, uint8_t isc,
+                                     uint16_t mode)
+{
+    QEMUS390FLICState *flic  = QEMU_S390_FLIC(fs);
+
+    switch (mode) {
+    case SIC_IRQ_MODE_ALL:
+        flic->simm &= ~AIS_MODE_MASK(isc);
+        flic->nimm &= ~AIS_MODE_MASK(isc);
+        break;
+    case SIC_IRQ_MODE_SINGLE:
+        flic->simm |= AIS_MODE_MASK(isc);
+        flic->nimm &= ~AIS_MODE_MASK(isc);
+        break;
+    default:
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int qemu_s390_inject_airq(S390FLICState *fs, uint8_t type,
+                                 uint8_t isc, uint8_t flags)
+{
+    QEMUS390FLICState *flic = QEMU_S390_FLIC(fs);
+    bool flag = flags & S390_ADAPTER_SUPPRESSIBLE;
+    uint32_t io_int_word = (isc << 27) | IO_INT_WORD_AI;
+
+    if (flag && (flic->nimm & AIS_MODE_MASK(isc))) {
+        trace_qemu_s390_airq_suppressed(type, isc);
+        return 0;
+    }
+
+    s390_io_interrupt(0, 0, 0, io_int_word);
+
+    if (flag && (flic->simm & AIS_MODE_MASK(isc))) {
+        flic->nimm |= AIS_MODE_MASK(isc);
+        trace_qemu_s390_suppress_airq(isc, "Single-Interruption Mode",
+                                      "NO-Interruptions Mode");
+    }
+
+    return 0;
+}
+
+static void qemu_s390_flic_reset(DeviceState *dev)
+{
+    QEMUS390FLICState *flic = QEMU_S390_FLIC(dev);
+
+    flic->simm = 0;
+    flic->nimm = 0;
+}
+
+bool ais_needed(void *opaque)
+{
+    S390FLICState *s = opaque;
+
+    return s->ais_supported;
+}
+
+static const VMStateDescription qemu_s390_flic_vmstate = {
+    .name = "qemu-s390-flic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = ais_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(simm, QEMUS390FLICState),
+        VMSTATE_UINT8(nimm, QEMUS390FLICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void qemu_s390_flic_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
 
+    dc->reset = qemu_s390_flic_reset;
+    dc->vmsd = &qemu_s390_flic_vmstate;
     fsc->register_io_adapter = qemu_s390_register_io_adapter;
     fsc->io_adapter_map = qemu_s390_io_adapter_map;
     fsc->add_adapter_routes = qemu_s390_add_adapter_routes;
     fsc->release_adapter_routes = qemu_s390_release_adapter_routes;
     fsc->clear_io_irq = qemu_s390_clear_io_flic;
+    fsc->modify_ais_mode = qemu_s390_modify_ais_mode;
+    fsc->inject_airq = qemu_s390_inject_airq;
 }
 
 static Property s390_flic_common_properties[] = {
@@ -98,12 +177,16 @@ static Property s390_flic_common_properties[] = {
 
 static void s390_flic_common_realize(DeviceState *dev, Error **errp)
 {
-    uint32_t max_batch = S390_FLIC_COMMON(dev)->adapter_routes_max_batch;
+    S390FLICState *fs = S390_FLIC_COMMON(dev);
+    uint32_t max_batch = fs->adapter_routes_max_batch;
 
     if (max_batch > ADAPTER_ROUTES_MAX_GSI) {
         error_setg(errp, "flic property adapter_routes_max_batch too big"
                    " (%d > %d)", max_batch, ADAPTER_ROUTES_MAX_GSI);
+        return;
     }
+
+    fs->ais_supported = s390_has_feat(S390_FEAT_ADAPTER_INT_SUPPRESSION);
 }
 
 static void s390_flic_class_init(ObjectClass *oc, void *data)
@@ -138,6 +221,22 @@ static void qemu_s390_flic_register_types(void)
 
 type_init(qemu_s390_flic_register_types)
 
+static bool adapter_info_so_needed(void *opaque)
+{
+    return css_migration_enabled();
+}
+
+const VMStateDescription vmstate_adapter_info_so = {
+    .name = "s390_adapter_info/summary_offset",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = adapter_info_so_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(summary_offset, AdapterInfo),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_adapter_info = {
     .name = "s390_adapter_info",
     .version_id = 1,
@@ -151,6 +250,10 @@ const VMStateDescription vmstate_adapter_info = {
          */
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_adapter_info_so,
+        NULL
+    }
 };
 
 const VMStateDescription vmstate_adapter_routes = {

hw/intc/s390_flic_kvm.c

@@ -20,6 +20,7 @@
 #include "sysemu/kvm.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/adapter.h"
+#include "hw/s390x/css.h"
 #include "trace.h"
 
 #define FLIC_SAVE_INITIAL_SIZE getpagesize()
@@ -149,6 +150,43 @@ static int kvm_s390_clear_io_flic(S390FLICState *fs, uint16_t subchannel_id,
     return rc ? -errno : 0;
 }
 
+static int kvm_s390_modify_ais_mode(S390FLICState *fs, uint8_t isc,
+                                    uint16_t mode)
+{
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    struct kvm_s390_ais_req req = {
+        .isc = isc,
+        .mode = mode,
+    };
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AISM,
+        .addr = (uint64_t)&req,
+    };
+
+    if (!fs->ais_supported) {
+        return -ENOSYS;
+    }
+
+    return ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr) ? -errno : 0;
+}
+
+static int kvm_s390_inject_airq(S390FLICState *fs, uint8_t type,
+                                uint8_t isc, uint8_t flags)
+{
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    uint32_t id = css_get_adapter_id(type, isc);
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AIRQ_INJECT,
+        .attr = id,
+    };
+
+    if (!fs->ais_supported) {
+        return -ENOSYS;
+    }
+
+    return ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr) ? -errno : 0;
+}
+
 /**
  * __get_all_irqs - store all pending irqs in buffer
  * @flic: pointer to flic device state
@@ -186,13 +224,14 @@ static int __get_all_irqs(KVMS390FLICState *flic,
 
 static int kvm_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
                                         uint8_t isc, bool swap,
-                                        bool is_maskable)
+                                        bool is_maskable, uint8_t flags)
 {
     struct kvm_s390_io_adapter adapter = {
         .id = id,
         .isc = isc,
         .maskable = is_maskable,
         .swap = swap,
+        .flags = flags,
     };
     KVMS390FLICState *flic = KVM_S390_FLIC(fs);
     int r;
@@ -374,7 +413,84 @@ out:
     return r;
 }
 
+typedef struct KVMS390FLICStateMigTmp {
+    KVMS390FLICState *parent;
+    uint8_t simm;
+    uint8_t nimm;
+} KVMS390FLICStateMigTmp;
+
+static void kvm_flic_ais_pre_save(void *opaque)
+{
+    KVMS390FLICStateMigTmp *tmp = opaque;
+    KVMS390FLICState *flic = tmp->parent;
+    struct kvm_s390_ais_all ais;
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AISM_ALL,
+        .addr = (uint64_t)&ais,
+        .attr = sizeof(ais),
+    };
+
+    if (ioctl(flic->fd, KVM_GET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to retrieve kvm flic ais states");
+        return;
+    }
+
+    tmp->simm = ais.simm;
+    tmp->nimm = ais.nimm;
+}
+
+static int kvm_flic_ais_post_load(void *opaque, int version_id)
+{
+    KVMS390FLICStateMigTmp *tmp = opaque;
+    KVMS390FLICState *flic = tmp->parent;
+    struct kvm_s390_ais_all ais = {
+        .simm = tmp->simm,
+        .nimm = tmp->nimm,
+    };
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_AISM_ALL,
+        .addr = (uint64_t)&ais,
+    };
+
+    /* This can happen when the user mis-configures its guests in an
+     * incompatible fashion or without a CPU model. For example using
+     * qemu with -cpu host (which is not migration safe) and do a
+     * migration from a host that has AIS to a host that has no AIS.
+     * In that case the target system will reject the migration here.
+     */
+    if (!ais_needed(flic)) {
+        return -ENOSYS;
+    }
+
+    return ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr) ? -errno : 0;
+}
+
+static const VMStateDescription kvm_s390_flic_ais_tmp = {
+    .name = "s390-flic-ais-tmp",
+    .pre_save = kvm_flic_ais_pre_save,
+    .post_load = kvm_flic_ais_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(simm, KVMS390FLICStateMigTmp),
+        VMSTATE_UINT8(nimm, KVMS390FLICStateMigTmp),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription kvm_s390_flic_vmstate_ais = {
+    .name = "s390-flic/ais",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = ais_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_WITH_TMP(KVMS390FLICState, KVMS390FLICStateMigTmp,
+                         kvm_s390_flic_ais_tmp),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription kvm_s390_flic_vmstate = {
+    /* should have been like kvm-s390-flic,
+     * can't change without breaking compat */
     .name = "s390-flic",
     .version_id = FLIC_SAVEVM_VERSION,
     .minimum_version_id = FLIC_SAVEVM_VERSION,
@@ -389,6 +505,10 @@ static const VMStateDescription kvm_s390_flic_vmstate = {
             .flags = VMS_SINGLE,
         },
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * []) {
+        &kvm_s390_flic_vmstate_ais,
+        NULL
     }
 };
 
@@ -436,7 +556,6 @@ static void kvm_s390_flic_realize(DeviceState *dev, Error **errp)
     test_attr.group = KVM_DEV_FLIC_CLEAR_IO_IRQ;
     flic_state->clear_io_supported = !ioctl(flic_state->fd,
                                             KVM_HAS_DEVICE_ATTR, test_attr);
-
     return;
 fail:
     error_propagate(errp, errp_local);
@@ -445,10 +564,12 @@ fail:
 static void kvm_s390_flic_reset(DeviceState *dev)
 {
     KVMS390FLICState *flic = KVM_S390_FLIC(dev);
+    S390FLICState *fs = S390_FLIC_COMMON(dev);
     struct kvm_device_attr attr = {
         .group = KVM_DEV_FLIC_CLEAR_IRQS,
     };
     int rc = 0;
+    uint8_t isc;
 
     if (flic->fd == -1) {
         return;
@@ -456,6 +577,16 @@ static void kvm_s390_flic_reset(DeviceState *dev)
 
     flic_disable_wait_pfault(flic);
 
+    if (fs->ais_supported) {
+        for (isc = 0; isc <= MAX_ISC; isc++) {
+            rc = kvm_s390_modify_ais_mode(fs, isc, SIC_IRQ_MODE_ALL);
+            if (rc) {
+                error_report("Failed to reset ais mode for isc %d: %s",
+                             isc, strerror(-rc));
+            }
+        }
+    }
+
     rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
     if (rc) {
         trace_flic_reset_failed(errno);
@@ -478,6 +609,8 @@ static void kvm_s390_flic_class_init(ObjectClass *oc, void *data)
     fsc->add_adapter_routes = kvm_s390_add_adapter_routes;
     fsc->release_adapter_routes = kvm_s390_release_adapter_routes;
     fsc->clear_io_irq = kvm_s390_clear_io_flic;
+    fsc->modify_ais_mode = kvm_s390_modify_ais_mode;
+    fsc->inject_airq = kvm_s390_inject_airq;
 }
 
 static const TypeInfo kvm_s390_flic_info = {

hw/intc/trace-events

@@ -73,6 +73,10 @@ flic_create_device(int err) "flic: create device failed %d"
 flic_no_device_api(int err) "flic: no Device Contral API support %d"
 flic_reset_failed(int err) "flic: reset failed %d"
 
+# hw/intc/s390_flic.c
+qemu_s390_airq_suppressed(uint8_t type, uint8_t isc) "flic: adapter I/O interrupt suppressed (type %x isc %x)"
+qemu_s390_suppress_airq(uint8_t isc, const char *from, const char *to) "flic: for isc %x, suppress airq by modifying ais mode from %s to %s"
+
 # hw/intc/aspeed_vic.c
 aspeed_vic_set_irq(int irq, int level) "Enabling IRQ %d: %d"
 aspeed_vic_update_fiq(int flags) "Raising FIQ: %d"

hw/s390x/Makefile.objs

@@ -13,5 +13,7 @@ obj-y += css-bridge.o
 obj-y += ccw-device.o
 obj-y += s390-pci-bus.o s390-pci-inst.o
 obj-y += s390-skeys.o
+obj-y += s390-stattrib.o
 obj-$(CONFIG_KVM) += s390-skeys-kvm.o
+obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
 obj-y += s390-ccw.o

hw/s390x/css-bridge.c

@@ -110,7 +110,7 @@ VirtualCssBus *virtual_css_bus_init(void)
     qbus_set_hotplug_handler(bus, dev, &error_abort);
 
     css_register_io_adapters(CSS_IO_ADAPTER_VIRTIO, true, false,
-                             &error_abort);
+                             0, &error_abort);
 
     return cbus;
  }

hw/s390x/css.c

@@ -29,12 +29,45 @@ typedef struct CrwContainer {
     QTAILQ_ENTRY(CrwContainer) sibling;
 } CrwContainer;
 
+static const VMStateDescription vmstate_crw = {
+    .name = "s390_crw",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(flags, CRW),
+        VMSTATE_UINT16(rsid, CRW),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static const VMStateDescription vmstate_crw_container = {
+    .name = "s390_crw_container",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(crw, CrwContainer, 0, vmstate_crw, CRW),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 typedef struct ChpInfo {
     uint8_t in_use;
     uint8_t type;
     uint8_t is_virtual;
 } ChpInfo;
 
+static const VMStateDescription vmstate_chp_info = {
+    .name = "s390_chp_info",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(in_use, ChpInfo),
+        VMSTATE_UINT8(type, ChpInfo),
+        VMSTATE_UINT8(is_virtual, ChpInfo),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 typedef struct SubchSet {
     SubchDev *sch[MAX_SCHID + 1];
     unsigned long schids_used[BITS_TO_LONGS(MAX_SCHID + 1)];
@@ -132,6 +165,36 @@ static const VMStateDescription vmstate_sense_id = {
     }
 };
 
+static const VMStateDescription vmstate_orb = {
+    .name = "s390_orb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(intparm, ORB),
+        VMSTATE_UINT16(ctrl0, ORB),
+        VMSTATE_UINT8(lpm, ORB),
+        VMSTATE_UINT8(ctrl1, ORB),
+        VMSTATE_UINT32(cpa, ORB),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool vmstate_schdev_orb_needed(void *opaque)
+{
+    return css_migration_enabled();
+}
+
+static const VMStateDescription vmstate_schdev_orb = {
+    .name = "s390_subch_dev/orb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vmstate_schdev_orb_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(orb, SubchDev, 1, vmstate_orb, ORB),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static int subch_dev_post_load(void *opaque, int version_id);
 static void subch_dev_pre_save(void *opaque);
 
@@ -160,6 +223,10 @@ const VMStateDescription vmstate_subch_dev = {
         VMSTATE_BOOL(ccw_fmt_1, SubchDev),
         VMSTATE_UINT8(ccw_no_data_cnt, SubchDev),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_schdev_orb,
+        NULL
     }
 };
 
@@ -221,10 +288,24 @@ typedef struct CssImage {
     ChpInfo chpids[MAX_CHPID + 1];
 } CssImage;
 
+static const VMStateDescription vmstate_css_img = {
+    .name = "s390_css_img",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        /* Subchannel sets have no relevant state. */
+        VMSTATE_STRUCT_ARRAY(chpids, CssImage, MAX_CHPID + 1, 0,
+                             vmstate_chp_info, ChpInfo),
+        VMSTATE_END_OF_LIST()
+    }
+
+};
+
 typedef struct IoAdapter {
     uint32_t id;
     uint8_t type;
     uint8_t isc;
+    uint8_t flags;
 } IoAdapter;
 
 typedef struct ChannelSubSys {
@@ -238,10 +319,34 @@ typedef struct ChannelSubSys {
     uint64_t chnmon_area;
     CssImage *css[MAX_CSSID + 1];
     uint8_t default_cssid;
+    /* don't migrate, see css_register_io_adapters */
     IoAdapter *io_adapters[CSS_IO_ADAPTER_TYPE_NUMS][MAX_ISC + 1];
+    /* don't migrate, see get_indicator and IndAddrPtrTmp */
     QTAILQ_HEAD(, IndAddr) indicator_addresses;
 } ChannelSubSys;
 
+static const VMStateDescription vmstate_css = {
+    .name = "s390_css",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_QTAILQ_V(pending_crws, ChannelSubSys, 1, vmstate_crw_container,
+                         CrwContainer, sibling),
+        VMSTATE_BOOL(sei_pending, ChannelSubSys),
+        VMSTATE_BOOL(do_crw_mchk, ChannelSubSys),
+        VMSTATE_BOOL(crws_lost, ChannelSubSys),
+        /* These were kind of migrated by virtio */
+        VMSTATE_UINT8(max_cssid, ChannelSubSys),
+        VMSTATE_UINT8(max_ssid, ChannelSubSys),
+        VMSTATE_BOOL(chnmon_active, ChannelSubSys),
+        VMSTATE_UINT64(chnmon_area, ChannelSubSys),
+        VMSTATE_ARRAY_OF_POINTER_TO_STRUCT(css, ChannelSubSys, MAX_CSSID + 1,
+                0, vmstate_css_img, CssImage),
+        VMSTATE_UINT8(default_cssid, ChannelSubSys),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static ChannelSubSys channel_subsys = {
     .pending_crws = QTAILQ_HEAD_INITIALIZER(channel_subsys.pending_crws),
     .do_crw_mchk = true,
@@ -281,6 +386,10 @@ static int subch_dev_post_load(void *opaque, int version_id)
         css_subch_assign(s->cssid, s->ssid, s->schid, s->devno, s);
     }
 
+    if (css_migration_enabled()) {
+        /* No compat voodoo to do ;) */
+        return 0;
+    }
     /*
      * Hack alert. If we don't migrate the channel subsystem status
      * we still need to find out if the guest enabled mss/mcss-e.
@@ -299,6 +408,11 @@ static int subch_dev_post_load(void *opaque, int version_id)
     return 0;
 }
 
+void css_register_vmstate(void)
+{
+    vmstate_register(NULL, 0, &vmstate_css, &channel_subsys);
+}
+
 IndAddr *get_indicator(hwaddr ind_addr, int len)
 {
     IndAddr *indicator;
@@ -392,10 +506,12 @@ uint32_t css_get_adapter_id(CssIoAdapterType type, uint8_t isc)
  *
  * @swap: an indication if byte swap is needed.
  * @maskable: an indication if the adapter is subject to the mask operation.
+ * @flags: further characteristics of the adapter.
+ *         e.g. suppressible, an indication if the adapter is subject to AIS.
  * @errp: location to store error information.
  */
 void css_register_io_adapters(CssIoAdapterType type, bool swap, bool maskable,
-                              Error **errp)
+                              uint8_t flags, Error **errp)
 {
     uint32_t id;
     int ret, isc;
@@ -413,12 +529,13 @@ void css_register_io_adapters(CssIoAdapterType type, bool swap, bool maskable,
 
     for (isc = 0; isc <= MAX_ISC; isc++) {
         id = (type << 3) | isc;
-        ret = fsc->register_io_adapter(fs, id, isc, swap, maskable);
+        ret = fsc->register_io_adapter(fs, id, isc, swap, maskable, flags);
         if (ret == 0) {
             adapter = g_new0(IoAdapter, 1);
             adapter->id = id;
             adapter->isc = isc;
             adapter->type = type;
+            adapter->flags = flags;
             channel_subsys.io_adapters[type][isc] = adapter;
         } else {
             error_setg_errno(errp, -ret, "Unexpected error %d when "
@@ -517,12 +634,52 @@ void css_conditional_io_interrupt(SubchDev *sch)
     }
 }
 
-void css_adapter_interrupt(uint8_t isc)
+int css_do_sic(CPUS390XState *env, uint8_t isc, uint16_t mode)
 {
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+    int r;
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        r = -PGM_PRIVILEGED;
+        goto out;
+    }
+
+    trace_css_do_sic(mode, isc);
+    switch (mode) {
+    case SIC_IRQ_MODE_ALL:
+    case SIC_IRQ_MODE_SINGLE:
+        break;
+    default:
+        r = -PGM_OPERAND;
+        goto out;
+    }
+
+    r = fsc->modify_ais_mode(fs, isc, mode) ? -PGM_OPERATION : 0;
+out:
+    return r;
+}
+
+void css_adapter_interrupt(CssIoAdapterType type, uint8_t isc)
+{
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
     uint32_t io_int_word = (isc << 27) | IO_INT_WORD_AI;
+    IoAdapter *adapter = channel_subsys.io_adapters[type][isc];
+
+    if (!adapter) {
+        return;
+    }
 
     trace_css_adapter_interrupt(isc);
-    s390_io_interrupt(0, 0, 0, io_int_word);
+    if (fs->ais_supported) {
+        if (fsc->inject_airq(fs, type, isc, adapter->flags)) {
+            error_report("Failed to inject airq with AIS supported");
+            exit(1);
+        }
+    } else {
+        s390_io_interrupt(0, 0, 0, io_int_word);
+    }
 }
 
 static void sch_handle_clear_func(SubchDev *sch)
@@ -752,7 +909,7 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr,
     return ret;
 }
 
-static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
+static void sch_handle_start_func_virtual(SubchDev *sch)
 {
 
     PMCW *p = &sch->curr_status.pmcw;
@@ -766,10 +923,10 @@ static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
 
     if (!(s->ctrl & SCSW_ACTL_SUSP)) {
         /* Start Function triggered via ssch, i.e. we have an ORB */
+        ORB *orb = &sch->orb;
         s->cstat = 0;
         s->dstat = 0;
         /* Look at the orb and try to execute the channel program. */
-        assert(orb != NULL); /* resume does not pass an orb */
         p->intparm = orb->intparm;
         if (!(orb->lpm & path)) {
             /* Generate a deferred cc 3 condition. */
@@ -783,8 +940,7 @@ static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
         sch->ccw_no_data_cnt = 0;
         suspend_allowed = !!(orb->ctrl0 & ORB_CTRL0_MASK_SPND);
     } else {
-        /* Start Function resumed via rsch, i.e. we don't have an
+        /* Start Function resumed via rsch */
-         * ORB */
         s->ctrl &= ~(SCSW_ACTL_SUSP | SCSW_ACTL_RESUME_PEND);
         /* The channel program had been suspended before. */
         suspend_allowed = true;
@@ -854,13 +1010,14 @@ static void sch_handle_start_func_virtual(SubchDev *sch, ORB *orb)
 
 }
 
-static int sch_handle_start_func_passthrough(SubchDev *sch, ORB *orb)
+static int sch_handle_start_func_passthrough(SubchDev *sch)
 {
 
     PMCW *p = &sch->curr_status.pmcw;
     SCSW *s = &sch->curr_status.scsw;
     int ret;
 
+    ORB *orb = &sch->orb;
     if (!(s->ctrl & SCSW_ACTL_SUSP)) {
         assert(orb != NULL);
         p->intparm = orb->intparm;
@@ -905,7 +1062,7 @@ static int sch_handle_start_func_passthrough(SubchDev *sch, ORB *orb)
  * read/writes) asynchronous later on if we start supporting more than
  * our current very simple devices.
  */
-int do_subchannel_work_virtual(SubchDev *sch, ORB *orb)
+int do_subchannel_work_virtual(SubchDev *sch)
 {
 
     SCSW *s = &sch->curr_status.scsw;
@@ -916,7 +1073,7 @@ int do_subchannel_work_virtual(SubchDev *sch, ORB *orb)
         sch_handle_halt_func(sch);
     } else if (s->ctrl & SCSW_FCTL_START_FUNC) {
         /* Triggered by both ssch and rsch. */
-        sch_handle_start_func_virtual(sch, orb);
+        sch_handle_start_func_virtual(sch);
     } else {
         /* Cannot happen. */
         return 0;
@@ -925,7 +1082,7 @@ int do_subchannel_work_virtual(SubchDev *sch, ORB *orb)
     return 0;
 }
 
-int do_subchannel_work_passthrough(SubchDev *sch, ORB *orb)
+int do_subchannel_work_passthrough(SubchDev *sch)
 {
     int ret;
     SCSW *s = &sch->curr_status.scsw;
@@ -939,7 +1096,7 @@ int do_subchannel_work_passthrough(SubchDev *sch, ORB *orb)
         sch_handle_halt_func(sch);
         ret = 0;
     } else if (s->ctrl & SCSW_FCTL_START_FUNC) {
-        ret = sch_handle_start_func_passthrough(sch, orb);
+        ret = sch_handle_start_func_passthrough(sch);
     } else {
         /* Cannot happen. */
         return -ENODEV;
@@ -948,10 +1105,10 @@ int do_subchannel_work_passthrough(SubchDev *sch, ORB *orb)
     return ret;
 }
 
-static int do_subchannel_work(SubchDev *sch, ORB *orb)
+static int do_subchannel_work(SubchDev *sch)
 {
     if (sch->do_subchannel_work) {
-        return sch->do_subchannel_work(sch, orb);
+        return sch->do_subchannel_work(sch);
     } else {
         return -EINVAL;
     }
@@ -1158,7 +1315,7 @@ int css_do_csch(SubchDev *sch)
     s->ctrl &= ~(SCSW_CTRL_MASK_FCTL | SCSW_CTRL_MASK_ACTL);
     s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_ACTL_CLEAR_PEND;
 
-    do_subchannel_work(sch, NULL);
+    do_subchannel_work(sch);
     ret = 0;
 
 out:
@@ -1199,7 +1356,7 @@ int css_do_hsch(SubchDev *sch)
     }
     s->ctrl |= SCSW_ACTL_HALT_PEND;
 
-    do_subchannel_work(sch, NULL);
+    do_subchannel_work(sch);
     ret = 0;
 
 out:
@@ -1268,12 +1425,13 @@ int css_do_ssch(SubchDev *sch, ORB *orb)
     if (channel_subsys.chnmon_active) {
         css_update_chnmon(sch);
     }
+    sch->orb = *orb;
     sch->channel_prog = orb->cpa;
     /* Trigger the start function. */
     s->ctrl |= (SCSW_FCTL_START_FUNC | SCSW_ACTL_START_PEND);
     s->flags &= ~SCSW_FLAGS_MASK_PNO;
 
-    ret = do_subchannel_work(sch, orb);
+    ret = do_subchannel_work(sch);
 
 out:
     return ret;
@@ -1552,7 +1710,7 @@ int css_do_rsch(SubchDev *sch)
     }
 
     s->ctrl |= SCSW_ACTL_RESUME_PEND;
-    do_subchannel_work(sch, NULL);
+    do_subchannel_work(sch);
     ret = 0;
 
 out:

hw/s390x/s390-pci-bus.c

@@ -500,7 +500,7 @@ static void s390_msi_ctrl_write(void *opaque, hwaddr addr, uint64_t data,
                    0x80 >> ((ind_bit + vec) % 8));
     if (!set_ind_atomic(pbdev->routes.adapter.summary_addr + sum_bit / 8,
                                        0x80 >> (sum_bit % 8))) {
-        css_adapter_interrupt(pbdev->isc);
+        css_adapter_interrupt(CSS_IO_ADAPTER_PCI, pbdev->isc);
     }
 }
 
@@ -579,7 +579,8 @@ static int s390_pcihost_init(SysBusDevice *dev)
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
 
-    css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false, &error_abort);
+    css_register_io_adapters(CSS_IO_ADAPTER_PCI, true, false,
+                             S390_ADAPTER_SUPPRESSIBLE, &error_abort);
 
     return 0;
 }

hw/s390x/s390-stattrib-kvm.c

@@ -0,0 +1,190 @@
+/*
+ * s390 storage attributes device -- KVM object
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Claudio Imbrenda <imbrenda@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "migration/qemu-file.h"
+#include "hw/s390x/storage-attributes.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "exec/ram_addr.h"
+#include "cpu.h"
+
+Object *kvm_s390_stattrib_create(void)
+{
+    if (kvm_enabled() &&
+                kvm_check_extension(kvm_state, KVM_CAP_S390_CMMA_MIGRATION)) {
+        return object_new(TYPE_KVM_S390_STATTRIB);
+    }
+    return NULL;
+}
+
+static void kvm_s390_stattrib_instance_init(Object *obj)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(obj);
+
+    sas->still_dirty = 0;
+}
+
+static int kvm_s390_stattrib_read_helper(S390StAttribState *sa,
+                                         uint64_t *start_gfn,
+                                         uint32_t count,
+                                         uint8_t *values,
+                                         uint32_t flags)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    int r;
+    struct kvm_s390_cmma_log clog = {
+        .values = (uint64_t)values,
+        .start_gfn = *start_gfn,
+        .count = count,
+        .flags = flags,
+    };
+
+    r = kvm_vm_ioctl(kvm_state, KVM_S390_GET_CMMA_BITS, &clog);
+    if (r < 0) {
+        error_report("KVM_S390_GET_CMMA_BITS failed: %s", strerror(-r));
+        return r;
+    }
+
+    *start_gfn = clog.start_gfn;
+    sas->still_dirty = clog.remaining;
+    return clog.count;
+}
+
+static int kvm_s390_stattrib_get_stattr(S390StAttribState *sa,
+                                        uint64_t *start_gfn,
+                                        uint32_t count,
+                                        uint8_t *values)
+{
+    return kvm_s390_stattrib_read_helper(sa, start_gfn, count, values, 0);
+}
+
+static int kvm_s390_stattrib_peek_stattr(S390StAttribState *sa,
+                                         uint64_t start_gfn,
+                                         uint32_t count,
+                                         uint8_t *values)
+{
+    return kvm_s390_stattrib_read_helper(sa, &start_gfn, count, values,
+                                         KVM_S390_CMMA_PEEK);
+}
+
+static int kvm_s390_stattrib_set_stattr(S390StAttribState *sa,
+                                        uint64_t start_gfn,
+                                        uint32_t count,
+                                        uint8_t *values)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    MachineState *machine = MACHINE(qdev_get_machine());
+    unsigned long max = machine->maxram_size / TARGET_PAGE_SIZE;
+
+    if (start_gfn + count > max) {
+        error_report("Out of memory bounds when setting storage attributes");
+        return -1;
+    }
+    if (!sas->incoming_buffer) {
+        sas->incoming_buffer = g_malloc0(max);
+    }
+
+    memcpy(sas->incoming_buffer + start_gfn, values, count);
+
+    return 0;
+}
+
+static void kvm_s390_stattrib_synchronize(S390StAttribState *sa)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    MachineState *machine = MACHINE(qdev_get_machine());
+    unsigned long max = machine->maxram_size / TARGET_PAGE_SIZE;
+    unsigned long cx, len = 1 << 19;
+    int r;
+    struct kvm_s390_cmma_log clog = {
+        .flags = 0,
+        .mask = ~0ULL,
+    };
+
+    if (sas->incoming_buffer) {
+        for (cx = 0; cx + len <= max; cx += len) {
+            clog.start_gfn = cx;
+            clog.count = len;
+            clog.values = (uint64_t)(sas->incoming_buffer + cx * len);
+            r = kvm_vm_ioctl(kvm_state, KVM_S390_SET_CMMA_BITS, &clog);
+            if (r) {
+                error_report("KVM_S390_SET_CMMA_BITS failed: %s", strerror(-r));
+                return;
+            }
+        }
+        if (cx < max) {
+            clog.start_gfn = cx;
+            clog.count = max - cx;
+            clog.values = (uint64_t)(sas->incoming_buffer + cx * len);
+            r = kvm_vm_ioctl(kvm_state, KVM_S390_SET_CMMA_BITS, &clog);
+            if (r) {
+                error_report("KVM_S390_SET_CMMA_BITS failed: %s", strerror(-r));
+            }
+        }
+        g_free(sas->incoming_buffer);
+        sas->incoming_buffer = NULL;
+    }
+}
+
+static int kvm_s390_stattrib_set_migrationmode(S390StAttribState *sa, bool val)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MIGRATION,
+        .attr = val,
+        .addr = 0,
+    };
+    return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+static long long kvm_s390_stattrib_get_dirtycount(S390StAttribState *sa)
+{
+    KVMS390StAttribState *sas = KVM_S390_STATTRIB(sa);
+    uint8_t val[8];
+
+    kvm_s390_stattrib_peek_stattr(sa, 0, 1, val);
+    return sas->still_dirty;
+}
+
+static int kvm_s390_stattrib_get_active(S390StAttribState *sa)
+{
+    return kvm_s390_cmma_active() && sa->migration_enabled;
+}
+
+static void kvm_s390_stattrib_class_init(ObjectClass *oc, void *data)
+{
+    S390StAttribClass *sac = S390_STATTRIB_CLASS(oc);
+
+    sac->get_stattr = kvm_s390_stattrib_get_stattr;
+    sac->peek_stattr = kvm_s390_stattrib_peek_stattr;
+    sac->set_stattr = kvm_s390_stattrib_set_stattr;
+    sac->set_migrationmode = kvm_s390_stattrib_set_migrationmode;
+    sac->get_dirtycount = kvm_s390_stattrib_get_dirtycount;
+    sac->synchronize = kvm_s390_stattrib_synchronize;
+    sac->get_active = kvm_s390_stattrib_get_active;
+}
+
+static const TypeInfo kvm_s390_stattrib_info = {
+    .name          = TYPE_KVM_S390_STATTRIB,
+    .parent        = TYPE_S390_STATTRIB,
+    .instance_init = kvm_s390_stattrib_instance_init,
+    .instance_size = sizeof(KVMS390StAttribState),
+    .class_init    = kvm_s390_stattrib_class_init,
+    .class_size    = sizeof(S390StAttribClass),
+};
+
+static void kvm_s390_stattrib_register_types(void)
+{
+    type_register_static(&kvm_s390_stattrib_info);
+}
+
+type_init(kvm_s390_stattrib_register_types)

hw/s390x/s390-stattrib.c

@@ -0,0 +1,404 @@
+/*
+ * s390 storage attributes device
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Claudio Imbrenda <imbrenda@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "qmp-commands.h"
+#include "migration/qemu-file.h"
+#include "migration/register.h"
+#include "hw/s390x/storage-attributes.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "exec/ram_addr.h"
+#include "qapi/error.h"
+
+#define CMMA_BLOCK_SIZE  (1 << 10)
+
+#define STATTR_FLAG_EOS     0x01ULL
+#define STATTR_FLAG_MORE    0x02ULL
+#define STATTR_FLAG_ERROR   0x04ULL
+#define STATTR_FLAG_DONE    0x08ULL
+
+static S390StAttribState *s390_get_stattrib_device(void)
+{
+    S390StAttribState *sas;
+
+    sas = S390_STATTRIB(object_resolve_path_type("", TYPE_S390_STATTRIB, NULL));
+    assert(sas);
+    return sas;
+}
+
+void s390_stattrib_init(void)
+{
+    Object *obj;
+
+    obj = kvm_s390_stattrib_create();
+    if (!obj) {
+        obj = object_new(TYPE_QEMU_S390_STATTRIB);
+    }
+
+    object_property_add_child(qdev_get_machine(), TYPE_S390_STATTRIB,
+                              obj, NULL);
+    object_unref(obj);
+
+    qdev_init_nofail(DEVICE(obj));
+}
+
+/* Console commands: */
+
+void hmp_migrationmode(Monitor *mon, const QDict *qdict)
+{
+    S390StAttribState *sas = s390_get_stattrib_device();
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint64_t what = qdict_get_int(qdict, "mode");
+    int r;
+
+    r = sac->set_migrationmode(sas, what);
+    if (r < 0) {
+        monitor_printf(mon, "Error: %s", strerror(-r));
+    }
+}
+
+void hmp_info_cmma(Monitor *mon, const QDict *qdict)
+{
+    S390StAttribState *sas = s390_get_stattrib_device();
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint64_t buflen = qdict_get_try_int(qdict, "count", 8);
+    uint8_t *vals;
+    int cx, len;
+
+    vals = g_try_malloc(buflen);
+    if (!vals) {
+        monitor_printf(mon, "Error: %s\n", strerror(errno));
+        return;
+    }
+
+    len = sac->peek_stattr(sas, addr / TARGET_PAGE_SIZE, buflen, vals);
+    if (len < 0) {
+        monitor_printf(mon, "Error: %s", strerror(-len));
+        goto out;
+    }
+
+    monitor_printf(mon, "  CMMA attributes, "
+                   "pages %" PRIu64 "+%d (0x%" PRIx64 "):\n",
+                   addr / TARGET_PAGE_SIZE, len, addr & ~TARGET_PAGE_MASK);
+    for (cx = 0; cx < len; cx++) {
+        if (cx % 8 == 7) {
+            monitor_printf(mon, "%02x\n", vals[cx]);
+        } else {
+            monitor_printf(mon, "%02x", vals[cx]);
+        }
+    }
+    monitor_printf(mon, "\n");
+
+out:
+    g_free(vals);
+}
+
+/* Migration support: */
+
+static int cmma_load(QEMUFile *f, void *opaque, int version_id)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint64_t count, cur_gfn;
+    int flags, ret = 0;
+    ram_addr_t addr;
+    uint8_t *buf;
+
+    while (!ret) {
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        switch (flags) {
+        case STATTR_FLAG_MORE: {
+            cur_gfn = addr / TARGET_PAGE_SIZE;
+            count = qemu_get_be64(f);
+            buf = g_try_malloc(count);
+            if (!buf) {
+                error_report("cmma_load could not allocate memory");
+                ret = -ENOMEM;
+                break;
+            }
+
+            qemu_get_buffer(f, buf, count);
+            ret = sac->set_stattr(sas, cur_gfn, count, buf);
+            if (ret < 0) {
+                error_report("Error %d while setting storage attributes", ret);
+            }
+            g_free(buf);
+            break;
+        }
+        case STATTR_FLAG_ERROR: {
+            error_report("Storage attributes data is incomplete");
+            ret = -EINVAL;
+            break;
+        }
+        case STATTR_FLAG_DONE:
+            /* This is after the last pre-copied value has been sent, nothing
+             * more will be sent after this. Pre-copy has finished, and we
+             * are done flushing all the remaining values. Now the target
+             * system is about to take over. We synchronize the buffer to
+             * apply the actual correct values where needed.
+             */
+             sac->synchronize(sas);
+            break;
+        case STATTR_FLAG_EOS:
+            /* Normal exit */
+            return 0;
+        default:
+            error_report("Unexpected storage attribute flag data: %#x", flags);
+            ret = -EINVAL;
+        }
+    }
+
+    return ret;
+}
+
+static int cmma_save_setup(QEMUFile *f, void *opaque)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    int res;
+    /*
+     * Signal that we want to start a migration, thus needing PGSTE dirty
+     * tracking.
+     */
+    res = sac->set_migrationmode(sas, 1);
+    if (res) {
+        return res;
+    }
+    qemu_put_be64(f, STATTR_FLAG_EOS);
+    return 0;
+}
+
+static void cmma_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                             uint64_t *non_postcopiable_pending,
+                             uint64_t *postcopiable_pending)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    long long res = sac->get_dirtycount(sas);
+
+    if (res >= 0) {
+        *non_postcopiable_pending += res;
+    }
+}
+
+static int cmma_save(QEMUFile *f, void *opaque, int final)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    uint8_t *buf;
+    int r, cx, reallen = 0, ret = 0;
+    uint32_t buflen = 1 << 19;   /* 512kB cover 2GB of guest memory */
+    uint64_t start_gfn = sas->migration_cur_gfn;
+
+    buf = g_try_malloc(buflen);
+    if (!buf) {
+        error_report("Could not allocate memory to save storage attributes");
+        return -ENOMEM;
+    }
+
+    while (final ? 1 : qemu_file_rate_limit(f) == 0) {
+        reallen = sac->get_stattr(sas, &start_gfn, buflen, buf);
+        if (reallen < 0) {
+            g_free(buf);
+            return reallen;
+        }
+
+        ret = 1;
+        if (!reallen) {
+            break;
+        }
+        qemu_put_be64(f, (start_gfn << TARGET_PAGE_BITS) | STATTR_FLAG_MORE);
+        qemu_put_be64(f, reallen);
+        for (cx = 0; cx < reallen; cx++) {
+            qemu_put_byte(f, buf[cx]);
+        }
+        if (!sac->get_dirtycount(sas)) {
+            break;
+        }
+    }
+
+    sas->migration_cur_gfn = start_gfn + reallen;
+    g_free(buf);
+    if (final) {
+        qemu_put_be64(f, STATTR_FLAG_DONE);
+    }
+    qemu_put_be64(f, STATTR_FLAG_EOS);
+
+    r = qemu_file_get_error(f);
+    if (r < 0) {
+        return r;
+    }
+
+    return ret;
+}
+
+static int cmma_save_iterate(QEMUFile *f, void *opaque)
+{
+    return cmma_save(f, opaque, 0);
+}
+
+static int cmma_save_complete(QEMUFile *f, void *opaque)
+{
+    return cmma_save(f, opaque, 1);
+}
+
+static void cmma_save_cleanup(void *opaque)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    sac->set_migrationmode(sas, 0);
+}
+
+static bool cmma_active(void *opaque)
+{
+    S390StAttribState *sas = S390_STATTRIB(opaque);
+    S390StAttribClass *sac = S390_STATTRIB_GET_CLASS(sas);
+    return sac->get_active(sas);
+}
+
+/* QEMU object: */
+
+static void qemu_s390_stattrib_instance_init(Object *obj)
+{
+}
+
+static int qemu_s390_peek_stattr_stub(S390StAttribState *sa, uint64_t start_gfn,
+                                     uint32_t count, uint8_t *values)
+{
+    return 0;
+}
+static void qemu_s390_synchronize_stub(S390StAttribState *sa)
+{
+}
+static int qemu_s390_get_stattr_stub(S390StAttribState *sa, uint64_t *start_gfn,
+                                     uint32_t count, uint8_t *values)
+{
+    return 0;
+}
+static long long qemu_s390_get_dirtycount_stub(S390StAttribState *sa)
+{
+    return 0;
+}
+static int qemu_s390_set_migrationmode_stub(S390StAttribState *sa, bool value)
+{
+    return 0;
+}
+
+static int qemu_s390_get_active(S390StAttribState *sa)
+{
+    return sa->migration_enabled;
+}
+
+static void qemu_s390_stattrib_class_init(ObjectClass *oc, void *data)
+{
+    S390StAttribClass *sa_cl = S390_STATTRIB_CLASS(oc);
+
+    sa_cl->synchronize = qemu_s390_synchronize_stub;
+    sa_cl->get_stattr = qemu_s390_get_stattr_stub;
+    sa_cl->set_stattr = qemu_s390_peek_stattr_stub;
+    sa_cl->peek_stattr = qemu_s390_peek_stattr_stub;
+    sa_cl->set_migrationmode = qemu_s390_set_migrationmode_stub;
+    sa_cl->get_dirtycount = qemu_s390_get_dirtycount_stub;
+    sa_cl->get_active = qemu_s390_get_active;
+}
+
+static const TypeInfo qemu_s390_stattrib_info = {
+    .name          = TYPE_QEMU_S390_STATTRIB,
+    .parent        = TYPE_S390_STATTRIB,
+    .instance_init = qemu_s390_stattrib_instance_init,
+    .instance_size = sizeof(QEMUS390StAttribState),
+    .class_init    = qemu_s390_stattrib_class_init,
+    .class_size    = sizeof(S390StAttribClass),
+};
+
+/* Generic abstract object: */
+
+static void s390_stattrib_realize(DeviceState *dev, Error **errp)
+{
+    bool ambiguous = false;
+
+    object_resolve_path_type("", TYPE_S390_STATTRIB, &ambiguous);
+    if (ambiguous) {
+        error_setg(errp, "storage_attributes device already exists");
+    }
+}
+
+static void s390_stattrib_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->realize = s390_stattrib_realize;
+}
+
+static inline bool s390_stattrib_get_migration_enabled(Object *obj, Error **e)
+{
+    S390StAttribState *s = S390_STATTRIB(obj);
+
+    return s->migration_enabled;
+}
+
+static inline void s390_stattrib_set_migration_enabled(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390StAttribState *s = S390_STATTRIB(obj);
+
+    s->migration_enabled = value;
+}
+
+static void s390_stattrib_instance_init(Object *obj)
+{
+    S390StAttribState *sas = S390_STATTRIB(obj);
+    SaveVMHandlers *ops;
+
+    /* ops will always be freed by qemu when unregistering */
+    ops = g_new0(SaveVMHandlers, 1);
+
+    ops->save_setup = cmma_save_setup;
+    ops->save_live_iterate = cmma_save_iterate;
+    ops->save_live_complete_precopy = cmma_save_complete;
+    ops->save_live_pending = cmma_save_pending;
+    ops->save_cleanup = cmma_save_cleanup;
+    ops->load_state = cmma_load;
+    ops->is_active = cmma_active;
+    register_savevm_live(NULL, TYPE_S390_STATTRIB, 0, 0, ops, sas);
+
+    object_property_add_bool(obj, "migration-enabled",
+                             s390_stattrib_get_migration_enabled,
+                             s390_stattrib_set_migration_enabled, NULL);
+    object_property_set_bool(obj, true, "migration-enabled", NULL);
+    sas->migration_cur_gfn = 0;
+}
+
+static const TypeInfo s390_stattrib_info = {
+    .name          = TYPE_S390_STATTRIB,
+    .parent        = TYPE_DEVICE,
+    .instance_init = s390_stattrib_instance_init,
+    .instance_size = sizeof(S390StAttribState),
+    .class_init    = s390_stattrib_class_init,
+    .class_size    = sizeof(S390StAttribClass),
+    .abstract      = true,
+};
+
+static void s390_stattrib_register_types(void)
+{
+    type_register_static(&s390_stattrib_info);
+    type_register_static(&qemu_s390_stattrib_info);
+}
+
+type_init(s390_stattrib_register_types)

hw/s390x/s390-virtio-ccw.c

@@ -24,11 +24,13 @@
 #include "qemu/config-file.h"
 #include "s390-pci-bus.h"
 #include "hw/s390x/storage-keys.h"
+#include "hw/s390x/storage-attributes.h"
 #include "hw/compat.h"
 #include "ipl.h"
 #include "hw/s390x/s390-virtio-ccw.h"
 #include "hw/s390x/css-bridge.h"
 #include "migration/register.h"
+#include "cpu_models.h"
 
 static const char *const reset_dev_types[] = {
     TYPE_VIRTUAL_CSS_BRIDGE,
@@ -103,6 +105,8 @@ void s390_memory_init(ram_addr_t mem_size)
 
     /* Initialize storage key device */
     s390_skeys_init();
+    /* Initialize storage attributes device */
+    s390_stattrib_init();
 }
 
 static SaveVMHandlers savevm_gtod = {
@@ -119,6 +123,9 @@ static void ccw_init(MachineState *machine)
     s390_sclp_init();
     s390_memory_init(machine->ram_size);
 
+    /* init CPUs */
+    s390_init_cpus(machine);
+
     s390_flic_init();
 
     /* get a BUS */
@@ -135,9 +142,6 @@ static void ccw_init(MachineState *machine)
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* init CPUs */
-    s390_init_cpus(machine);
-
     if (kvm_enabled()) {
         kvm_s390_enable_css_support(s390_cpu_addr2state(0));
     }
@@ -206,6 +210,8 @@ static void ccw_machine_class_init(ObjectClass *oc, void *data)
 
     s390mc->ri_allowed = true;
     s390mc->cpu_model_allowed = true;
+    s390mc->css_migration_enabled = true;
+    s390mc->gs_allowed = true;
     mc->init = ccw_init;
     mc->reset = s390_machine_reset;
     mc->hot_add_cpu = s390_hot_add_cpu;
@@ -252,7 +258,38 @@ static inline void machine_set_dea_key_wrap(Object *obj, bool value,
     ms->dea_key_wrap = value;
 }
 
+static S390CcwMachineClass *current_mc;
+
+static S390CcwMachineClass *get_machine_class(void)
+{
+    if (unlikely(!current_mc)) {
+        /*
+        * No s390 ccw machine was instantiated, we are likely to
+        * be called for the 'none' machine. The properties will
+        * have their after-initialization values.
+        */
+        current_mc = S390_MACHINE_CLASS(
+                     object_class_by_name(TYPE_S390_CCW_MACHINE));
+    }
+    return current_mc;
+}
+
 bool ri_allowed(void)
+{
+    if (!kvm_enabled()) {
+        return false;
+    }
+    /* for "none" machine this results in true */
+    return get_machine_class()->ri_allowed;
+}
+
+bool cpu_model_allowed(void)
+{
+    /* for "none" machine this results in true */
+    return get_machine_class()->cpu_model_allowed;
+}
+
+bool gs_allowed(void)
 {
     if (kvm_enabled()) {
         MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
@@ -260,28 +297,12 @@ bool ri_allowed(void)
                                       TYPE_S390_CCW_MACHINE)) {
             S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
 
-            return s390mc->ri_allowed;
+            return s390mc->gs_allowed;
         }
-        /*
+        /* Make sure the "none" machine can have gs */
-         * Make sure the "none" machine can have ri, otherwise it won't * be
-         * unlocked in KVM and therefore the host CPU model might be wrong.
-         */
         return true;
     }
-    return 0;
+    return false;
-}
-
-bool cpu_model_allowed(void)
-{
-    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
-    if (object_class_dynamic_cast(OBJECT_CLASS(mc),
-                                  TYPE_S390_CCW_MACHINE)) {
-        S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
-
-        return s390mc->cpu_model_allowed;
-    }
-    /* allow CPU model qmp queries with the "none" machine */
-    return true;
 }
 
 static char *machine_get_loadparm(Object *obj, Error **errp)
@@ -376,6 +397,11 @@ static const TypeInfo ccw_machine_info = {
     },
 };
 
+bool css_migration_enabled(void)
+{
+    return get_machine_class()->css_migration_enabled;
+}
+
 #define DEFINE_CCW_MACHINE(suffix, verstr, latest)                            \
     static void ccw_machine_##suffix##_class_init(ObjectClass *oc,            \
                                                   void *data)                 \
@@ -391,6 +417,7 @@ static const TypeInfo ccw_machine_info = {
     static void ccw_machine_##suffix##_instance_init(Object *obj)             \
     {                                                                         \
         MachineState *machine = MACHINE(obj);                                 \
+        current_mc = S390_MACHINE_CLASS(MACHINE_GET_CLASS(machine));          \
         ccw_machine_##suffix##_instance_options(machine);                     \
     }                                                                         \
     static const TypeInfo ccw_machine_##suffix##_info = {                     \
@@ -406,7 +433,12 @@ static const TypeInfo ccw_machine_info = {
     type_init(ccw_machine_register_##suffix)
 
 #define CCW_COMPAT_2_9 \
-        HW_COMPAT_2_9
+        HW_COMPAT_2_9 \
+        {\
+            .driver   = TYPE_S390_STATTRIB,\
+            .property = "migration-enabled",\
+            .value    = "off",\
+        },
 
 #define CCW_COMPAT_2_8 \
         HW_COMPAT_2_8 \
@@ -476,6 +508,9 @@ static const TypeInfo ccw_machine_info = {
 
 static void ccw_machine_2_10_instance_options(MachineState *machine)
 {
+    if (css_migration_enabled()) {
+        css_register_vmstate();
+    }
 }
 
 static void ccw_machine_2_10_class_options(MachineClass *mc)
@@ -486,12 +521,21 @@ DEFINE_CCW_MACHINE(2_10, "2.10", true);
 static void ccw_machine_2_9_instance_options(MachineState *machine)
 {
     ccw_machine_2_10_instance_options(machine);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ESOP);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ZPCI);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ADAPTER_INT_SUPPRESSION);
+    s390_cpudef_featoff_greater(12, 1, S390_FEAT_ADAPTER_EVENT_NOTIFICATION);
 }
 
 static void ccw_machine_2_9_class_options(MachineClass *mc)
 {
+    S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
+
+    s390mc->gs_allowed = false;
     ccw_machine_2_10_class_options(mc);
     SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_9);
+    s390mc->css_migration_enabled = false;
 }
 DEFINE_CCW_MACHINE(2_9, "2.9", false);
 

hw/s390x/trace-events

@@ -8,6 +8,7 @@ css_new_image(uint8_t cssid, const char *default_cssid) "CSS: add css image %02x
 css_assign_subch(const char *do_assign, uint8_t cssid, uint8_t ssid, uint16_t schid, uint16_t devno) "CSS: %s %x.%x.%04x (devno %04x)"
 css_io_interrupt(int cssid, int ssid, int schid, uint32_t intparm, uint8_t isc, const char *conditional) "CSS: I/O interrupt on sch %x.%x.%04x (intparm %08x, isc %x) %s"
 css_adapter_interrupt(uint8_t isc) "CSS: adapter I/O interrupt (isc %x)"
+css_do_sic(uint16_t mode, uint8_t isc) "CSS: set interruption mode %x on isc %x"
 
 # hw/s390x/virtio-ccw.c
 virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code) "VIRTIO-CCW: %x.%x.%04x: interpret command %x"

hw/s390x/virtio-ccw.c

@@ -1070,7 +1070,7 @@ static void virtio_ccw_notify(DeviceState *d, uint16_t vector)
                                   0x80 >> ((ind_bit + vector) % 8));
             if (!virtio_set_ind_atomic(sch, dev->summary_indicator->addr,
                                        0x01)) {
-                css_adapter_interrupt(dev->thinint_isc);
+                css_adapter_interrupt(CSS_IO_ADAPTER_VIRTIO, dev->thinint_isc);
             }
         } else {
             indicators = address_space_ldq(&address_space_memory,

include/elf.h

@@ -1476,6 +1476,7 @@ typedef struct elf64_shdr {
 #define NT_TASKSTRUCT	4
 #define NT_AUXV		6
 #define NT_PRXFPREG     0x46e62b7f      /* copied from gdb5.1/include/elf/common.h */
+#define NT_S390_GS_CB   0x30b           /* s390 guarded storage registers */
 #define NT_S390_VXRS_HIGH 0x30a         /* s390 vector registers 16-31 */
 #define NT_S390_VXRS_LOW  0x309         /* s390 vector registers 0-15 (lower half) */
 #define NT_S390_PREFIX  0x305           /* s390 prefix register */

include/hw/s390x/css.h

@@ -12,6 +12,7 @@
 #ifndef CSS_H
 #define CSS_H
 
+#include "cpu.h"
 #include "hw/s390x/adapter.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/ioinst.h"
@@ -89,10 +90,11 @@ struct SubchDev {
     bool thinint_active;
     uint8_t ccw_no_data_cnt;
     uint16_t migrated_schid; /* used for missmatch detection */
+    ORB orb;
     /* transport-provided data: */
     int (*ccw_cb) (SubchDev *, CCW1);
     void (*disable_cb)(SubchDev *);
-    int (*do_subchannel_work) (SubchDev *, ORB *);
+    int (*do_subchannel_work) (SubchDev *);
     SenseId id;
     void *driver_data;
 };
@@ -154,10 +156,9 @@ void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
 void css_generate_chp_crws(uint8_t cssid, uint8_t chpid);
 void css_generate_css_crws(uint8_t cssid);
 void css_clear_sei_pending(void);
-void css_adapter_interrupt(uint8_t isc);
 int s390_ccw_cmd_request(ORB *orb, SCSW *scsw, void *data);
-int do_subchannel_work_virtual(SubchDev *sub, ORB *orb);
+int do_subchannel_work_virtual(SubchDev *sub);
-int do_subchannel_work_passthrough(SubchDev *sub, ORB *orb);
+int do_subchannel_work_passthrough(SubchDev *sub);
 
 typedef enum {
     CSS_IO_ADAPTER_VIRTIO = 0,
@@ -165,9 +166,17 @@ typedef enum {
     CSS_IO_ADAPTER_TYPE_NUMS,
 } CssIoAdapterType;
 
+void css_adapter_interrupt(CssIoAdapterType type, uint8_t isc);
+int css_do_sic(CPUS390XState *env, uint8_t isc, uint16_t mode);
 uint32_t css_get_adapter_id(CssIoAdapterType type, uint8_t isc);
 void css_register_io_adapters(CssIoAdapterType type, bool swap, bool maskable,
-                              Error **errp);
+                              uint8_t flags, Error **errp);
+
+#ifndef CONFIG_KVM
+#define S390_ADAPTER_SUPPRESSIBLE 0x01
+#else
+#define S390_ADAPTER_SUPPRESSIBLE KVM_S390_ADAPTER_SUPPRESSIBLE
+#endif
 
 #ifndef CONFIG_USER_ONLY
 SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
@@ -225,4 +234,8 @@ extern const PropertyInfo css_devid_ro_propinfo;
  */
 SubchDev *css_create_sch(CssDevId bus_id, bool is_virtual, bool squash_mcss,
                          Error **errp);
+
+/** Turn on css migration */
+void css_register_vmstate(void);
+
 #endif

include/hw/s390x/s390-virtio-ccw.h

@@ -39,11 +39,21 @@ typedef struct S390CcwMachineClass {
     /*< public >*/
     bool ri_allowed;
     bool cpu_model_allowed;
+    bool css_migration_enabled;
+    bool gs_allowed;
 } S390CcwMachineClass;
 
 /* runtime-instrumentation allowed by the machine */
 bool ri_allowed(void);
 /* cpu model allowed by the machine */
 bool cpu_model_allowed(void);
+/* guarded-storage allowed by the machine */
+bool gs_allowed(void);
+
+/**
+ * Returns true if (vmstate based) migration of the channel subsystem
+ * is enabled, false if it is disabled.
+ */
+bool css_migration_enabled(void);
 
 #endif

include/hw/s390x/s390_flic.h

@@ -44,7 +44,7 @@ typedef struct S390FLICState {
     SysBusDevice parent_obj;
     /* to limit AdapterRoutes.num_routes for compat */
     uint32_t adapter_routes_max_batch;
-
+    bool ais_supported;
 } S390FLICState;
 
 #define S390_FLIC_COMMON_CLASS(klass) \
@@ -56,13 +56,16 @@ typedef struct S390FLICStateClass {
     DeviceClass parent_class;
 
     int (*register_io_adapter)(S390FLICState *fs, uint32_t id, uint8_t isc,
-                               bool swap, bool maskable);
+                               bool swap, bool maskable, uint8_t flags);
     int (*io_adapter_map)(S390FLICState *fs, uint32_t id, uint64_t map_addr,
                           bool do_map);
     int (*add_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
     void (*release_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
     int (*clear_io_irq)(S390FLICState *fs, uint16_t subchannel_id,
                         uint16_t subchannel_nr);
+    int (*modify_ais_mode)(S390FLICState *fs, uint8_t isc, uint16_t mode);
+    int (*inject_airq)(S390FLICState *fs, uint8_t type, uint8_t isc,
+                       uint8_t flags);
 } S390FLICStateClass;
 
 #define TYPE_KVM_S390_FLIC "s390-flic-kvm"
@@ -73,13 +76,20 @@ typedef struct S390FLICStateClass {
 #define QEMU_S390_FLIC(obj) \
     OBJECT_CHECK(QEMUS390FLICState, (obj), TYPE_QEMU_S390_FLIC)
 
+#define SIC_IRQ_MODE_ALL 0
+#define SIC_IRQ_MODE_SINGLE 1
+#define AIS_MODE_MASK(isc) (0x80 >> isc)
+
 typedef struct QEMUS390FLICState {
     S390FLICState parent_obj;
+    uint8_t simm;
+    uint8_t nimm;
 } QEMUS390FLICState;
 
 void s390_flic_init(void);
 
 S390FLICState *s390_get_flic(void);
+bool ais_needed(void *opaque);
 
 #ifdef CONFIG_KVM
 DeviceState *s390_flic_kvm_create(void);

include/hw/s390x/sclp.h

@@ -123,8 +123,7 @@ typedef struct ReadInfo {
     uint64_t facilities;                /* 48-55 */
     uint8_t  _reserved0[76 - 56];       /* 56-75 */
     uint32_t ibc_val;
-    uint8_t  conf_char[96 - 80];        /* 80-95 */
+    uint8_t  conf_char[99 - 80];        /* 80-98 */
-    uint8_t  _reserved4[99 - 96];       /* 96-98 */
     uint8_t mha_pow;
     uint32_t rnsize2;
     uint64_t rnmax2;

include/hw/s390x/storage-attributes.h

@@ -0,0 +1,81 @@
+/*
+ * s390 storage attributes device
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Claudio Imbrenda <imbrenda@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef S390_STORAGE_ATTRIBUTES_H
+#define S390_STORAGE_ATTRIBUTES_H
+
+#include <hw/qdev.h>
+#include "monitor/monitor.h"
+
+#define TYPE_S390_STATTRIB "s390-storage_attributes"
+#define TYPE_QEMU_S390_STATTRIB "s390-storage_attributes-qemu"
+#define TYPE_KVM_S390_STATTRIB "s390-storage_attributes-kvm"
+
+#define S390_STATTRIB(obj) \
+    OBJECT_CHECK(S390StAttribState, (obj), TYPE_S390_STATTRIB)
+
+typedef struct S390StAttribState {
+    DeviceState parent_obj;
+    uint64_t migration_cur_gfn;
+    bool migration_enabled;
+} S390StAttribState;
+
+#define S390_STATTRIB_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390StAttribClass, (klass), TYPE_S390_STATTRIB)
+#define S390_STATTRIB_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390StAttribClass, (obj), TYPE_S390_STATTRIB)
+
+typedef struct S390StAttribClass {
+    DeviceClass parent_class;
+    /* Return value: < 0 on error, or new count */
+    int (*get_stattr)(S390StAttribState *sa, uint64_t *start_gfn,
+                      uint32_t count, uint8_t *values);
+    int (*peek_stattr)(S390StAttribState *sa, uint64_t start_gfn,
+                       uint32_t count, uint8_t *values);
+    int (*set_stattr)(S390StAttribState *sa, uint64_t start_gfn,
+                      uint32_t count, uint8_t *values);
+    void (*synchronize)(S390StAttribState *sa);
+    int (*set_migrationmode)(S390StAttribState *sa, bool value);
+    int (*get_active)(S390StAttribState *sa);
+    long long (*get_dirtycount)(S390StAttribState *sa);
+} S390StAttribClass;
+
+#define QEMU_S390_STATTRIB(obj) \
+    OBJECT_CHECK(QEMUS390StAttribState, (obj), TYPE_QEMU_S390_STATTRIB)
+
+typedef struct QEMUS390StAttribState {
+    S390StAttribState parent_obj;
+} QEMUS390StAttribState;
+
+#define KVM_S390_STATTRIB(obj) \
+    OBJECT_CHECK(KVMS390StAttribState, (obj), TYPE_KVM_S390_STATTRIB)
+
+typedef struct KVMS390StAttribState {
+    S390StAttribState parent_obj;
+    uint64_t still_dirty;
+    uint8_t *incoming_buffer;
+} KVMS390StAttribState;
+
+void s390_stattrib_init(void);
+
+#ifdef CONFIG_KVM
+Object *kvm_s390_stattrib_create(void);
+#else
+static inline Object *kvm_s390_stattrib_create(void)
+{
+    return NULL;
+}
+#endif
+
+void hmp_info_cmma(Monitor *mon, const QDict *qdict);
+void hmp_migrationmode(Monitor *mon, const QDict *qdict);
+
+#endif /* S390_STORAGE_ATTRIBUTES_H */

include/standard-headers/asm-x86/hyperv.h

@@ -34,16 +34,10 @@
 #define HV_X64_MSR_REFERENCE_TSC		0x40000021
 
 /*
- * There is a single feature flag that signifies the presence of the MSR
+ * There is a single feature flag that signifies if the partition has access
- * that can be used to retrieve both the local APIC Timer frequency as
+ * to MSRs with local APIC and TSC frequencies.
- * well as the TSC frequency.
  */
-
+#define HV_X64_ACCESS_FREQUENCY_MSRS		(1 << 11)
-/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
-#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
-
-/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
-#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
 
 /*
  * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
@@ -73,6 +67,9 @@
   */
 #define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
 
+/* Frequency MSRs available */
+#define HV_FEATURE_FREQUENCY_MSRS_AVAILABLE	(1 << 8)
+
 /* Crash MSR available */
 #define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE (1 << 10)
 
@@ -152,6 +149,12 @@
  */
 #define HV_X64_DEPRECATING_AEOI_RECOMMENDED	(1 << 9)
 
+/*
+ * HV_VP_SET available
+ */
+#define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED	(1 << 11)
+
+
 /*
  * Crash notification flag.
  */

include/standard-headers/linux/input-event-codes.h

@@ -600,6 +600,7 @@
 #define KEY_APPSELECT		0x244	/* AL Select Task/Application */
 #define KEY_SCREENSAVER		0x245	/* AL Screen Saver */
 #define KEY_VOICECOMMAND		0x246	/* Listening Voice Command */
+#define KEY_ASSISTANT		0x247	/* AL Context-aware desktop assistant */
 
 #define KEY_BRIGHTNESS_MIN		0x250	/* Set Brightness to Minimum */
 #define KEY_BRIGHTNESS_MAX		0x251	/* Set Brightness to Maximum */

include/standard-headers/linux/pci_regs.h

@@ -517,6 +517,7 @@
 #define  PCI_EXP_LNKCAP_SLS	0x0000000f /* Supported Link Speeds */
 #define  PCI_EXP_LNKCAP_SLS_2_5GB 0x00000001 /* LNKCAP2 SLS Vector bit 0 */
 #define  PCI_EXP_LNKCAP_SLS_5_0GB 0x00000002 /* LNKCAP2 SLS Vector bit 1 */
+#define  PCI_EXP_LNKCAP_SLS_8_0GB 0x00000003 /* LNKCAP2 SLS Vector bit 2 */
 #define  PCI_EXP_LNKCAP_MLW	0x000003f0 /* Maximum Link Width */
 #define  PCI_EXP_LNKCAP_ASPMS	0x00000c00 /* ASPM Support */
 #define  PCI_EXP_LNKCAP_L0SEL	0x00007000 /* L0s Exit Latency */

linux-headers/asm-arm/kvm.h

@@ -203,6 +203,14 @@ struct kvm_arch_memory_slot {
 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INTID_MASK 0x3ff
 #define VGIC_LEVEL_INFO_LINE_LEVEL	0
 
+/* Device Control API on vcpu fd */
+#define KVM_ARM_VCPU_PMU_V3_CTRL	0
+#define   KVM_ARM_VCPU_PMU_V3_IRQ	0
+#define   KVM_ARM_VCPU_PMU_V3_INIT	1
+#define KVM_ARM_VCPU_TIMER_CTRL		1
+#define   KVM_ARM_VCPU_TIMER_IRQ_VTIMER		0
+#define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER		1
+
 #define   KVM_DEV_ARM_VGIC_CTRL_INIT		0
 #define   KVM_DEV_ARM_ITS_SAVE_TABLES		1
 #define   KVM_DEV_ARM_ITS_RESTORE_TABLES	2

linux-headers/asm-arm64/kvm.h

@@ -232,6 +232,9 @@ struct kvm_arch_memory_slot {
 #define KVM_ARM_VCPU_PMU_V3_CTRL	0
 #define   KVM_ARM_VCPU_PMU_V3_IRQ	0
 #define   KVM_ARM_VCPU_PMU_V3_INIT	1
+#define KVM_ARM_VCPU_TIMER_CTRL		1
+#define   KVM_ARM_VCPU_TIMER_IRQ_VTIMER		0
+#define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER		1
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT		24

linux-headers/asm-powerpc/kvm.h

@@ -60,6 +60,12 @@ struct kvm_regs {
 
 #define KVM_SREGS_E_FSL_PIDn	(1 << 0) /* PID1/PID2 */
 
+/* flags for kvm_run.flags */
+#define KVM_RUN_PPC_NMI_DISP_MASK		(3 << 0)
+#define   KVM_RUN_PPC_NMI_DISP_FULLY_RECOV	(1 << 0)
+#define   KVM_RUN_PPC_NMI_DISP_LIMITED_RECOV	(2 << 0)
+#define   KVM_RUN_PPC_NMI_DISP_NOT_RECOV	(3 << 0)
+
 /*
  * Feature bits indicate which sections of the sregs struct are valid,
  * both in KVM_GET_SREGS and KVM_SET_SREGS.  On KVM_SET_SREGS, registers

linux-headers/asm-s390/kvm.h

@@ -28,6 +28,7 @@
 #define KVM_DEV_FLIC_CLEAR_IO_IRQ	8
 #define KVM_DEV_FLIC_AISM		9
 #define KVM_DEV_FLIC_AIRQ_INJECT	10
+#define KVM_DEV_FLIC_AISM_ALL		11
 /*
  * We can have up to 4*64k pending subchannels + 8 adapter interrupts,
  * as well as up  to ASYNC_PF_PER_VCPU*KVM_MAX_VCPUS pfault done interrupts.
@@ -53,6 +54,11 @@ struct kvm_s390_ais_req {
 	__u16 mode;
 };
 
+struct kvm_s390_ais_all {
+	__u8 simm;
+	__u8 nimm;
+};
+
 #define KVM_S390_IO_ADAPTER_MASK 1
 #define KVM_S390_IO_ADAPTER_MAP 2
 #define KVM_S390_IO_ADAPTER_UNMAP 3
@@ -70,6 +76,7 @@ struct kvm_s390_io_adapter_req {
 #define KVM_S390_VM_TOD			1
 #define KVM_S390_VM_CRYPTO		2
 #define KVM_S390_VM_CPU_MODEL		3
+#define KVM_S390_VM_MIGRATION		4
 
 /* kvm attributes for mem_ctrl */
 #define KVM_S390_VM_MEM_ENABLE_CMMA	0
@@ -151,6 +158,11 @@ struct kvm_s390_vm_cpu_subfunc {
 #define KVM_S390_VM_CRYPTO_DISABLE_AES_KW	2
 #define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW	3
 
+/* kvm attributes for migration mode */
+#define KVM_S390_VM_MIGRATION_STOP	0
+#define KVM_S390_VM_MIGRATION_START	1
+#define KVM_S390_VM_MIGRATION_STATUS	2
+
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 	/* general purpose regs for s390 */

linux-headers/linux/kvm.h

@@ -155,6 +155,35 @@ struct kvm_s390_skeys {
 	__u32 reserved[9];
 };
 
+#define KVM_S390_CMMA_PEEK (1 << 0)
+
+/**
+ * kvm_s390_cmma_log - Used for CMMA migration.
+ *
+ * Used both for input and output.
+ *
+ * @start_gfn: Guest page number to start from.
+ * @count: Size of the result buffer.
+ * @flags: Control operation mode via KVM_S390_CMMA_* flags
+ * @remaining: Used with KVM_S390_GET_CMMA_BITS. Indicates how many dirty
+ *             pages are still remaining.
+ * @mask: Used with KVM_S390_SET_CMMA_BITS. Bitmap of bits to actually set
+ *        in the PGSTE.
+ * @values: Pointer to the values buffer.
+ *
+ * Used in KVM_S390_{G,S}ET_CMMA_BITS ioctls.
+ */
+struct kvm_s390_cmma_log {
+	__u64 start_gfn;
+	__u32 count;
+	__u32 flags;
+	union {
+		__u64 remaining;
+		__u64 mask;
+	};
+	__u64 values;
+};
+
 struct kvm_hyperv_exit {
 #define KVM_EXIT_HYPERV_SYNIC          1
 #define KVM_EXIT_HYPERV_HCALL          2
@@ -895,6 +924,9 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_SPAPR_TCE_VFIO 142
 #define KVM_CAP_X86_GUEST_MWAIT 143
 #define KVM_CAP_ARM_USER_IRQ 144
+#define KVM_CAP_S390_CMMA_MIGRATION 145
+#define KVM_CAP_PPC_FWNMI 146
+#define KVM_CAP_PPC_SMT_POSSIBLE 147
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1318,6 +1350,9 @@ struct kvm_s390_ucas_mapping {
 #define KVM_S390_GET_IRQ_STATE	  _IOW(KVMIO, 0xb6, struct kvm_s390_irq_state)
 /* Available with KVM_CAP_X86_SMM */
 #define KVM_SMI                   _IO(KVMIO,   0xb7)
+/* Available with KVM_CAP_S390_CMMA_MIGRATION */
+#define KVM_S390_GET_CMMA_BITS      _IOW(KVMIO, 0xb8, struct kvm_s390_cmma_log)
+#define KVM_S390_SET_CMMA_BITS      _IOW(KVMIO, 0xb9, struct kvm_s390_cmma_log)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)

monitor.c

@@ -81,6 +81,7 @@
 
 #if defined(TARGET_S390X)
 #include "hw/s390x/storage-keys.h"
+#include "hw/s390x/storage-attributes.h"
 #endif
 
 /*

pc-bios/s390-ccw/Makefile

@@ -9,14 +9,14 @@ $(call set-vpath, $(SRC_PATH)/pc-bios/s390-ccw)
 
 .PHONY : all clean build-all
 
-OBJECTS = start.o main.o bootmap.o sclp.o virtio.o virtio-scsi.o
+OBJECTS = start.o main.o bootmap.o sclp.o virtio.o virtio-scsi.o virtio-blkdev.o
 QEMU_CFLAGS := $(filter -W%, $(QEMU_CFLAGS))
 QEMU_CFLAGS += -ffreestanding -fno-delete-null-pointer-checks -msoft-float
 QEMU_CFLAGS += -march=z900 -fPIE -fno-strict-aliasing
 QEMU_CFLAGS += $(call cc-option, $(QEMU_CFLAGS), -fno-stack-protector)
 LDFLAGS += -Wl,-pie -nostdlib
 
-build-all: s390-ccw.img
+build-all: s390-ccw.img s390-netboot.img
 
 s390-ccw.elf: $(OBJECTS)
 	$(call quiet-command,$(CC) $(LDFLAGS) -o $@ $(OBJECTS),"BUILD","$(TARGET_DIR)$@")
@@ -28,5 +28,12 @@ s390-ccw.img: s390-ccw.elf
 
 $(OBJECTS): Makefile
 
+ifneq ($(wildcard $(SRC_PATH)/roms/SLOF/lib/libnet),)
+include $(SRC_PATH)/pc-bios/s390-ccw/netboot.mak
+else
+s390-netboot.img:
+	@echo "s390-netboot.img not built since roms/SLOF/ is not available."
+endif
+
 clean:
-	rm -f *.o *.d *.img *.elf *~
+	$(RM) *.o *.d *.img *.elf *~ *.a

pc-bios/s390-ccw/bootmap.c

@@ -8,9 +8,11 @@
  * directory.
  */
 
+#include "libc.h"
 #include "s390-ccw.h"
 #include "bootmap.h"
 #include "virtio.h"
+#include "bswap.h"
 
 #ifdef DEBUG
 /* #define DEBUG_FALLBACK */

pc-bios/s390-ccw/bootmap.h

@@ -324,32 +324,6 @@ static inline int _memcmp(const void *s1, const void *s2, size_t n)
     return 0;
 }
 
-/* from include/qemu/bswap.h */
-
-/* El Torito is always little-endian */
-static inline uint16_t bswap16(uint16_t x)
-{
-    return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8);
-}
-
-static inline uint32_t bswap32(uint32_t x)
-{
-    return ((x & 0x000000ffU) << 24) | ((x & 0x0000ff00U) <<  8) |
-           ((x & 0x00ff0000U) >>  8) | ((x & 0xff000000U) >> 24);
-}
-
-static inline uint64_t bswap64(uint64_t x)
-{
-    return ((x & 0x00000000000000ffULL) << 56) |
-           ((x & 0x000000000000ff00ULL) << 40) |
-           ((x & 0x0000000000ff0000ULL) << 24) |
-           ((x & 0x00000000ff000000ULL) <<  8) |
-           ((x & 0x000000ff00000000ULL) >>  8) |
-           ((x & 0x0000ff0000000000ULL) >> 24) |
-           ((x & 0x00ff000000000000ULL) >> 40) |
-           ((x & 0xff00000000000000ULL) >> 56);
-}
-
 static inline uint32_t iso_733_to_u32(uint64_t x)
 {
     return (uint32_t)x;

pc-bios/s390-ccw/bswap.h

@@ -0,0 +1,30 @@
+/*
+ * Byte swap functions - taken from include/qemu/bswap.h
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+static inline uint16_t bswap16(uint16_t x)
+{
+    return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return ((x & 0x000000ffU) << 24) | ((x & 0x0000ff00U) <<  8) |
+           ((x & 0x00ff0000U) >>  8) | ((x & 0xff000000U) >> 24);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return ((x & 0x00000000000000ffULL) << 56) |
+           ((x & 0x000000000000ff00ULL) << 40) |
+           ((x & 0x0000000000ff0000ULL) << 24) |
+           ((x & 0x00000000ff000000ULL) <<  8) |
+           ((x & 0x000000ff00000000ULL) >>  8) |
+           ((x & 0x0000ff0000000000ULL) >> 24) |
+           ((x & 0x00ff000000000000ULL) >> 40) |
+           ((x & 0xff00000000000000ULL) >> 56);
+}

pc-bios/s390-ccw/libc.h

@@ -0,0 +1,45 @@
+/*
+ * libc-style definitions and functions
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#ifndef S390_CCW_LIBC_H
+#define S390_CCW_LIBC_H
+
+typedef long               size_t;
+typedef int                bool;
+typedef unsigned char      uint8_t;
+typedef unsigned short     uint16_t;
+typedef unsigned int       uint32_t;
+typedef unsigned long long uint64_t;
+
+static inline void *memset(void *s, int c, size_t n)
+{
+    int i;
+    unsigned char *p = s;
+
+    for (i = 0; i < n; i++) {
+        p[i] = c;
+    }
+
+    return s;
+}
+
+static inline void *memcpy(void *s1, const void *s2, size_t n)
+{
+    uint8_t *dest = s1;
+    const uint8_t *src = s2;
+    int i;
+
+    for (i = 0; i < n; i++) {
+        dest[i] = src[i];
+    }
+
+    return s1;
+}
+
+#endif

pc-bios/s390-ccw/main.c

@@ -8,6 +8,7 @@
  * directory.
  */
 
+#include "libc.h"
 #include "s390-ccw.h"
 #include "virtio.h"
 
@@ -16,17 +17,6 @@ static SubChannelId blk_schid = { .one = 1 };
 IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE)));
 static char loadparm[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
 
-const unsigned char ebc2asc[256] =
-      /* 0123456789abcdef0123456789abcdef */
-        "................................" /* 1F */
-        "................................" /* 3F */
-        " ...........<(+|&.........!$*);." /* 5F first.chr.here.is.real.space */
-        "-/.........,%_>?.........`:#@'=\""/* 7F */
-        ".abcdefghi.......jklmnopqr......" /* 9F */
-        "..stuvwxyz......................" /* BF */
-        ".ABCDEFGHI.......JKLMNOPQR......" /* DF */
-        "..STUVWXYZ......0123456789......";/* FF */
-
 /*
  * Priniciples of Operations (SA22-7832-09) chapter 17 requires that
  * a subsystem-identification is at 184-187 and bytes 188-191 are zero
@@ -154,7 +144,7 @@ static void virtio_setup(void)
         sclp_print("Network boot device detected\n");
         vdev->netboot_start_addr = iplb.ccw.netboot_start_addr;
     } else {
-        virtio_setup_device(blk_schid);
+        virtio_blk_setup_device(blk_schid);
 
         IPL_assert(virtio_ipl_disk_is_valid(), "No valid IPL device detected");
     }

pc-bios/s390-ccw/netboot.mak

@@ -0,0 +1,59 @@
+
+SLOF_DIR := $(SRC_PATH)/roms/SLOF
+
+NETOBJS := start.o sclp.o virtio.o virtio-net.o netmain.o libnet.a libc.a
+
+LIBC_INC := -nostdinc -I$(SLOF_DIR)/lib/libc/include
+LIBNET_INC := -I$(SLOF_DIR)/lib/libnet
+
+NETLDFLAGS := $(LDFLAGS) -Ttext=0x7800000
+
+$(NETOBJS): QEMU_CFLAGS += $(LIBC_INC) $(LIBNET_INC)
+
+s390-netboot.elf: $(NETOBJS)
+	$(call quiet-command,$(CC) $(NETLDFLAGS) -o $@ $(NETOBJS),"BUILD","$(TARGET_DIR)$@")
+
+s390-netboot.img: s390-netboot.elf
+	$(call quiet-command,$(STRIP) --strip-unneeded $< -o $@,"STRIP","$(TARGET_DIR)$@")
+
+# libc files:
+
+LIBC_CFLAGS :=  $(QEMU_CFLAGS) $(LIBC_INC) $(LIBNET_INC)
+
+CTYPE_OBJS = isdigit.o isxdigit.o toupper.o
+%.o : $(SLOF_DIR)/lib/libc/ctype/%.c
+	$(call quiet-command,$(CC) $(LIBC_CFLAGS) -c -o $@ $<,"CC","$(TARGET_DIR)$@")
+
+STRING_OBJS = strcat.o strchr.o strcmp.o strcpy.o strlen.o strncmp.o strncpy.o \
+	      strstr.o memset.o memcpy.o memmove.o memcmp.o
+%.o : $(SLOF_DIR)/lib/libc/string/%.c
+	$(call quiet-command,$(CC) $(LIBC_CFLAGS) -c -o $@ $<,"CC","$(TARGET_DIR)$@")
+
+STDLIB_OBJS = atoi.o atol.o strtoul.o strtol.o rand.o malloc.o free.o
+%.o : $(SLOF_DIR)/lib/libc/stdlib/%.c
+	$(call quiet-command,$(CC) $(LIBC_CFLAGS) -c -o $@ $<,"CC","$(TARGET_DIR)$@")
+
+STDIO_OBJS = sprintf.o vfprintf.o vsnprintf.o vsprintf.o fprintf.o \
+	     printf.o putc.o puts.o putchar.o stdchnls.o fileno.o
+%.o : $(SLOF_DIR)/lib/libc/stdio/%.c
+	$(call quiet-command,$(CC) $(LIBC_CFLAGS) -c -o $@ $<,"CC","$(TARGET_DIR)$@")
+
+sbrk.o: $(SLOF_DIR)/slof/sbrk.c
+	$(call quiet-command,$(CC) $(LIBC_CFLAGS) -c -o $@ $<,"CC","$(TARGET_DIR)$@")
+
+LIBCOBJS := $(STRING_OBJS) $(CTYPE_OBJS) $(STDLIB_OBJS) $(STDIO_OBJS) sbrk.o
+
+libc.a: $(LIBCOBJS)
+	$(call quiet-command,$(AR) -rc $@ $^,"AR","$(TARGET_DIR)$@")
+
+# libnet files:
+
+LIBNETOBJS := args.o dhcp.o dns.o icmpv6.o ipv6.o tcp.o udp.o bootp.o \
+	      dhcpv6.o ethernet.o ipv4.o ndp.o tftp.o
+LIBNETCFLAGS := $(QEMU_CFLAGS) $(LIBC_INC) $(LIBNET_INC)
+
+%.o : $(SLOF_DIR)/lib/libnet/%.c
+	$(call quiet-command,$(CC) $(LIBNETCFLAGS) -c -o $@ $<,"CC","$(TARGET_DIR)$@")
+
+libnet.a: $(LIBNETOBJS)
+	$(call quiet-command,$(AR) -rc $@ $^,"AR","$(TARGET_DIR)$@")

pc-bios/s390-ccw/netmain.c

@@ -0,0 +1,361 @@
+/*
+ * S390 virtio-ccw network boot loading program
+ *
+ * Copyright 2017 Thomas Huth, Red Hat Inc.
+ *
+ * Based on the S390 virtio-ccw loading program (main.c)
+ * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
+ *
+ * And based on the network loading code from SLOF (netload.c)
+ * Copyright (c) 2004, 2008 IBM Corporation
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <tftp.h>
+#include <ethernet.h>
+#include <dhcp.h>
+#include <dhcpv6.h>
+#include <ipv4.h>
+#include <ipv6.h>
+#include <dns.h>
+#include <time.h>
+
+#include "s390-ccw.h"
+#include "virtio.h"
+
+#define DEFAULT_BOOT_RETRIES 10
+#define DEFAULT_TFTP_RETRIES 20
+
+extern char _start[];
+
+char stack[PAGE_SIZE * 8] __attribute__((aligned(PAGE_SIZE)));
+IplParameterBlock iplb __attribute__((aligned(PAGE_SIZE)));
+
+static SubChannelId net_schid = { .one = 1 };
+static int ip_version = 4;
+static uint64_t dest_timer;
+
+static uint64_t get_timer_ms(void)
+{
+    uint64_t clk;
+
+    asm volatile(" stck %0 " : : "Q"(clk) : "memory");
+
+    /* Bit 51 is incremented each microsecond */
+    return (clk >> (63 - 51)) / 1000;
+}
+
+void set_timer(int val)
+{
+    dest_timer = get_timer_ms() + val;
+}
+
+int get_timer(void)
+{
+    return dest_timer - get_timer_ms();
+}
+
+int get_sec_ticks(void)
+{
+    return 1000;    /* number of ticks in 1 second */
+}
+
+/**
+ * Obtain IP and configuration info from DHCP server (either IPv4 or IPv6).
+ * @param  fn_ip     contains the following configuration information:
+ *                   client MAC, client IP, TFTP-server MAC, TFTP-server IP,
+ *                   boot file name
+ * @param  retries   Number of DHCP attempts
+ * @return           0 : IP and configuration info obtained;
+ *                   non-0 : error condition occurred.
+ */
+static int dhcp(struct filename_ip *fn_ip, int retries)
+{
+    int i = retries + 1;
+    int rc = -1;
+
+    printf("  Requesting information via DHCP:     ");
+
+    dhcpv4_generate_transaction_id();
+    dhcpv6_generate_transaction_id();
+
+    do {
+        printf("\b\b\b%03d", i - 1);
+        if (!--i) {
+            printf("\nGiving up after %d DHCP requests\n", retries);
+            return -1;
+        }
+        ip_version = 4;
+        rc = dhcpv4(NULL, fn_ip);
+        if (rc == -1) {
+            ip_version = 6;
+            set_ipv6_address(fn_ip->fd, 0);
+            rc = dhcpv6(NULL, fn_ip);
+            if (rc == 0) {
+                memcpy(&fn_ip->own_ip6, get_ipv6_address(), 16);
+                break;
+            }
+        }
+        if (rc != -1) {    /* either success or non-dhcp failure */
+            break;
+        }
+    } while (1);
+    printf("\b\b\b\bdone\n");
+
+    return rc;
+}
+
+/**
+ * Seed the random number generator with our mac and current timestamp
+ */
+static void seed_rng(uint8_t mac[])
+{
+    uint64_t seed;
+
+    asm volatile(" stck %0 " : : "Q"(seed) : "memory");
+    seed ^= (mac[2] << 24) | (mac[3] << 16) | (mac[4] << 8) | mac[5];
+    srand(seed);
+}
+
+static int tftp_load(filename_ip_t *fnip, void *buffer, int len,
+                     unsigned int retries, int ip_vers)
+{
+    tftp_err_t tftp_err;
+    int rc;
+
+    rc = tftp(fnip, buffer, len, retries, &tftp_err, 1, 1428, ip_vers);
+
+    if (rc > 0) {
+        printf("  TFTP: Received %s (%d KBytes)\n", fnip->filename,
+               rc / 1024);
+    } else if (rc == -1) {
+        puts("unknown TFTP error");
+    } else if (rc == -2) {
+        printf("TFTP buffer of %d bytes is too small for %s\n",
+            len, fnip->filename);
+    } else if (rc == -3) {
+        printf("file not found: %s\n", fnip->filename);
+    } else if (rc == -4) {
+        puts("TFTP access violation");
+    } else if (rc == -5) {
+        puts("illegal TFTP operation");
+    } else if (rc == -6) {
+        puts("unknown TFTP transfer ID");
+    } else if (rc == -7) {
+        puts("no such TFTP user");
+    } else if (rc == -8) {
+        puts("TFTP blocksize negotiation failed");
+    } else if (rc == -9) {
+        puts("file exceeds maximum TFTP transfer size");
+    } else if (rc <= -10 && rc >= -15) {
+        const char *icmp_err_str;
+        switch (rc) {
+        case -ICMP_NET_UNREACHABLE - 10:
+            icmp_err_str = "net unreachable";
+            break;
+        case -ICMP_HOST_UNREACHABLE - 10:
+            icmp_err_str = "host unreachable";
+            break;
+        case -ICMP_PROTOCOL_UNREACHABLE - 10:
+            icmp_err_str = "protocol unreachable";
+            break;
+        case -ICMP_PORT_UNREACHABLE - 10:
+            icmp_err_str = "port unreachable";
+            break;
+        case -ICMP_FRAGMENTATION_NEEDED - 10:
+            icmp_err_str = "fragmentation needed and DF set";
+            break;
+        case -ICMP_SOURCE_ROUTE_FAILED - 10:
+            icmp_err_str = "source route failed";
+            break;
+        default:
+            icmp_err_str = " UNKNOWN";
+            break;
+        }
+        printf("ICMP ERROR \"%s\"\n", icmp_err_str);
+    } else if (rc == -40) {
+        printf("TFTP error occurred after %d bad packets received",
+            tftp_err.bad_tftp_packets);
+    } else if (rc == -41) {
+        printf("TFTP error occurred after missing %d responses",
+            tftp_err.no_packets);
+    } else if (rc == -42) {
+        printf("TFTP error missing block %d, expected block was %d",
+            tftp_err.blocks_missed,
+            tftp_err.blocks_received);
+    }
+
+    return rc;
+}
+
+static int net_load(char *buffer, int len)
+{
+    filename_ip_t fn_ip;
+    uint8_t mac[6];
+    int rc;
+
+    memset(&fn_ip, 0, sizeof(filename_ip_t));
+
+    rc = virtio_net_init(mac);
+    if (rc < 0) {
+        puts("Could not initialize network device");
+        return -101;
+    }
+    fn_ip.fd = rc;
+
+    printf("  Using MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
+           mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
+
+    set_mac_address(mac);    /* init ethernet layer */
+    seed_rng(mac);
+
+    rc = dhcp(&fn_ip, DEFAULT_BOOT_RETRIES);
+    if (rc >= 0) {
+        if (ip_version == 4) {
+            set_ipv4_address(fn_ip.own_ip);
+        }
+    } else {
+        puts("Could not get IP address");
+        return -101;
+    }
+
+    if (ip_version == 4) {
+        printf("  Using IPv4 address: %d.%d.%d.%d\n",
+              (fn_ip.own_ip >> 24) & 0xFF, (fn_ip.own_ip >> 16) & 0xFF,
+              (fn_ip.own_ip >>  8) & 0xFF, fn_ip.own_ip & 0xFF);
+    } else if (ip_version == 6) {
+        char ip6_str[40];
+        ipv6_to_str(fn_ip.own_ip6.addr, ip6_str);
+        printf("  Using IPv6 address: %s\n", ip6_str);
+    }
+
+    if (rc == -2) {
+        printf("ARP request to TFTP server (%d.%d.%d.%d) failed\n",
+               (fn_ip.server_ip >> 24) & 0xFF, (fn_ip.server_ip >> 16) & 0xFF,
+               (fn_ip.server_ip >>  8) & 0xFF, fn_ip.server_ip & 0xFF);
+        return -102;
+    }
+    if (rc == -4 || rc == -3) {
+        puts("Can't obtain TFTP server IP address");
+        return -107;
+    }
+
+    if (ip_version == 4) {
+        printf("  Requesting file \"%s\" via TFTP from %d.%d.%d.%d\n",
+               fn_ip.filename,
+               (fn_ip.server_ip >> 24) & 0xFF, (fn_ip.server_ip >> 16) & 0xFF,
+               (fn_ip.server_ip >>  8) & 0xFF, fn_ip.server_ip & 0xFF);
+    } else if (ip_version == 6) {
+        char ip6_str[40];
+        printf("  Requesting file \"%s\" via TFTP from ", fn_ip.filename);
+        ipv6_to_str(fn_ip.server_ip6.addr, ip6_str);
+        printf("%s\n", ip6_str);
+    }
+
+    /* Do the TFTP load and print error message if necessary */
+    rc = tftp_load(&fn_ip, buffer, len, DEFAULT_TFTP_RETRIES, ip_version);
+
+    if (ip_version == 4) {
+        dhcp_send_release(fn_ip.fd);
+    }
+
+    return rc;
+}
+
+void panic(const char *string)
+{
+    sclp_print(string);
+    for (;;) {
+        disabled_wait();
+    }
+}
+
+static bool find_net_dev(Schib *schib, int dev_no)
+{
+    int i, r;
+
+    for (i = 0; i < 0x10000; i++) {
+        net_schid.sch_no = i;
+        r = stsch_err(net_schid, schib);
+        if (r == 3 || r == -EIO) {
+            break;
+        }
+        if (!schib->pmcw.dnv) {
+            continue;
+        }
+        if (!virtio_is_supported(net_schid)) {
+            continue;
+        }
+        if (virtio_get_device_type() != VIRTIO_ID_NET) {
+            continue;
+        }
+        if (dev_no < 0 || schib->pmcw.dev == dev_no) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void virtio_setup(void)
+{
+    Schib schib;
+    int ssid;
+    bool found = false;
+    uint16_t dev_no;
+
+    /*
+     * We unconditionally enable mss support. In every sane configuration,
+     * this will succeed; and even if it doesn't, stsch_err() can deal
+     * with the consequences.
+     */
+    enable_mss_facility();
+
+    if (store_iplb(&iplb)) {
+        IPL_assert(iplb.pbt == S390_IPL_TYPE_CCW, "IPL_TYPE_CCW expected");
+        dev_no = iplb.ccw.devno;
+        debug_print_int("device no. ", dev_no);
+        net_schid.ssid = iplb.ccw.ssid & 0x3;
+        debug_print_int("ssid ", net_schid.ssid);
+        found = find_net_dev(&schib, dev_no);
+    } else {
+        for (ssid = 0; ssid < 0x3; ssid++) {
+            net_schid.ssid = ssid;
+            found = find_net_dev(&schib, -1);
+            if (found) {
+                break;
+            }
+        }
+    }
+
+    IPL_assert(found, "No virtio net device found");
+}
+
+void main(void)
+{
+    int rc;
+
+    sclp_setup();
+    sclp_print("Network boot starting...\n");
+
+    virtio_setup();
+
+    rc = net_load(NULL, (long)_start);
+    if (rc > 0) {
+        sclp_print("Network loading done, starting kernel...\n");
+        asm volatile (" lpsw 0(%0) " : : "r"(0) : "memory");
+    }
+
+    panic("Failed to load OS from network\n");
+}

pc-bios/s390-ccw/s390-ccw.h

@@ -18,12 +18,6 @@ typedef unsigned short     u16;
 typedef unsigned int       u32;
 typedef unsigned long long u64;
 typedef unsigned long      ulong;
-typedef long               size_t;
-typedef int                bool;
-typedef unsigned char      uint8_t;
-typedef unsigned short     uint16_t;
-typedef unsigned int       uint32_t;
-typedef unsigned long long uint64_t;
 typedef unsigned char      __u8;
 typedef unsigned short     __u16;
 typedef unsigned int       __u32;
@@ -50,6 +44,8 @@ typedef unsigned long long __u64;
                             ((b) == 0 ? (a) : (MIN(a, b))))
 #endif
 
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+
 #include "cio.h"
 #include "iplb.h"
 
@@ -80,7 +76,7 @@ void sclp_get_loadparm_ascii(char *loadparm);
 unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
                                  ulong subchan_id, void *load_addr);
 bool virtio_is_supported(SubChannelId schid);
-void virtio_setup_device(SubChannelId schid);
+void virtio_blk_setup_device(SubChannelId schid);
 int virtio_read(ulong sector, void *load_addr);
 int enable_mss_facility(void);
 ulong get_second(void);
@@ -88,18 +84,6 @@ ulong get_second(void);
 /* bootmap.c */
 void zipl_load(void);
 
-static inline void *memset(void *s, int c, size_t n)
-{
-    int i;
-    unsigned char *p = s;
-
-    for (i = 0; i < n; i++) {
-        p[i] = c;
-    }
-
-    return s;
-}
-
 static inline void fill_hex(char *out, unsigned char val)
 {
     const char hex[] = "0123456789abcdef";
@@ -169,17 +153,6 @@ static inline void sleep(unsigned int seconds)
     }
 }
 
-static inline void *memcpy(void *s1, const void *s2, size_t n)
-{
-    uint8_t *p1 = s1;
-    const uint8_t *p2 = s2;
-
-    while (n--) {
-        p1[n] = p2[n];
-    }
-    return s1;
-}
-
 static inline void IPL_assert(bool term, const char *message)
 {
     if (!term) {

pc-bios/s390-ccw/sclp.c

@@ -8,11 +8,25 @@
  * directory.
  */
 
+#include "libc.h"
 #include "s390-ccw.h"
 #include "sclp.h"
 
+long write(int fd, const void *str, size_t len);
+
 static char _sccb[PAGE_SIZE] __attribute__((__aligned__(4096)));
 
+const unsigned char ebc2asc[256] =
+      /* 0123456789abcdef0123456789abcdef */
+        "................................" /* 1F */
+        "................................" /* 3F */
+        " ...........<(+|&.........!$*);." /* 5F first.chr.here.is.real.space */
+        "-/.........,%_>?.........`:#@'=\""/* 7F */
+        ".abcdefghi.......jklmnopqr......" /* 9F */
+        "..stuvwxyz......................" /* BF */
+        ".ABCDEFGHI.......JKLMNOPQR......" /* DF */
+        "..STUVWXYZ......0123456789......";/* FF */
+
 /* Perform service call. Return 0 on success, non-zero otherwise. */
 static int sclp_service_call(unsigned int command, void *sccb)
 {
@@ -59,26 +73,29 @@ static int _strlen(const char *str)
     return i;
 }
 
-static void _memcpy(char *dest, const char *src, int len)
+long write(int fd, const void *str, size_t len)
 {
-    int i;
-    for (i = 0; i < len; i++)
-        dest[i] = src[i];
-}
-
-void sclp_print(const char *str)
-{
-    int len = _strlen(str);
     WriteEventData *sccb = (void *)_sccb;
 
+    if (fd != 1 && fd != 2) {
+        return -EIO;
+    }
+
     sccb->h.length = sizeof(WriteEventData) + len;
     sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
     sccb->ebh.length = sizeof(EventBufferHeader) + len;
     sccb->ebh.type = SCLP_EVENT_ASCII_CONSOLE_DATA;
     sccb->ebh.flags = 0;
-    _memcpy(sccb->data, str, len);
+    memcpy(sccb->data, str, len);
 
     sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
+
+    return len;
+}
+
+void sclp_print(const char *str)
+{
+    write(1, str, _strlen(str));
 }
 
 void sclp_get_loadparm_ascii(char *loadparm)

pc-bios/s390-ccw/virtio-blkdev.c

@@ -0,0 +1,296 @@
+/*
+ * Virtio driver bits
+ *
+ * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "libc.h"
+#include "s390-ccw.h"
+#include "virtio.h"
+#include "virtio-scsi.h"
+
+static int virtio_blk_read_many(VDev *vdev, ulong sector, void *load_addr,
+                                int sec_num)
+{
+    VirtioBlkOuthdr out_hdr;
+    u8 status;
+    VRing *vr = &vdev->vrings[vdev->cmd_vr_idx];
+
+    /* Tell the host we want to read */
+    out_hdr.type = VIRTIO_BLK_T_IN;
+    out_hdr.ioprio = 99;
+    out_hdr.sector = virtio_sector_adjust(sector);
+
+    vring_send_buf(vr, &out_hdr, sizeof(out_hdr), VRING_DESC_F_NEXT);
+
+    /* This is where we want to receive data */
+    vring_send_buf(vr, load_addr, virtio_get_block_size() * sec_num,
+                   VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN |
+                   VRING_DESC_F_NEXT);
+
+    /* status field */
+    vring_send_buf(vr, &status, sizeof(u8),
+                   VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN);
+
+    /* Now we can tell the host to read */
+    vring_wait_reply();
+
+    if (drain_irqs(vr->schid)) {
+        /* Well, whatever status is supposed to contain... */
+        status = 1;
+    }
+    return status;
+}
+
+int virtio_read_many(ulong sector, void *load_addr, int sec_num)
+{
+    VDev *vdev = virtio_get_device();
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return virtio_blk_read_many(vdev, sector, load_addr, sec_num);
+    case VIRTIO_ID_SCSI:
+        return virtio_scsi_read_many(vdev, sector, load_addr, sec_num);
+    }
+    panic("\n! No readable IPL device !\n");
+    return -1;
+}
+
+unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
+                                 ulong subchan_id, void *load_addr)
+{
+    u8 status;
+    int sec = rec_list1;
+    int sec_num = ((rec_list2 >> 32) & 0xffff) + 1;
+    int sec_len = rec_list2 >> 48;
+    ulong addr = (ulong)load_addr;
+
+    if (sec_len != virtio_get_block_size()) {
+        return -1;
+    }
+
+    sclp_print(".");
+    status = virtio_read_many(sec, (void *)addr, sec_num);
+    if (status) {
+        panic("I/O Error");
+    }
+    addr += sec_num * virtio_get_block_size();
+
+    return addr;
+}
+
+int virtio_read(ulong sector, void *load_addr)
+{
+    return virtio_read_many(sector, load_addr, 1);
+}
+
+/*
+ * Other supported value pairs, if any, would need to be added here.
+ * Note: head count is always 15.
+ */
+static inline u8 virtio_eckd_sectors_for_block_size(int size)
+{
+    switch (size) {
+    case 512:
+        return 49;
+    case 1024:
+        return 33;
+    case 2048:
+        return 21;
+    case 4096:
+        return 12;
+    }
+    return 0;
+}
+
+VirtioGDN virtio_guessed_disk_nature(void)
+{
+    return virtio_get_device()->guessed_disk_nature;
+}
+
+void virtio_assume_scsi(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev->guessed_disk_nature = VIRTIO_GDN_SCSI;
+        vdev->config.blk.blk_size = VIRTIO_SCSI_BLOCK_SIZE;
+        vdev->config.blk.physical_block_exp = 0;
+        vdev->blk_factor = 1;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev->scsi_block_size = VIRTIO_SCSI_BLOCK_SIZE;
+        break;
+    }
+}
+
+void virtio_assume_iso9660(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev->guessed_disk_nature = VIRTIO_GDN_SCSI;
+        vdev->config.blk.blk_size = VIRTIO_ISO_BLOCK_SIZE;
+        vdev->config.blk.physical_block_exp = 0;
+        vdev->blk_factor = VIRTIO_ISO_BLOCK_SIZE / VIRTIO_SECTOR_SIZE;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev->scsi_block_size = VIRTIO_ISO_BLOCK_SIZE;
+        break;
+    }
+}
+
+void virtio_assume_eckd(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    vdev->guessed_disk_nature = VIRTIO_GDN_DASD;
+    vdev->blk_factor = 1;
+    vdev->config.blk.physical_block_exp = 0;
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev->config.blk.blk_size = 4096;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev->config.blk.blk_size = vdev->scsi_block_size;
+        break;
+    }
+    vdev->config.blk.geometry.heads = 15;
+    vdev->config.blk.geometry.sectors =
+        virtio_eckd_sectors_for_block_size(vdev->config.blk.blk_size);
+}
+
+bool virtio_disk_is_scsi(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    if (vdev->guessed_disk_nature == VIRTIO_GDN_SCSI) {
+        return true;
+    }
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return (vdev->config.blk.geometry.heads == 255)
+            && (vdev->config.blk.geometry.sectors == 63)
+            && (virtio_get_block_size()  == VIRTIO_SCSI_BLOCK_SIZE);
+    case VIRTIO_ID_SCSI:
+        return true;
+    }
+    return false;
+}
+
+bool virtio_disk_is_eckd(void)
+{
+    VDev *vdev = virtio_get_device();
+    const int block_size = virtio_get_block_size();
+
+    if (vdev->guessed_disk_nature == VIRTIO_GDN_DASD) {
+        return true;
+    }
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return (vdev->config.blk.geometry.heads == 15)
+            && (vdev->config.blk.geometry.sectors ==
+                virtio_eckd_sectors_for_block_size(block_size));
+    case VIRTIO_ID_SCSI:
+        return false;
+    }
+    return false;
+}
+
+bool virtio_ipl_disk_is_valid(void)
+{
+    return virtio_disk_is_scsi() || virtio_disk_is_eckd();
+}
+
+int virtio_get_block_size(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev->config.blk.blk_size << vdev->config.blk.physical_block_exp;
+    case VIRTIO_ID_SCSI:
+        return vdev->scsi_block_size;
+    }
+    return 0;
+}
+
+uint8_t virtio_get_heads(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev->config.blk.geometry.heads;
+    case VIRTIO_ID_SCSI:
+        return vdev->guessed_disk_nature == VIRTIO_GDN_DASD
+               ? vdev->config.blk.geometry.heads : 255;
+    }
+    return 0;
+}
+
+uint8_t virtio_get_sectors(void)
+{
+    VDev *vdev = virtio_get_device();
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev->config.blk.geometry.sectors;
+    case VIRTIO_ID_SCSI:
+        return vdev->guessed_disk_nature == VIRTIO_GDN_DASD
+               ? vdev->config.blk.geometry.sectors : 63;
+    }
+    return 0;
+}
+
+uint64_t virtio_get_blocks(void)
+{
+    VDev *vdev = virtio_get_device();
+    const uint64_t factor = virtio_get_block_size() / VIRTIO_SECTOR_SIZE;
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev->config.blk.capacity / factor;
+    case VIRTIO_ID_SCSI:
+        return vdev->scsi_last_block / factor;
+    }
+    return 0;
+}
+
+void virtio_blk_setup_device(SubChannelId schid)
+{
+    VDev *vdev = virtio_get_device();
+
+    vdev->schid = schid;
+    virtio_setup_ccw(vdev);
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        sclp_print("Using virtio-blk.\n");
+        if (!virtio_ipl_disk_is_valid()) {
+            /* make sure all getters but blocksize return 0 for
+             * invalid IPL disk
+             */
+            memset(&vdev->config.blk, 0, sizeof(vdev->config.blk));
+            virtio_assume_scsi();
+        }
+        break;
+    case VIRTIO_ID_SCSI:
+        IPL_assert(vdev->config.scsi.sense_size == VIRTIO_SCSI_SENSE_SIZE,
+            "Config: sense size mismatch");
+        IPL_assert(vdev->config.scsi.cdb_size == VIRTIO_SCSI_CDB_SIZE,
+            "Config: CDB size mismatch");
+
+        sclp_print("Using virtio-scsi.\n");
+        virtio_scsi_setup(vdev);
+        break;
+    default:
+        panic("\n! No IPL device available !\n");
+    }
+}

pc-bios/s390-ccw/virtio-net.c

@@ -0,0 +1,135 @@
+/*
+ * Virtio-net driver for the s390-ccw firmware
+ *
+ * Copyright 2017 Thomas Huth, Red Hat Inc.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <ethernet.h>
+#include "s390-ccw.h"
+#include "virtio.h"
+
+#ifndef DEBUG_VIRTIO_NET
+#define DEBUG_VIRTIO_NET 0
+#endif
+
+#define VIRTIO_NET_F_MAC_BIT  (1 << 5)
+
+#define VQ_RX 0         /* Receive queue */
+#define VQ_TX 1         /* Transmit queue */
+
+struct VirtioNetHdr {
+    uint8_t flags;
+    uint8_t gso_type;
+    uint16_t hdr_len;
+    uint16_t gso_size;
+    uint16_t csum_start;
+    uint16_t csum_offset;
+    /*uint16_t num_buffers;*/ /* Only with VIRTIO_NET_F_MRG_RXBUF or VIRTIO1 */
+};
+typedef struct VirtioNetHdr VirtioNetHdr;
+
+static uint16_t rx_last_idx;  /* Last index in receive queue "used" ring */
+
+int virtio_net_init(void *mac_addr)
+{
+    VDev *vdev = virtio_get_device();
+    VRing *rxvq = &vdev->vrings[VQ_RX];
+    void *buf;
+    int i;
+
+    vdev->guest_features[0] = VIRTIO_NET_F_MAC_BIT;
+    virtio_setup_ccw(vdev);
+
+    IPL_assert(vdev->guest_features[0] & VIRTIO_NET_F_MAC_BIT,
+               "virtio-net device does not support the MAC address feature");
+    memcpy(mac_addr, vdev->config.net.mac, ETH_ALEN);
+
+    for (i = 0; i < 64; i++) {
+        buf = malloc(ETH_MTU_SIZE + sizeof(VirtioNetHdr));
+        IPL_assert(buf != NULL, "Can not allocate memory for receive buffers");
+        vring_send_buf(rxvq, buf, ETH_MTU_SIZE + sizeof(VirtioNetHdr),
+                       VRING_DESC_F_WRITE);
+    }
+    vring_notify(rxvq);
+
+    return 0;
+}
+
+int send(int fd, const void *buf, int len, int flags)
+{
+    VirtioNetHdr tx_hdr;
+    VDev *vdev = virtio_get_device();
+    VRing *txvq = &vdev->vrings[VQ_TX];
+
+    /* Set up header - we do not use anything special, so simply clear it */
+    memset(&tx_hdr, 0, sizeof(tx_hdr));
+
+    vring_send_buf(txvq, &tx_hdr, sizeof(tx_hdr), VRING_DESC_F_NEXT);
+    vring_send_buf(txvq, (void *)buf, len, VRING_HIDDEN_IS_CHAIN);
+    while (!vr_poll(txvq)) {
+        yield();
+    }
+    if (drain_irqs(txvq->schid)) {
+        puts("send: drain irqs failed");
+        return -1;
+    }
+
+    return len;
+}
+
+int recv(int fd, void *buf, int maxlen, int flags)
+{
+    VDev *vdev = virtio_get_device();
+    VRing *rxvq = &vdev->vrings[VQ_RX];
+    int len, id;
+    uint8_t *pkt;
+
+    if (rx_last_idx == rxvq->used->idx) {
+        return 0;
+    }
+
+    len = rxvq->used->ring[rx_last_idx % rxvq->num].len - sizeof(VirtioNetHdr);
+    if (len > maxlen) {
+        puts("virtio-net: Receive buffer too small");
+        len = maxlen;
+    }
+    id = rxvq->used->ring[rx_last_idx % rxvq->num].id % rxvq->num;
+    pkt = (uint8_t *)(rxvq->desc[id].addr + sizeof(VirtioNetHdr));
+
+#if DEBUG_VIRTIO_NET   /* Dump packet */
+    int i;
+    printf("\nbuf %p: len=%i\n", (void *)rxvq->desc[id].addr, len);
+    for (i = 0; i < 64; i++) {
+        printf(" %02x", pkt[i]);
+        if ((i % 16) == 15) {
+            printf("\n");
+        }
+    }
+    printf("\n");
+#endif
+
+    /* Copy data to destination buffer */
+    memcpy(buf, pkt, len);
+
+    /* Mark buffer as available to the host again */
+    rxvq->avail->ring[rxvq->avail->idx % rxvq->num] = id;
+    rxvq->avail->idx = rxvq->avail->idx + 1;
+    vring_notify(rxvq);
+
+    /* Move index to next entry */
+    rx_last_idx = rx_last_idx + 1;
+
+    return len;
+}

pc-bios/s390-ccw/virtio-scsi.c

@@ -9,6 +9,7 @@
  * directory.
  */
 
+#include "libc.h"
 #include "s390-ccw.h"
 #include "virtio.h"
 #include "scsi.h"

pc-bios/s390-ccw/virtio.c

@@ -8,9 +8,11 @@
  * directory.
  */
 
+#include "libc.h"
 #include "s390-ccw.h"
 #include "virtio.h"
 #include "virtio-scsi.h"
+#include "bswap.h"
 
 #define VRING_WAIT_REPLY_TIMEOUT 3
 
@@ -69,7 +71,7 @@ static long virtio_notify(SubChannelId schid, int vq_idx, long cookie)
  *             Virtio functions                *
  ***********************************************/
 
-static int drain_irqs(SubChannelId schid)
+int drain_irqs(SubChannelId schid)
 {
     Irb irb = {};
     int r = 0;
@@ -148,13 +150,13 @@ static void vring_init(VRing *vr, VqInfo *info)
     debug_print_addr("init vr", vr);
 }
 
-static bool vring_notify(VRing *vr)
+bool vring_notify(VRing *vr)
 {
     vr->cookie = virtio_notify(vr->schid, vr->id, vr->cookie);
     return vr->cookie >= 0;
 }
 
-static void vring_send_buf(VRing *vr, void *p, int len, int flags)
+void vring_send_buf(VRing *vr, void *p, int len, int flags)
 {
     /* For follow-up chains we need to keep the first entry point */
     if (!(flags & VRING_HIDDEN_IS_CHAIN)) {
@@ -187,7 +189,7 @@ ulong get_second(void)
     return (get_clock() >> 12) / 1000000;
 }
 
-static int vr_poll(VRing *vr)
+int vr_poll(VRing *vr)
 {
     if (vr->used->idx == vr->used_idx) {
         vring_notify(vr);
@@ -209,7 +211,7 @@ static int vr_poll(VRing *vr)
  *
  * Returns 0 on success, 1 on timeout.
  */
-static int vring_wait_reply(void)
+int vring_wait_reply(void)
 {
     ulong target_second = get_second() + vdev.wait_reply_timeout;
 
@@ -246,245 +248,14 @@ int virtio_run(VDev *vdev, int vqid, VirtioCmd *cmd)
     return 0;
 }
 
-/***********************************************
+void virtio_setup_ccw(VDev *vdev)
- *               Virtio block                  *
- ***********************************************/
-
-static int virtio_blk_read_many(VDev *vdev,
-                                ulong sector, void *load_addr, int sec_num)
 {
-    VirtioBlkOuthdr out_hdr;
+    int i, rc, cfg_size = 0;
-    u8 status;
-    VRing *vr = &vdev->vrings[vdev->cmd_vr_idx];
-
-    /* Tell the host we want to read */
-    out_hdr.type = VIRTIO_BLK_T_IN;
-    out_hdr.ioprio = 99;
-    out_hdr.sector = virtio_sector_adjust(sector);
-
-    vring_send_buf(vr, &out_hdr, sizeof(out_hdr), VRING_DESC_F_NEXT);
-
-    /* This is where we want to receive data */
-    vring_send_buf(vr, load_addr, virtio_get_block_size() * sec_num,
-                   VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN |
-                   VRING_DESC_F_NEXT);
-
-    /* status field */
-    vring_send_buf(vr, &status, sizeof(u8),
-                   VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN);
-
-    /* Now we can tell the host to read */
-    vring_wait_reply();
-
-    if (drain_irqs(vr->schid)) {
-        /* Well, whatever status is supposed to contain... */
-        status = 1;
-    }
-    return status;
-}
-
-int virtio_read_many(ulong sector, void *load_addr, int sec_num)
-{
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return virtio_blk_read_many(&vdev, sector, load_addr, sec_num);
-    case VIRTIO_ID_SCSI:
-        return virtio_scsi_read_many(&vdev, sector, load_addr, sec_num);
-    }
-    panic("\n! No readable IPL device !\n");
-    return -1;
-}
-
-unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
-                                 ulong subchan_id, void *load_addr)
-{
-    u8 status;
-    int sec = rec_list1;
-    int sec_num = ((rec_list2 >> 32) & 0xffff) + 1;
-    int sec_len = rec_list2 >> 48;
-    ulong addr = (ulong)load_addr;
-
-    if (sec_len != virtio_get_block_size()) {
-        return -1;
-    }
-
-    sclp_print(".");
-    status = virtio_read_many(sec, (void *)addr, sec_num);
-    if (status) {
-        panic("I/O Error");
-    }
-    addr += sec_num * virtio_get_block_size();
-
-    return addr;
-}
-
-int virtio_read(ulong sector, void *load_addr)
-{
-    return virtio_read_many(sector, load_addr, 1);
-}
-
-/*
- * Other supported value pairs, if any, would need to be added here.
- * Note: head count is always 15.
- */
-static inline u8 virtio_eckd_sectors_for_block_size(int size)
-{
-    switch (size) {
-    case 512:
-        return 49;
-    case 1024:
-        return 33;
-    case 2048:
-        return 21;
-    case 4096:
-        return 12;
-    }
-    return 0;
-}
-
-VirtioGDN virtio_guessed_disk_nature(void)
-{
-    return vdev.guessed_disk_nature;
-}
-
-void virtio_assume_scsi(void)
-{
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        vdev.guessed_disk_nature = VIRTIO_GDN_SCSI;
-        vdev.config.blk.blk_size = VIRTIO_SCSI_BLOCK_SIZE;
-        vdev.config.blk.physical_block_exp = 0;
-        vdev.blk_factor = 1;
-        break;
-    case VIRTIO_ID_SCSI:
-        vdev.scsi_block_size = VIRTIO_SCSI_BLOCK_SIZE;
-        break;
-    }
-}
-
-void virtio_assume_iso9660(void)
-{
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        vdev.guessed_disk_nature = VIRTIO_GDN_SCSI;
-        vdev.config.blk.blk_size = VIRTIO_ISO_BLOCK_SIZE;
-        vdev.config.blk.physical_block_exp = 0;
-        vdev.blk_factor = VIRTIO_ISO_BLOCK_SIZE / VIRTIO_SECTOR_SIZE;
-        break;
-    case VIRTIO_ID_SCSI:
-        vdev.scsi_block_size = VIRTIO_ISO_BLOCK_SIZE;
-        break;
-    }
-}
-
-void virtio_assume_eckd(void)
-{
-    vdev.guessed_disk_nature = VIRTIO_GDN_DASD;
-    vdev.blk_factor = 1;
-    vdev.config.blk.physical_block_exp = 0;
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        vdev.config.blk.blk_size = 4096;
-        break;
-    case VIRTIO_ID_SCSI:
-        vdev.config.blk.blk_size = vdev.scsi_block_size;
-        break;
-    }
-    vdev.config.blk.geometry.heads = 15;
-    vdev.config.blk.geometry.sectors =
-        virtio_eckd_sectors_for_block_size(vdev.config.blk.blk_size);
-}
-
-bool virtio_disk_is_scsi(void)
-{
-    if (vdev.guessed_disk_nature == VIRTIO_GDN_SCSI) {
-        return true;
-    }
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return (vdev.config.blk.geometry.heads == 255)
-            && (vdev.config.blk.geometry.sectors == 63)
-            && (virtio_get_block_size()  == VIRTIO_SCSI_BLOCK_SIZE);
-    case VIRTIO_ID_SCSI:
-        return true;
-    }
-    return false;
-}
-
-bool virtio_disk_is_eckd(void)
-{
-    const int block_size = virtio_get_block_size();
-
-    if (vdev.guessed_disk_nature == VIRTIO_GDN_DASD) {
-        return true;
-    }
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return (vdev.config.blk.geometry.heads == 15)
-            && (vdev.config.blk.geometry.sectors ==
-                virtio_eckd_sectors_for_block_size(block_size));
-    case VIRTIO_ID_SCSI:
-        return false;
-    }
-    return false;
-}
-
-bool virtio_ipl_disk_is_valid(void)
-{
-    return virtio_disk_is_scsi() || virtio_disk_is_eckd();
-}
-
-int virtio_get_block_size(void)
-{
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return vdev.config.blk.blk_size << vdev.config.blk.physical_block_exp;
-    case VIRTIO_ID_SCSI:
-        return vdev.scsi_block_size;
-    }
-    return 0;
-}
-
-uint8_t virtio_get_heads(void)
-{
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return vdev.config.blk.geometry.heads;
-    case VIRTIO_ID_SCSI:
-        return vdev.guessed_disk_nature == VIRTIO_GDN_DASD
-               ? vdev.config.blk.geometry.heads : 255;
-    }
-    return 0;
-}
-
-uint8_t virtio_get_sectors(void)
-{
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return vdev.config.blk.geometry.sectors;
-    case VIRTIO_ID_SCSI:
-        return vdev.guessed_disk_nature == VIRTIO_GDN_DASD
-               ? vdev.config.blk.geometry.sectors : 63;
-    }
-    return 0;
-}
-
-uint64_t virtio_get_blocks(void)
-{
-    const uint64_t factor = virtio_get_block_size() / VIRTIO_SECTOR_SIZE;
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        return vdev.config.blk.capacity / factor;
-    case VIRTIO_ID_SCSI:
-        return vdev.scsi_last_block / factor;
-    }
-    return 0;
-}
-
-static void virtio_setup_ccw(VDev *vdev)
-{
-    int i, cfg_size = 0;
     unsigned char status = VIRTIO_CONFIG_S_DRIVER_OK;
+    struct VirtioFeatureDesc {
+        uint32_t features;
+        uint8_t index;
+    } __attribute__((packed)) feats;
 
     IPL_assert(virtio_is_supported(vdev->schid), "PE");
     /* device ID has been established now */
@@ -495,6 +266,11 @@ static void virtio_setup_ccw(VDev *vdev)
     run_ccw(vdev, CCW_CMD_VDEV_RESET, NULL, 0);
 
     switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_NET:
+        vdev->nr_vqs = 2;
+        vdev->cmd_vr_idx = 0;
+        cfg_size = sizeof(vdev->config.net);
+        break;
     case VIRTIO_ID_BLOCK:
         vdev->nr_vqs = 1;
         vdev->cmd_vr_idx = 0;
@@ -511,11 +287,17 @@ static void virtio_setup_ccw(VDev *vdev)
     IPL_assert(run_ccw(vdev, CCW_CMD_READ_CONF, &vdev->config, cfg_size) == 0,
                "Could not get block device configuration");
 
-    /*
+    /* Feature negotiation */
-     * Skipping CCW_CMD_READ_FEAT. We're not doing anything fancy, and
+    for (i = 0; i < ARRAY_SIZE(vdev->guest_features); i++) {
-     * we'll just stop dead anyway if anything does not work like we
+        feats.features = 0;
-     * expect it.
+        feats.index = i;
-     */
+        rc = run_ccw(vdev, CCW_CMD_READ_FEAT, &feats, sizeof(feats));
+        IPL_assert(rc == 0, "Could not get features bits");
+        vdev->guest_features[i] &= bswap32(feats.features);
+        feats.features = bswap32(vdev->guest_features[i]);
+        rc = run_ccw(vdev, CCW_CMD_WRITE_FEAT, &feats, sizeof(feats));
+        IPL_assert(rc == 0, "Could not set features bits");
+    }
 
     for (i = 0; i < vdev->nr_vqs; i++) {
         VqInfo info = {
@@ -543,36 +325,6 @@ static void virtio_setup_ccw(VDev *vdev)
         "Could not write status to host");
 }
 
-void virtio_setup_device(SubChannelId schid)
-{
-    vdev.schid = schid;
-    virtio_setup_ccw(&vdev);
-
-    switch (vdev.senseid.cu_model) {
-    case VIRTIO_ID_BLOCK:
-        sclp_print("Using virtio-blk.\n");
-        if (!virtio_ipl_disk_is_valid()) {
-            /* make sure all getters but blocksize return 0 for
-             * invalid IPL disk
-             */
-            memset(&vdev.config.blk, 0, sizeof(vdev.config.blk));
-            virtio_assume_scsi();
-        }
-        break;
-    case VIRTIO_ID_SCSI:
-        IPL_assert(vdev.config.scsi.sense_size == VIRTIO_SCSI_SENSE_SIZE,
-            "Config: sense size mismatch");
-        IPL_assert(vdev.config.scsi.cdb_size == VIRTIO_SCSI_CDB_SIZE,
-            "Config: CDB size mismatch");
-
-        sclp_print("Using virtio-scsi.\n");
-        virtio_scsi_setup(&vdev);
-        break;
-    default:
-        panic("\n! No IPL device available !\n");
-    }
-}
-
 bool virtio_is_supported(SubChannelId schid)
 {
     vdev.schid = schid;

pc-bios/s390-ccw/virtio.h

@@ -11,8 +11,6 @@
 #ifndef VIRTIO_H
 #define VIRTIO_H
 
-#include "s390-ccw.h"
-
 /* Status byte for guest to report progress, and synchronize features. */
 /* We have seen device and processed generic fields (VIRTIO_CONFIG_F_VIRTIO) */
 #define VIRTIO_CONFIG_S_ACKNOWLEDGE     1
@@ -32,24 +30,6 @@ enum VirtioDevType {
 };
 typedef enum VirtioDevType VirtioDevType;
 
-struct VirtioDevHeader {
-    VirtioDevType type:8;
-    uint8_t num_vq;
-    uint8_t feature_len;
-    uint8_t config_len;
-    uint8_t status;
-    uint8_t vqconfig[];
-} __attribute__((packed));
-typedef struct VirtioDevHeader VirtioDevHeader;
-
-struct VirtioVqConfig {
-    uint64_t token;
-    uint64_t address;
-    uint16_t num;
-    uint8_t pad[6];
-} __attribute__((packed));
-typedef struct VirtioVqConfig VirtioVqConfig;
-
 struct VqInfo {
     uint64_t queue;
     uint32_t align;
@@ -64,15 +44,6 @@ struct VqConfig {
 } __attribute__((packed));
 typedef struct VqConfig VqConfig;
 
-struct VirtioDev {
-    VirtioDevHeader *header;
-    VirtioVqConfig *vqconfig;
-    char *host_features;
-    char *guest_features;
-    char *config;
-};
-typedef struct VirtioDev VirtioDev;
-
 #define VIRTIO_RING_SIZE            (PAGE_SIZE * 8)
 #define VIRTIO_MAX_VQS              3
 #define KVM_S390_VIRTIO_RING_ALIGN  4096
@@ -254,6 +225,13 @@ struct ScsiDevice {
 };
 typedef struct ScsiDevice ScsiDevice;
 
+struct VirtioNetConfig {
+    uint8_t mac[6];
+    /* uint16_t status; */               /* Only with VIRTIO_NET_F_STATUS */
+    /* uint16_t max_virtqueue_pairs; */  /* Only with VIRTIO_NET_F_MQ */
+};
+typedef struct VirtioNetConfig VirtioNetConfig;
+
 struct VDev {
     int nr_vqs;
     VRing *vrings;
@@ -266,6 +244,7 @@ struct VDev {
     union {
         VirtioBlkConfig blk;
         VirtioScsiConfig scsi;
+        VirtioNetConfig net;
     } config;
     ScsiDevice *scsi_device;
     bool is_cdrom;
@@ -278,6 +257,7 @@ struct VDev {
     ScsiDevice selected_scsi_device;
     uint64_t netboot_start_addr;
     uint32_t max_transfer;
+    uint32_t guest_features[2];
 };
 typedef struct VDev VDev;
 
@@ -291,6 +271,14 @@ struct VirtioCmd {
 };
 typedef struct VirtioCmd VirtioCmd;
 
+bool vring_notify(VRing *vr);
+int drain_irqs(SubChannelId schid);
+void vring_send_buf(VRing *vr, void *p, int len, int flags);
+int vr_poll(VRing *vr);
+int vring_wait_reply(void);
 int virtio_run(VDev *vdev, int vqid, VirtioCmd *cmd);
+void virtio_setup_ccw(VDev *vdev);
+
+int virtio_net_init(void *mac_addr);
 
 #endif /* VIRTIO_H */

roms/SLOF

@@ -1 +1 @@
-Subproject commit 66d250ef0fd06bb88b7399b9563b5008201f2d63
+Subproject commit 834113a1c67d6fb53dea153c3313d182238f2d36

target/s390x/arch_dump.c

@@ -57,6 +57,12 @@ struct S390xElfVregsHiStruct {
 
 typedef struct S390xElfVregsHiStruct S390xElfVregsHi;
 
+struct S390xElfGSCBStruct {
+    uint64_t gsregs[4];
+} QEMU_PACKED;
+
+typedef struct S390xElfGSCBStruct S390xElfGSCB;
+
 typedef struct noteStruct {
     Elf64_Nhdr hdr;
     char name[8];
@@ -65,6 +71,7 @@ typedef struct noteStruct {
         S390xElfFpregset fpregset;
         S390xElfVregsLo vregslo;
         S390xElfVregsHi vregshi;
+        S390xElfGSCB gscb;
         uint32_t prefix;
         uint64_t timer;
         uint64_t todcmp;
@@ -126,6 +133,16 @@ static void s390x_write_elf64_vregshi(Note *note, S390CPU *cpu, int id)
     }
 }
 
+static void s390x_write_elf64_gscb(Note *note, S390CPU *cpu, int id)
+{
+    int i;
+
+    note->hdr.n_type = cpu_to_be32(NT_S390_GS_CB);
+    for (i = 0; i < 4; i++) {
+        note->contents.gscb.gsregs[i] = cpu_to_be64(cpu->env.gscb[i]);
+    }
+}
+
 static void s390x_write_elf64_timer(Note *note, S390CPU *cpu, int id)
 {
     note->hdr.n_type = cpu_to_be32(NT_S390_TIMER);
@@ -181,6 +198,7 @@ static const NoteFuncDesc note_linux[] = {
     {sizeof(((Note *)0)->contents.todpreg),  s390x_write_elf64_todpreg},
     {sizeof(((Note *)0)->contents.vregslo),  s390x_write_elf64_vregslo},
     {sizeof(((Note *)0)->contents.vregshi),  s390x_write_elf64_vregshi},
+    {sizeof(((Note *)0)->contents.gscb),     s390x_write_elf64_gscb},
     { 0, NULL}
 };
 

target/s390x/cpu.h

@@ -89,6 +89,7 @@ typedef struct CPUS390XState {
     CPU_DoubleU vregs[32][2];  /* vector registers */
     uint32_t aregs[16];    /* access registers */
     uint8_t riccb[64];     /* runtime instrumentation control */
+    uint64_t gscb[4];      /* guarded storage control */
 
     /* Fields up to this point are not cleared by initial CPU reset */
     struct {} start_initial_reset_fields;
@@ -1158,6 +1159,7 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
 int kvm_s390_get_memslot_count(KVMState *s);
+int kvm_s390_cmma_active(void);
 void kvm_s390_cmma_reset(void);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_reset_vcpu(S390CPU *cpu);
@@ -1165,6 +1167,7 @@ int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
 int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
 int kvm_s390_get_ri(void);
+int kvm_s390_get_gs(void);
 void kvm_s390_crypto_reset(void);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
@@ -1219,6 +1222,10 @@ static inline int kvm_s390_get_ri(void)
 {
     return 0;
 }
+static inline int kvm_s390_get_gs(void)
+{
+    return 0;
+}
 static inline void kvm_s390_crypto_reset(void)
 {
 }
@@ -1327,6 +1334,7 @@ static inline bool s390_get_squash_mcss(void)
 #define MCIC_VB_CR 0x0000000400000000ULL
 #define MCIC_VB_ST 0x0000000100000000ULL
 #define MCIC_VB_AR 0x0000000040000000ULL
+#define MCIC_VB_GS 0x0000000008000000ULL
 #define MCIC_VB_PR 0x0000000000200000ULL
 #define MCIC_VB_FC 0x0000000000100000ULL
 #define MCIC_VB_CT 0x0000000000020000ULL

target/s390x/cpu_features.c

@@ -59,6 +59,7 @@ static const S390FeatDef s390_features[] = {
     FEAT_INIT("exrl", S390_FEAT_TYPE_STFL, 35, "Execute-extensions facility"),
     FEAT_INIT("emon", S390_FEAT_TYPE_STFL, 36, "Enhanced-monitor facility"),
     FEAT_INIT("fpe", S390_FEAT_TYPE_STFL, 37, "Floating-point extension facility"),
+    FEAT_INIT("opc", S390_FEAT_TYPE_STFL, 38, "Order Preserving Compression facility"),
     FEAT_INIT("sprogp", S390_FEAT_TYPE_STFL, 40, "Set-program-parameters facility"),
     FEAT_INIT("fpseh", S390_FEAT_TYPE_STFL, 41, "Floating-point-support-enhancement facilities"),
     FEAT_INIT("dfp", S390_FEAT_TYPE_STFL, 42, "DFP (decimal-floating-point) facility"),
@@ -72,8 +73,15 @@ static const S390FeatDef s390_features[] = {
     FEAT_INIT("ltlbc", S390_FEAT_TYPE_STFL, 51, "Local-TLB-clearing facility"),
     FEAT_INIT("iacc2", S390_FEAT_TYPE_STFL, 52, "Interlocked-access facility 2"),
     FEAT_INIT("stfle53", S390_FEAT_TYPE_STFL, 53, "Various facilities introduced with z13"),
+    FEAT_INIT("eec", S390_FEAT_TYPE_STFL, 54, "Entropy encoding compression facility"),
     FEAT_INIT("msa5-base", S390_FEAT_TYPE_STFL, 57, "Message-security-assist-extension-5 facility (excluding subfunctions)"),
+    FEAT_INIT("minste2", S390_FEAT_TYPE_STFL, 58, "Miscellaneous-instruction-extensions facility 2"),
+    FEAT_INIT("sema", S390_FEAT_TYPE_STFL, 59, "Semaphore-assist facility"),
+    FEAT_INIT("tsi", S390_FEAT_TYPE_STFL, 60, "Time-slice Instrumentation facility"),
     FEAT_INIT("ri", S390_FEAT_TYPE_STFL, 64, "CPU runtime-instrumentation facility"),
+    FEAT_INIT("zpci", S390_FEAT_TYPE_STFL, 69, "z/PCI facility"),
+    FEAT_INIT("aen", S390_FEAT_TYPE_STFL, 71, "General-purpose-adapter-event-notification facility"),
+    FEAT_INIT("ais", S390_FEAT_TYPE_STFL, 72, "General-purpose-adapter-interruption-suppression facility"),
     FEAT_INIT("te", S390_FEAT_TYPE_STFL, 73, "Transactional-execution facility"),
     FEAT_INIT("sthyi", S390_FEAT_TYPE_STFL, 74, "Store-hypervisor-information facility"),
     FEAT_INIT("aefsi", S390_FEAT_TYPE_STFL, 75, "Access-exception-fetch/store-indication facility"),
@@ -82,10 +90,24 @@ static const S390FeatDef s390_features[] = {
     FEAT_INIT("edat2", S390_FEAT_TYPE_STFL, 78, "Enhanced-DAT facility 2"),
     FEAT_INIT("dfppc", S390_FEAT_TYPE_STFL, 80, "Decimal-floating-point packed-conversion facility"),
     FEAT_INIT("vx", S390_FEAT_TYPE_STFL, 129, "Vector facility"),
+    FEAT_INIT("iep", S390_FEAT_TYPE_STFL, 130, "Instruction-execution-protection facility"),
+    FEAT_INIT("sea_esop2", S390_FEAT_TYPE_STFL, 131, "Side-effect-access facility and Enhanced-suppression-on-protection facility 2"),
+    FEAT_INIT("gs", S390_FEAT_TYPE_STFL, 133, "Guarded-storage facility"),
+    FEAT_INIT("vxpd", S390_FEAT_TYPE_STFL, 134, "Vector packed decimal facility"),
+    FEAT_INIT("vxeh", S390_FEAT_TYPE_STFL, 135, "Vector enhancements facility"),
+    FEAT_INIT("mepoch", S390_FEAT_TYPE_STFL, 139, "Multiple-epoch facility"),
+    FEAT_INIT("tpei", S390_FEAT_TYPE_STFL, 144, "Test-pending-external-interruption facility"),
+    FEAT_INIT("irbm", S390_FEAT_TYPE_STFL, 145, "Insert-reference-bits-multiple facility"),
+    FEAT_INIT("msa8-base", S390_FEAT_TYPE_STFL, 146, "Message-security-assist-extension-8 facility (excluding subfunctions)"),
+    FEAT_INIT("cmmnt", S390_FEAT_TYPE_STFL, 147, "CMM: ESSA-enhancement (no translate) facility"),
 
+    /* SCLP SCCB Byte 80 - 98  (bit numbers relative to byte-80) */
     FEAT_INIT("gsls", S390_FEAT_TYPE_SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility"),
     FEAT_INIT("esop", S390_FEAT_TYPE_SCLP_CONF_CHAR, 46, "Enhanced-suppression-on-protection facility"),
+    FEAT_INIT("hpma2", S390_FEAT_TYPE_SCLP_CONF_CHAR, 90, "Host page management assist 2 Facility"), /* 91-2 */
+    FEAT_INIT("kss", S390_FEAT_TYPE_SCLP_CONF_CHAR, 151, "SIE: Keyless-subset facility"),  /* 98-7 */
 
+    /* SCLP SCCB Byte 116 - 119 (bit numbers relative to byte-116) */
     FEAT_INIT("64bscao", S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT, 0, "SIE: 64-bit-SCAO facility"),
     FEAT_INIT("cmma", S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT, 1, "SIE: Collaborative-memory-management assist"),
     FEAT_INIT("pfmfi", S390_FEAT_TYPE_SCLP_CONF_CHAR_EXT, 9, "SIE: PFMF interpretation facility"),
@@ -182,10 +204,23 @@ static const S390FeatDef s390_features[] = {
     FEAT_INIT("kimd-sha-1", S390_FEAT_TYPE_KIMD, 1, "KIMD SHA-1"),
     FEAT_INIT("kimd-sha-256", S390_FEAT_TYPE_KIMD, 2, "KIMD SHA-256"),
     FEAT_INIT("kimd-sha-512", S390_FEAT_TYPE_KIMD, 3, "KIMD SHA-512"),
+    FEAT_INIT("kimd-sha3-224", S390_FEAT_TYPE_KIMD, 32, "KIMD SHA3-224"),
+    FEAT_INIT("kimd-sha3-256", S390_FEAT_TYPE_KIMD, 33, "KIMD SHA3-256"),
+    FEAT_INIT("kimd-sha3-384", S390_FEAT_TYPE_KIMD, 34, "KIMD SHA3-384"),
+    FEAT_INIT("kimd-sha3-512", S390_FEAT_TYPE_KIMD, 35, "KIMD SHA3-512"),
+    FEAT_INIT("kimd-shake-128", S390_FEAT_TYPE_KIMD, 36, "KIMD SHAKE-128"),
+    FEAT_INIT("kimd-shake-256", S390_FEAT_TYPE_KIMD, 37, "KIMD SHAKE-256"),
     FEAT_INIT("kimd-ghash", S390_FEAT_TYPE_KIMD, 65, "KIMD GHASH"),
+
     FEAT_INIT("klmd-sha-1", S390_FEAT_TYPE_KLMD, 1, "KLMD SHA-1"),
     FEAT_INIT("klmd-sha-256", S390_FEAT_TYPE_KLMD, 2, "KLMD SHA-256"),
     FEAT_INIT("klmd-sha-512", S390_FEAT_TYPE_KLMD, 3, "KLMD SHA-512"),
+    FEAT_INIT("klmd-sha3-224", S390_FEAT_TYPE_KLMD, 32, "KLMD SHA3-224"),
+    FEAT_INIT("klmd-sha3-256", S390_FEAT_TYPE_KLMD, 33, "KLMD SHA3-256"),
+    FEAT_INIT("klmd-sha3-384", S390_FEAT_TYPE_KLMD, 34, "KLMD SHA3-384"),
+    FEAT_INIT("klmd-sha3-512", S390_FEAT_TYPE_KLMD, 35, "KLMD SHA3-512"),
+    FEAT_INIT("klmd-shake-128", S390_FEAT_TYPE_KLMD, 36, "KLMD SHAKE-128"),
+    FEAT_INIT("klmd-shake-256", S390_FEAT_TYPE_KLMD, 37, "KLMD SHAKE-256"),
 
     FEAT_INIT("pckmo-edea", S390_FEAT_TYPE_PCKMO, 1, "PCKMO Encrypted-DEA-Key"),
     FEAT_INIT("pckmo-etdea-128", S390_FEAT_TYPE_PCKMO, 2, "PCKMO Encrypted-TDEA-128-Key"),
@@ -251,6 +286,15 @@ static const S390FeatDef s390_features[] = {
     FEAT_INIT("pcc-xts-eaes-256", S390_FEAT_TYPE_PCC, 60, "PCC Compute-XTS-Parameter-Using-Encrypted-AES-256"),
 
     FEAT_INIT("ppno-sha-512-drng", S390_FEAT_TYPE_PPNO, 3, "PPNO SHA-512-DRNG"),
+    FEAT_INIT("prno-trng-qrtcr", S390_FEAT_TYPE_PPNO, 112, "PRNO TRNG-Query-Raw-to-Conditioned-Ratio"),
+    FEAT_INIT("prno-trng", S390_FEAT_TYPE_PPNO, 114, "PRNO TRNG"),
+
+    FEAT_INIT("kma-gcm-aes-128", S390_FEAT_TYPE_KMA, 18, "KMA GCM-AES-128"),
+    FEAT_INIT("kma-gcm-aes-192", S390_FEAT_TYPE_KMA, 19, "KMA GCM-AES-192"),
+    FEAT_INIT("kma-gcm-aes-256", S390_FEAT_TYPE_KMA, 20, "KMA GCM-AES-256"),
+    FEAT_INIT("kma-gcm-eaes-128", S390_FEAT_TYPE_KMA, 26, "KMA GCM-Encrypted-AES-128"),
+    FEAT_INIT("kma-gcm-eaes-192", S390_FEAT_TYPE_KMA, 27, "KMA GCM-Encrypted-AES-192"),
+    FEAT_INIT("kma-gcm-eaes-256", S390_FEAT_TYPE_KMA, 28, "KMA GCM-Encrypted-AES-256"),
 };
 
 const S390FeatDef *s390_feat_def(S390Feat feat)
@@ -293,8 +337,9 @@ void s390_fill_feat_block(const S390FeatBitmap features, S390FeatType type,
     int bit_nr;
 
     if (type == S390_FEAT_TYPE_STFL && test_bit(S390_FEAT_ZARCH, features)) {
-        /* z/Architecture is always active if around */
+        /* Features that are always active */
-        data[0] |= 0x20;
+        data[0] |= 0x20;  /* z/Architecture */
+        data[17] |= 0x20; /* Configuration-z-architectural-mode */
     }
 
     feat = find_first_bit(features, S390_FEAT_MAX);
@@ -383,6 +428,9 @@ static S390FeatGroupDef s390_feature_groups[] = {
     FEAT_GROUP_INIT("msa3", MSA_EXT_3, "Message-security-assist-extension 3 facility"),
     FEAT_GROUP_INIT("msa4", MSA_EXT_4, "Message-security-assist-extension 4 facility"),
     FEAT_GROUP_INIT("msa5", MSA_EXT_5, "Message-security-assist-extension 5 facility"),
+    FEAT_GROUP_INIT("msa6", MSA_EXT_6, "Message-security-assist-extension 6 facility"),
+    FEAT_GROUP_INIT("msa7", MSA_EXT_7, "Message-security-assist-extension 7 facility"),
+    FEAT_GROUP_INIT("msa8", MSA_EXT_8, "Message-security-assist-extension 8 facility"),
 };
 
 const S390FeatGroupDef *s390_feat_group_def(S390FeatGroup group)

target/s390x/cpu_features.h

@@ -37,6 +37,7 @@ typedef enum {
     S390_FEAT_TYPE_KMO,
     S390_FEAT_TYPE_PCC,
     S390_FEAT_TYPE_PPNO,
+    S390_FEAT_TYPE_KMA,
 } S390FeatType;
 
 /* Definition of a CPU feature */
@@ -74,6 +75,9 @@ typedef enum {
     S390_FEAT_GROUP_MSA_EXT_3,
     S390_FEAT_GROUP_MSA_EXT_4,
     S390_FEAT_GROUP_MSA_EXT_5,
+    S390_FEAT_GROUP_MSA_EXT_6,
+    S390_FEAT_GROUP_MSA_EXT_7,
+    S390_FEAT_GROUP_MSA_EXT_8,
     S390_FEAT_GROUP_MAX,
 } S390FeatGroup;
 

target/s390x/cpu_features_def.h

@@ -15,6 +15,7 @@
 #define TARGET_S390X_CPU_FEATURES_DEF_H
 
 typedef enum {
+    /* Stfle */
     S390_FEAT_ESAN3 = 0,
     S390_FEAT_ZARCH,
     S390_FEAT_DAT_ENH,
@@ -49,6 +50,7 @@ typedef enum {
     S390_FEAT_EXECUTE_EXT,
     S390_FEAT_ENHANCED_MONITOR,
     S390_FEAT_FLOATING_POINT_EXT,
+    S390_FEAT_ORDER_PRESERVING_COMPRESSION,
     S390_FEAT_SET_PROGRAM_PARAMETERS,
     S390_FEAT_FLOATING_POINT_SUPPPORT_ENH,
     S390_FEAT_DFP,
@@ -62,8 +64,15 @@ typedef enum {
     S390_FEAT_LOCAL_TLB_CLEARING,
     S390_FEAT_INTERLOCKED_ACCESS_2,
     S390_FEAT_STFLE_53,
+    S390_FEAT_ENTROPY_ENC_COMP,
     S390_FEAT_MSA_EXT_5,
+    S390_FEAT_MISC_INSTRUCTION_EXT,
+    S390_FEAT_SEMAPHORE_ASSIST,
+    S390_FEAT_TIME_SLICE_INSTRUMENTATION,
     S390_FEAT_RUNTIME_INSTRUMENTATION,
+    S390_FEAT_ZPCI,
+    S390_FEAT_ADAPTER_EVENT_NOTIFICATION,
+    S390_FEAT_ADAPTER_INT_SUPPRESSION,
     S390_FEAT_TRANSACTIONAL_EXE,
     S390_FEAT_STORE_HYPERVISOR_INFO,
     S390_FEAT_ACCESS_EXCEPTION_FS_INDICATION,
@@ -72,12 +81,30 @@ typedef enum {
     S390_FEAT_EDAT_2,
     S390_FEAT_DFP_PACKED_CONVERSION,
     S390_FEAT_VECTOR,
+    S390_FEAT_INSTRUCTION_EXEC_PROT,
+    S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2,
+    S390_FEAT_GUARDED_STORAGE,
+    S390_FEAT_VECTOR_PACKED_DECIMAL,
+    S390_FEAT_VECTOR_ENH,
+    S390_FEAT_MULTIPLE_EPOCH,
+    S390_FEAT_TEST_PENDING_EXT_INTERRUPTION,
+    S390_FEAT_INSERT_REFERENCE_BITS_MULT,
+    S390_FEAT_MSA_EXT_8,
+    S390_FEAT_CMM_NT,
+
+    /* Sclp Conf Char */
     S390_FEAT_SIE_GSLS,
     S390_FEAT_ESOP,
+    S390_FEAT_HPMA2,
+    S390_FEAT_SIE_KSS,
+
+    /* Sclp Conf Char Ext */
     S390_FEAT_SIE_64BSCAO,
     S390_FEAT_SIE_CMMA,
     S390_FEAT_SIE_PFMFI,
     S390_FEAT_SIE_IBS,
+
+    /* Sclp Cpu */
     S390_FEAT_SIE_F2,
     S390_FEAT_SIE_SKEY,
     S390_FEAT_SIE_GPERE,
@@ -85,8 +112,12 @@ typedef enum {
     S390_FEAT_SIE_SIGPIF,
     S390_FEAT_SIE_IB,
     S390_FEAT_SIE_CEI,
+
+    /* Misc */
     S390_FEAT_DAT_ENH_2,
     S390_FEAT_CMM,
+
+    /* PLO */
     S390_FEAT_PLO_CL,
     S390_FEAT_PLO_CLG,
     S390_FEAT_PLO_CLGR,
@@ -111,6 +142,8 @@ typedef enum {
     S390_FEAT_PLO_CSTSTG,
     S390_FEAT_PLO_CSTSTGR,
     S390_FEAT_PLO_CSTSTX,
+
+    /* PTFF */
     S390_FEAT_PTFF_QTO,
     S390_FEAT_PTFF_QSI,
     S390_FEAT_PTFF_QPT,
@@ -118,6 +151,8 @@ typedef enum {
     S390_FEAT_PTFF_QTOU,
     S390_FEAT_PTFF_STO,
     S390_FEAT_PTFF_STOU,
+
+    /* KMAC */
     S390_FEAT_KMAC_DEA,
     S390_FEAT_KMAC_TDEA_128,
     S390_FEAT_KMAC_TDEA_192,
@@ -130,6 +165,8 @@ typedef enum {
     S390_FEAT_KMAC_EAES_128,
     S390_FEAT_KMAC_EAES_192,
     S390_FEAT_KMAC_EAES_256,
+
+    /* KMC */
     S390_FEAT_KMC_DEA,
     S390_FEAT_KMC_TDEA_128,
     S390_FEAT_KMC_TDEA_192,
@@ -143,6 +180,8 @@ typedef enum {
     S390_FEAT_KMC_EAES_192,
     S390_FEAT_KMC_EAES_256,
     S390_FEAT_KMC_PRNG,
+
+    /* KM */
     S390_FEAT_KM_DEA,
     S390_FEAT_KM_TDEA_128,
     S390_FEAT_KM_TDEA_192,
@@ -159,19 +198,39 @@ typedef enum {
     S390_FEAT_KM_XTS_AES_256,
     S390_FEAT_KM_XTS_EAES_128,
     S390_FEAT_KM_XTS_EAES_256,
+
+    /* KIMD */
     S390_FEAT_KIMD_SHA_1,
     S390_FEAT_KIMD_SHA_256,
     S390_FEAT_KIMD_SHA_512,
+    S390_FEAT_KIMD_SHA3_224,
+    S390_FEAT_KIMD_SHA3_256,
+    S390_FEAT_KIMD_SHA3_384,
+    S390_FEAT_KIMD_SHA3_512,
+    S390_FEAT_KIMD_SHAKE_128,
+    S390_FEAT_KIMD_SHAKE_256,
     S390_FEAT_KIMD_GHASH,
+
+    /* KLMD */
     S390_FEAT_KLMD_SHA_1,
     S390_FEAT_KLMD_SHA_256,
     S390_FEAT_KLMD_SHA_512,
+    S390_FEAT_KLMD_SHA3_224,
+    S390_FEAT_KLMD_SHA3_256,
+    S390_FEAT_KLMD_SHA3_384,
+    S390_FEAT_KLMD_SHA3_512,
+    S390_FEAT_KLMD_SHAKE_128,
+    S390_FEAT_KLMD_SHAKE_256,
+
+    /* PCKMO */
     S390_FEAT_PCKMO_EDEA,
     S390_FEAT_PCKMO_ETDEA_128,
     S390_FEAT_PCKMO_ETDEA_256,
     S390_FEAT_PCKMO_AES_128,
     S390_FEAT_PCKMO_AES_192,
     S390_FEAT_PCKMO_AES_256,
+
+    /* KMCTR */
     S390_FEAT_KMCTR_DEA,
     S390_FEAT_KMCTR_TDEA_128,
     S390_FEAT_KMCTR_TDEA_192,
@@ -184,6 +243,8 @@ typedef enum {
     S390_FEAT_KMCTR_EAES_128,
     S390_FEAT_KMCTR_EAES_192,
     S390_FEAT_KMCTR_EAES_256,
+
+    /* KMF */
     S390_FEAT_KMF_DEA,
     S390_FEAT_KMF_TDEA_128,
     S390_FEAT_KMF_TDEA_192,
@@ -196,6 +257,8 @@ typedef enum {
     S390_FEAT_KMF_EAES_128,
     S390_FEAT_KMF_EAES_192,
     S390_FEAT_KMF_EAES_256,
+
+    /* KMO */
     S390_FEAT_KMO_DEA,
     S390_FEAT_KMO_TDEA_128,
     S390_FEAT_KMO_TDEA_192,
@@ -208,6 +271,8 @@ typedef enum {
     S390_FEAT_KMO_EAES_128,
     S390_FEAT_KMO_EAES_192,
     S390_FEAT_KMO_EAES_256,
+
+    /* PCC */
     S390_FEAT_PCC_CMAC_DEA,
     S390_FEAT_PCC_CMAC_TDEA_128,
     S390_FEAT_PCC_CMAC_TDEA_192,
@@ -224,7 +289,19 @@ typedef enum {
     S390_FEAT_PCC_XTS_AES_256,
     S390_FEAT_PCC_XTS_EAES_128,
     S390_FEAT_PCC_XTS_EAES_256,
+
+    /* PPNO/PRNO */
     S390_FEAT_PPNO_SHA_512_DRNG,
+    S390_FEAT_PRNO_TRNG_QRTCR,
+    S390_FEAT_PRNO_TRNG,
+
+    /* KMA */
+    S390_FEAT_KMA_GCM_AES_128,
+    S390_FEAT_KMA_GCM_AES_192,
+    S390_FEAT_KMA_GCM_AES_256 ,
+    S390_FEAT_KMA_GCM_EAES_128,
+    S390_FEAT_KMA_GCM_EAES_192,
+    S390_FEAT_KMA_GCM_EAES_256,
     S390_FEAT_MAX,
 } S390Feat;
 

target/s390x/cpu_models.c

@@ -77,6 +77,32 @@ static S390CPUDef s390_cpu_defs[] = {
     CPUDEF_INIT(0x2965, 13, 2, 47, 0x08000000U, "z13s", "IBM z13s GA1"),
 };
 
+void s390_cpudef_featoff(uint8_t gen, uint8_t ec_ga, S390Feat feat)
+{
+    const S390CPUDef *def;
+
+    def = s390_find_cpu_def(0, gen, ec_ga, NULL);
+    clear_bit(feat, (unsigned long *)&def->default_feat);
+}
+
+void s390_cpudef_featoff_greater(uint8_t gen, uint8_t ec_ga, S390Feat feat)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(s390_cpu_defs); i++) {
+        const S390CPUDef *def = &s390_cpu_defs[i];
+
+        if (def->gen < gen) {
+            continue;
+        }
+        if (def->gen == gen && def->ec_ga < ec_ga) {
+            continue;
+        }
+
+        clear_bit(feat, (unsigned long *)&def->default_feat);
+    }
+}
+
 uint32_t s390_get_hmfai(void)
 {
     static S390CPU *cpu;
@@ -671,6 +697,31 @@ static void check_consistency(const S390CPUModel *model)
         { S390_FEAT_SIE_CMMA, S390_FEAT_CMM },
         { S390_FEAT_SIE_CMMA, S390_FEAT_SIE_GSLS },
         { S390_FEAT_SIE_PFMFI, S390_FEAT_EDAT },
+        { S390_FEAT_MSA_EXT_8, S390_FEAT_MSA_EXT_3 },
+        { S390_FEAT_MULTIPLE_EPOCH, S390_FEAT_TOD_CLOCK_STEERING },
+        { S390_FEAT_VECTOR_PACKED_DECIMAL, S390_FEAT_VECTOR },
+        { S390_FEAT_VECTOR_ENH, S390_FEAT_VECTOR },
+        { S390_FEAT_INSTRUCTION_EXEC_PROT, S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2 },
+        { S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2, S390_FEAT_ESOP },
+        { S390_FEAT_CMM_NT, S390_FEAT_CMM },
+        { S390_FEAT_GUARDED_STORAGE, S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2 },
+        { S390_FEAT_MULTIPLE_EPOCH, S390_FEAT_STORE_CLOCK_FAST },
+        { S390_FEAT_MULTIPLE_EPOCH, S390_FEAT_TOD_CLOCK_STEERING },
+        { S390_FEAT_SEMAPHORE_ASSIST, S390_FEAT_STFLE_49 },
+        { S390_FEAT_KIMD_SHA3_224, S390_FEAT_MSA },
+        { S390_FEAT_KIMD_SHA3_256, S390_FEAT_MSA },
+        { S390_FEAT_KIMD_SHA3_384, S390_FEAT_MSA },
+        { S390_FEAT_KIMD_SHA3_512, S390_FEAT_MSA },
+        { S390_FEAT_KIMD_SHAKE_128, S390_FEAT_MSA },
+        { S390_FEAT_KIMD_SHAKE_256, S390_FEAT_MSA },
+        { S390_FEAT_KLMD_SHA3_224, S390_FEAT_MSA },
+        { S390_FEAT_KLMD_SHA3_256, S390_FEAT_MSA },
+        { S390_FEAT_KLMD_SHA3_384, S390_FEAT_MSA },
+        { S390_FEAT_KLMD_SHA3_512, S390_FEAT_MSA },
+        { S390_FEAT_KLMD_SHAKE_128, S390_FEAT_MSA },
+        { S390_FEAT_KLMD_SHAKE_256, S390_FEAT_MSA },
+        { S390_FEAT_PRNO_TRNG_QRTCR, S390_FEAT_MSA_EXT_5 },
+        { S390_FEAT_PRNO_TRNG, S390_FEAT_MSA_EXT_5 },
     };
     int i;
 

target/s390x/cpu_models.h

@@ -72,6 +72,8 @@ typedef struct S390CPUModel {
 #define ibc_gen(x)        (x == 0 ? 0 : ((x >> 4) + S390_GEN_Z10))
 #define ibc_ec_ga(x)      (x & 0xf)
 
+void s390_cpudef_featoff(uint8_t gen, uint8_t ec_ga, S390Feat feat);
+void s390_cpudef_featoff_greater(uint8_t gen, uint8_t ec_ga, S390Feat feat);
 uint32_t s390_get_hmfai(void);
 uint8_t s390_get_mha_pow(void);
 uint32_t s390_get_ibc_val(void);

target/s390x/gdbstub.c

@@ -286,6 +286,26 @@ static int cpu_write_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
 }
 #endif
 
+/* the values represent the positions in s390-gs.xml */
+#define S390_GS_RESERVED_REGNUM 0
+#define S390_GS_GSD_REGNUM      1
+#define S390_GS_GSSM_REGNUM     2
+#define S390_GS_GSEPLA_REGNUM   3
+/* total number of registers in s390-gs.xml */
+#define S390_NUM_GS_REGS 4
+
+static int cpu_read_gs_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    return gdb_get_regl(mem_buf, env->gscb[n]);
+}
+
+static int cpu_write_gs_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    env->gscb[n] = ldtul_p(mem_buf);
+    cpu_synchronize_post_init(ENV_GET_CPU(env));
+    return 8;
+}
+
 void s390_cpu_gdb_init(CPUState *cs)
 {
     gdb_register_coprocessor(cs, cpu_read_ac_reg,
@@ -300,6 +320,10 @@ void s390_cpu_gdb_init(CPUState *cs)
                              cpu_write_vreg,
                              S390_NUM_VREGS, "s390-vx.xml", 0);
 
+    gdb_register_coprocessor(cs, cpu_read_gs_reg,
+                             cpu_write_gs_reg,
+                             S390_NUM_GS_REGS, "s390-gs.xml", 0);
+
 #ifndef CONFIG_USER_ONLY
     gdb_register_coprocessor(cs, cpu_read_c_reg,
                              cpu_write_c_reg,

target/s390x/gen-features.c

@@ -182,6 +182,33 @@
     S390_FEAT_MSA_EXT_5, \
     S390_FEAT_PPNO_SHA_512_DRNG
 
+#define S390_FEAT_GROUP_MSA_EXT_6 \
+    S390_FEAT_KIMD_SHA3_224, \
+    S390_FEAT_KIMD_SHA3_256, \
+    S390_FEAT_KIMD_SHA3_384, \
+    S390_FEAT_KIMD_SHA3_512, \
+    S390_FEAT_KIMD_SHAKE_128, \
+    S390_FEAT_KIMD_SHAKE_256, \
+    S390_FEAT_KLMD_SHA3_224, \
+    S390_FEAT_KLMD_SHA3_256, \
+    S390_FEAT_KLMD_SHA3_384, \
+    S390_FEAT_KLMD_SHA3_512, \
+    S390_FEAT_KLMD_SHAKE_128, \
+    S390_FEAT_KLMD_SHAKE_256
+
+#define S390_FEAT_GROUP_MSA_EXT_7 \
+    S390_FEAT_PRNO_TRNG_QRTCR, \
+    S390_FEAT_PRNO_TRNG
+
+#define S390_FEAT_GROUP_MSA_EXT_8 \
+    S390_FEAT_MSA_EXT_8, \
+    S390_FEAT_KMA_GCM_AES_128, \
+    S390_FEAT_KMA_GCM_AES_192, \
+    S390_FEAT_KMA_GCM_AES_256 , \
+    S390_FEAT_KMA_GCM_EAES_128, \
+    S390_FEAT_KMA_GCM_EAES_192, \
+    S390_FEAT_KMA_GCM_EAES_256
+
 /* cpu feature groups */
 static uint16_t group_PLO[] = {
     S390_FEAT_GROUP_PLO,
@@ -210,15 +237,30 @@ static uint16_t group_MSA_EXT_4[] = {
 static uint16_t group_MSA_EXT_5[] = {
     S390_FEAT_GROUP_MSA_EXT_5,
 };
+static uint16_t group_MSA_EXT_6[] = {
+    S390_FEAT_GROUP_MSA_EXT_6,
+};
+static uint16_t group_MSA_EXT_7[] = {
+    S390_FEAT_GROUP_MSA_EXT_7,
+};
+static uint16_t group_MSA_EXT_8[] = {
+    S390_FEAT_GROUP_MSA_EXT_8,
+};
 
-/* base features in order of release */
+/* Base features (in order of release)
+ * Only non-hypervisor managed features belong here.
+ * Base feature sets are static meaning they do not change in future QEMU
+ * releases.
+ */
 static uint16_t base_GEN7_GA1[] = {
     S390_FEAT_GROUP_PLO,
     S390_FEAT_ESAN3,
     S390_FEAT_ZARCH,
 };
+
 #define base_GEN7_GA2 EmptyFeat
 #define base_GEN7_GA3 EmptyFeat
+
 static uint16_t base_GEN8_GA1[] = {
     S390_FEAT_DAT_ENH,
     S390_FEAT_EXTENDED_TRANSLATION_2,
@@ -227,10 +269,12 @@ static uint16_t base_GEN8_GA1[] = {
     S390_FEAT_LONG_DISPLACEMENT_FAST,
     S390_FEAT_HFP_MADDSUB,
 };
+
 #define base_GEN8_GA2 EmptyFeat
 #define base_GEN8_GA3 EmptyFeat
 #define base_GEN8_GA4 EmptyFeat
 #define base_GEN8_GA5 EmptyFeat
+
 static uint16_t base_GEN9_GA1[] = {
     S390_FEAT_IDTE_SEGMENT,
     S390_FEAT_ASN_LX_REUSE,
@@ -245,8 +289,10 @@ static uint16_t base_GEN9_GA1[] = {
     S390_FEAT_ETF3_ENH,
     S390_FEAT_DAT_ENH_2,
 };
+
 #define base_GEN9_GA2 EmptyFeat
 #define base_GEN9_GA3 EmptyFeat
+
 static uint16_t base_GEN10_GA1[] = {
     S390_FEAT_CONDITIONAL_SSKE,
     S390_FEAT_PARSING_ENH,
@@ -263,6 +309,7 @@ static uint16_t base_GEN10_GA1[] = {
 };
 #define base_GEN10_GA2 EmptyFeat
 #define base_GEN10_GA3 EmptyFeat
+
 static uint16_t base_GEN11_GA1[] = {
     S390_FEAT_NONQ_KEY_SETTING,
     S390_FEAT_ENHANCED_MONITOR,
@@ -272,21 +319,30 @@ static uint16_t base_GEN11_GA1[] = {
     S390_FEAT_CMPSC_ENH,
     S390_FEAT_INTERLOCKED_ACCESS_2,
 };
+
 #define base_GEN11_GA2 EmptyFeat
+
 static uint16_t base_GEN12_GA1[] = {
     S390_FEAT_DFP_ZONED_CONVERSION,
     S390_FEAT_STFLE_49,
     S390_FEAT_LOCAL_TLB_CLEARING,
 };
+
 #define base_GEN12_GA2 EmptyFeat
+
 static uint16_t base_GEN13_GA1[] = {
     S390_FEAT_STFLE_53,
     S390_FEAT_DFP_PACKED_CONVERSION,
     S390_FEAT_GROUP_GEN13_PTFF,
 };
+
 #define base_GEN13_GA2 EmptyFeat
 
-/* full features differing to the base in order of release */
+/* Full features (in order of release)
+ * Automatically includes corresponding base features.
+ * Full features are all features this hardware supports even if kvm/QEMU do not
+ * support these features yet.
+ */
 static uint16_t full_GEN7_GA1[] = {
     S390_FEAT_SIE_F2,
     S390_FEAT_SIE_SKEY,
@@ -294,30 +350,38 @@ static uint16_t full_GEN7_GA1[] = {
     S390_FEAT_SIE_IB,
     S390_FEAT_SIE_CEI,
 };
+
 static uint16_t full_GEN7_GA2[] = {
     S390_FEAT_EXTENDED_TRANSLATION_2,
 };
+
 static uint16_t full_GEN7_GA3[] = {
     S390_FEAT_LONG_DISPLACEMENT,
     S390_FEAT_SIE_SIIF,
 };
+
 static uint16_t full_GEN8_GA1[] = {
     S390_FEAT_SIE_GSLS,
     S390_FEAT_SIE_64BSCAO,
 };
+
 #define full_GEN8_GA2 EmptyFeat
+
 static uint16_t full_GEN8_GA3[] = {
     S390_FEAT_ASN_LX_REUSE,
     S390_FEAT_EXTENDED_TRANSLATION_3,
 };
+
 #define full_GEN8_GA4 EmptyFeat
 #define full_GEN8_GA5 EmptyFeat
+
 static uint16_t full_GEN9_GA1[] = {
     S390_FEAT_STORE_HYPERVISOR_INFO,
     S390_FEAT_GROUP_MSA_EXT_1,
     S390_FEAT_CMM,
     S390_FEAT_SIE_CMMA,
 };
+
 static uint16_t full_GEN9_GA2[] = {
     S390_FEAT_MOVE_WITH_OPTIONAL_SPEC,
     S390_FEAT_EXTRACT_CPU_TIME,
@@ -325,10 +389,12 @@ static uint16_t full_GEN9_GA2[] = {
     S390_FEAT_FLOATING_POINT_SUPPPORT_ENH,
     S390_FEAT_DFP,
 };
+
 static uint16_t full_GEN9_GA3[] = {
     S390_FEAT_CONDITIONAL_SSKE,
     S390_FEAT_PFPO,
 };
+
 static uint16_t full_GEN10_GA1[] = {
     S390_FEAT_EDAT,
     S390_FEAT_CONFIGURATION_TOPOLOGY,
@@ -337,34 +403,50 @@ static uint16_t full_GEN10_GA1[] = {
     S390_FEAT_SIE_PFMFI,
     S390_FEAT_SIE_SIGPIF,
 };
+
 static uint16_t full_GEN10_GA2[] = {
     S390_FEAT_SET_PROGRAM_PARAMETERS,
     S390_FEAT_SIE_IBS,
 };
+
 static uint16_t full_GEN10_GA3[] = {
     S390_FEAT_GROUP_MSA_EXT_3,
 };
+
 static uint16_t full_GEN11_GA1[] = {
     S390_FEAT_IPTE_RANGE,
     S390_FEAT_ACCESS_EXCEPTION_FS_INDICATION,
     S390_FEAT_GROUP_MSA_EXT_4,
 };
+
 #define full_GEN11_GA2 EmptyFeat
+
 static uint16_t full_GEN12_GA1[] = {
     S390_FEAT_CONSTRAINT_TRANSACTIONAL_EXE,
     S390_FEAT_TRANSACTIONAL_EXE,
     S390_FEAT_RUNTIME_INSTRUMENTATION,
+    S390_FEAT_ZPCI,
+    S390_FEAT_ADAPTER_EVENT_NOTIFICATION,
+    S390_FEAT_ADAPTER_INT_SUPPRESSION,
     S390_FEAT_EDAT_2,
+    S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2,
 };
+
 static uint16_t full_GEN12_GA2[] = {
     S390_FEAT_GROUP_MSA_EXT_5,
 };
+
 static uint16_t full_GEN13_GA1[] = {
     S390_FEAT_VECTOR,
 };
+
 #define full_GEN13_GA2 EmptyFeat
 
-/* default features differing to the base in order of release */
+/* Default features (in order of release)
+ * Automatically includes corresponding base features.
+ * Default features are all features this version of QEMU supports for this
+ * hardware model. Default feature sets can grow with new QEMU releases.
+ */
 #define default_GEN7_GA1 EmptyFeat
 #define default_GEN7_GA2 EmptyFeat
 #define default_GEN7_GA3 EmptyFeat
@@ -373,37 +455,51 @@ static uint16_t full_GEN13_GA1[] = {
 #define default_GEN8_GA3 EmptyFeat
 #define default_GEN8_GA4 EmptyFeat
 #define default_GEN8_GA5 EmptyFeat
+
 static uint16_t default_GEN9_GA1[] = {
     S390_FEAT_STORE_HYPERVISOR_INFO,
     S390_FEAT_GROUP_MSA_EXT_1,
     S390_FEAT_CMM,
 };
+
 #define default_GEN9_GA2 EmptyFeat
 #define default_GEN9_GA3 EmptyFeat
+
 static uint16_t default_GEN10_GA1[] = {
     S390_FEAT_EDAT,
     S390_FEAT_GROUP_MSA_EXT_2,
 };
+
 #define default_GEN10_GA2 EmptyFeat
 #define default_GEN10_GA3 EmptyFeat
+
 static uint16_t default_GEN11_GA1[] = {
     S390_FEAT_GROUP_MSA_EXT_3,
     S390_FEAT_IPTE_RANGE,
     S390_FEAT_ACCESS_EXCEPTION_FS_INDICATION,
     S390_FEAT_GROUP_MSA_EXT_4,
 };
+
 #define default_GEN11_GA2 EmptyFeat
+
 static uint16_t default_GEN12_GA1[] = {
     S390_FEAT_CONSTRAINT_TRANSACTIONAL_EXE,
     S390_FEAT_TRANSACTIONAL_EXE,
     S390_FEAT_RUNTIME_INSTRUMENTATION,
+    S390_FEAT_ZPCI,
+    S390_FEAT_ADAPTER_EVENT_NOTIFICATION,
     S390_FEAT_EDAT_2,
+    S390_FEAT_ESOP,
+    S390_FEAT_SIDE_EFFECT_ACCESS_ESOP2,
 };
+
 #define default_GEN12_GA2 EmptyFeat
+
 static uint16_t default_GEN13_GA1[] = {
     S390_FEAT_GROUP_MSA_EXT_5,
     S390_FEAT_VECTOR,
 };
+
 #define default_GEN13_GA2 EmptyFeat
 
 /****** END FEATURE DEFS ******/
@@ -491,6 +587,9 @@ static FeatGroupDefSpec FeatGroupDef[] = {
     FEAT_GROUP_INITIALIZER(MSA_EXT_3),
     FEAT_GROUP_INITIALIZER(MSA_EXT_4),
     FEAT_GROUP_INITIALIZER(MSA_EXT_5),
+    FEAT_GROUP_INITIALIZER(MSA_EXT_6),
+    FEAT_GROUP_INITIALIZER(MSA_EXT_7),
+    FEAT_GROUP_INITIALIZER(MSA_EXT_8),
 };
 
 static void set_bits(uint64_t list[], BitSpec bits)

target/s390x/kvm.c

@@ -139,6 +139,9 @@ static int cap_async_pf;
 static int cap_mem_op;
 static int cap_s390_irq;
 static int cap_ri;
+static int cap_gs;
+
+static int active_cmma;
 
 static void *legacy_s390_alloc(size_t size, uint64_t *align);
 
@@ -177,6 +180,11 @@ int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit)
     return kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
 }
 
+int kvm_s390_cmma_active(void)
+{
+    return active_cmma;
+}
+
 static bool kvm_s390_cmma_available(void)
 {
     static bool initialized, value;
@@ -197,7 +205,7 @@ void kvm_s390_cmma_reset(void)
         .attr = KVM_S390_VM_MEM_CLR_CMMA,
     };
 
-    if (mem_path || !kvm_s390_cmma_available()) {
+    if (!kvm_s390_cmma_active()) {
         return;
     }
 
@@ -213,7 +221,13 @@ static void kvm_s390_enable_cmma(void)
         .attr = KVM_S390_VM_MEM_ENABLE_CMMA,
     };
 
+    if (mem_path) {
+        error_report("Warning: CMM will not be enabled because it is not "
+                     "compatible to hugetlbfs.");
+        return;
+    }
     rc = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+    active_cmma = !rc;
     trace_kvm_enable_cmma(rc);
 }
 
@@ -288,6 +302,14 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
             cap_ri = 1;
         }
     }
+    if (gs_allowed()) {
+        if (kvm_vm_enable_cap(s, KVM_CAP_S390_GS, 0) == 0) {
+            cap_gs = 1;
+        }
+    }
+
+    /* Try to enable AIS facility */
+    kvm_vm_enable_cap(s, KVM_CAP_S390_AIS, 0);
 
     qemu_mutex_init(&qemu_sigp_mutex);
 
@@ -456,6 +478,11 @@ int kvm_arch_put_registers(CPUState *cs, int level)
         }
     }
 
+    if (can_sync_regs(cs, KVM_SYNC_GSCB)) {
+        memcpy(cs->kvm_run->s.regs.gscb, env->gscb, 32);
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GSCB;
+    }
+
     /* Finally the prefix */
     if (can_sync_regs(cs, KVM_SYNC_PREFIX)) {
         cs->kvm_run->s.regs.prefix = env->psa;
@@ -562,6 +589,10 @@ int kvm_arch_get_registers(CPUState *cs)
         memcpy(env->riccb, cs->kvm_run->s.regs.riccb, 64);
     }
 
+    if (can_sync_regs(cs, KVM_SYNC_GSCB)) {
+        memcpy(env->gscb, cs->kvm_run->s.regs.gscb, 32);
+    }
+
     /* pfault parameters */
     if (can_sync_regs(cs, KVM_SYNC_PFAULT)) {
         env->pfault_token = cs->kvm_run->s.regs.pft;
@@ -1193,7 +1224,21 @@ static int kvm_stpcifc_service_call(S390CPU *cpu, struct kvm_run *run)
 
 static int kvm_sic_service_call(S390CPU *cpu, struct kvm_run *run)
 {
-    /* NOOP */
+    CPUS390XState *env = &cpu->env;
+    uint8_t r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    uint8_t r3 = run->s390_sieic.ipa & 0x000f;
+    uint8_t isc;
+    uint16_t mode;
+    int r;
+
+    cpu_synchronize_state(CPU(cpu));
+    mode = env->regs[r1] & 0xffff;
+    isc = (env->regs[r3] >> 27) & 0x7;
+    r = css_do_sic(env, isc, mode);
+    if (r) {
+        enter_pgmcheck(cpu, -r);
+    }
+
     return 0;
 }
 
@@ -1444,22 +1489,28 @@ static void sigp_stop(CPUState *cs, run_on_cpu_data arg)
     si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
 }
 
-#define ADTL_SAVE_AREA_SIZE 1024
+#define ADTL_GS_OFFSET   1024 /* offset of GS data in adtl save area */
-static int kvm_s390_store_adtl_status(S390CPU *cpu, hwaddr addr)
+#define ADTL_GS_MIN_SIZE 2048 /* minimal size of adtl save area for GS */
+static int do_store_adtl_status(S390CPU *cpu, hwaddr addr, hwaddr len)
 {
+    hwaddr save = len;
     void *mem;
-    hwaddr len = ADTL_SAVE_AREA_SIZE;
 
-    mem = cpu_physical_memory_map(addr, &len, 1);
+    mem = cpu_physical_memory_map(addr, &save, 1);
     if (!mem) {
         return -EFAULT;
     }
-    if (len != ADTL_SAVE_AREA_SIZE) {
+    if (save != len) {
         cpu_physical_memory_unmap(mem, len, 1, 0);
         return -EFAULT;
     }
 
-    memcpy(mem, &cpu->env.vregs, 512);
+    if (s390_has_feat(S390_FEAT_VECTOR)) {
+        memcpy(mem, &cpu->env.vregs, 512);
+    }
+    if (s390_has_feat(S390_FEAT_GUARDED_STORAGE) && len >= ADTL_GS_MIN_SIZE) {
+        memcpy(mem + ADTL_GS_OFFSET, &cpu->env.gscb, 32);
+    }
 
     cpu_physical_memory_unmap(mem, len, 1, len);
 
@@ -1555,12 +1606,17 @@ static void sigp_store_status_at_address(CPUState *cs, run_on_cpu_data arg)
     si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
 }
 
+#define ADTL_SAVE_LC_MASK  0xfUL
 static void sigp_store_adtl_status(CPUState *cs, run_on_cpu_data arg)
 {
     S390CPU *cpu = S390_CPU(cs);
     SigpInfo *si = arg.host_ptr;
+    uint8_t lc = si->param & ADTL_SAVE_LC_MASK;
+    hwaddr addr = si->param & ~ADTL_SAVE_LC_MASK;
+    hwaddr len = 1UL << (lc ? lc : 10);
 
-    if (!s390_has_feat(S390_FEAT_VECTOR)) {
+    if (!s390_has_feat(S390_FEAT_VECTOR) &&
+        !s390_has_feat(S390_FEAT_GUARDED_STORAGE)) {
         set_sigp_status(si, SIGP_STAT_INVALID_ORDER);
         return;
     }
@@ -1571,15 +1627,32 @@ static void sigp_store_adtl_status(CPUState *cs, run_on_cpu_data arg)
         return;
     }
 
-    /* parameter must be aligned to 1024-byte boundary */
+    /* address must be aligned to length */
-    if (si->param & 0x3ff) {
+    if (addr & (len - 1)) {
+        set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
+        return;
+    }
+
+    /* no GS: only lc == 0 is valid */
+    if (!s390_has_feat(S390_FEAT_GUARDED_STORAGE) &&
+        lc != 0) {
+        set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
+        return;
+    }
+
+    /* GS: 0, 10, 11, 12 are valid */
+    if (s390_has_feat(S390_FEAT_GUARDED_STORAGE) &&
+        lc != 0 &&
+        lc != 10 &&
+        lc != 11 &&
+        lc != 12) {
         set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
         return;
     }
 
     cpu_synchronize_state(cs);
 
-    if (kvm_s390_store_adtl_status(cpu, si->param)) {
+    if (do_store_adtl_status(cpu, addr, len)) {
         set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
         return;
     }
@@ -1727,41 +1800,25 @@ static int sigp_set_architecture(S390CPU *cpu, uint32_t param,
 {
     CPUState *cur_cs;
     S390CPU *cur_cpu;
+    bool all_stopped = true;
 
-    /* due to the BQL, we are the only active cpu */
     CPU_FOREACH(cur_cs) {
         cur_cpu = S390_CPU(cur_cs);
-        if (cur_cpu->env.sigp_order != 0) {
+
-            return SIGP_CC_BUSY;
+        if (cur_cpu == cpu) {
+            continue;
         }
-        cpu_synchronize_state(cur_cs);
+        if (s390_cpu_get_state(cur_cpu) != CPU_STATE_STOPPED) {
-        /* all but the current one have to be stopped */
+            all_stopped = false;
-        if (cur_cpu != cpu &&
-            s390_cpu_get_state(cur_cpu) != CPU_STATE_STOPPED) {
-            *status_reg &= 0xffffffff00000000ULL;
-            *status_reg |= SIGP_STAT_INCORRECT_STATE;
-            return SIGP_CC_STATUS_STORED;
         }
     }
 
-    switch (param & 0xff) {
+    *status_reg &= 0xffffffff00000000ULL;
-    case SIGP_MODE_ESA_S390:
-        /* not supported */
-        return SIGP_CC_NOT_OPERATIONAL;
-    case SIGP_MODE_Z_ARCH_TRANS_ALL_PSW:
-    case SIGP_MODE_Z_ARCH_TRANS_CUR_PSW:
-        CPU_FOREACH(cur_cs) {
-            cur_cpu = S390_CPU(cur_cs);
-            cur_cpu->env.pfault_token = -1UL;
-        }
-        break;
-    default:
-        *status_reg &= 0xffffffff00000000ULL;
-        *status_reg |= SIGP_STAT_INVALID_PARAMETER;
-        return SIGP_CC_STATUS_STORED;
-    }
 
-    return SIGP_CC_ORDER_CODE_ACCEPTED;
+    /* Reject set arch order, with czam we're always in z/Arch mode. */
+    *status_reg |= (all_stopped ? SIGP_STAT_INVALID_PARAMETER :
+                    SIGP_STAT_INCORRECT_STATE);
+    return SIGP_CC_STATUS_STORED;
 }
 
 static int handle_sigp(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
@@ -2174,6 +2231,9 @@ static uint64_t build_channel_report_mcic(void)
     if (s390_has_feat(S390_FEAT_VECTOR)) {
         mcic |= MCIC_VB_VR;
     }
+    if (s390_has_feat(S390_FEAT_GUARDED_STORAGE)) {
+        mcic |= MCIC_VB_GS;
+    }
     return mcic;
 }
 
@@ -2239,6 +2299,11 @@ int kvm_s390_get_ri(void)
     return cap_ri;
 }
 
+int kvm_s390_get_gs(void)
+{
+    return cap_gs;
+}
+
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state)
 {
     struct kvm_mp_state mp_state = {};
@@ -2417,6 +2482,9 @@ static int query_cpu_subfunc(S390FeatBitmap features)
     if (test_bit(S390_FEAT_MSA_EXT_5, features)) {
         s390_add_from_feat_block(features, S390_FEAT_TYPE_PPNO, prop.ppno);
     }
+    if (test_bit(S390_FEAT_MSA_EXT_8, features)) {
+        s390_add_from_feat_block(features, S390_FEAT_TYPE_KMA, prop.kma);
+    }
     return 0;
 }
 
@@ -2470,6 +2538,10 @@ static int configure_cpu_subfunc(const S390FeatBitmap features)
         s390_fill_feat_block(features, S390_FEAT_TYPE_PPNO, prop.ppno);
         prop.ppno[0] |= 0x80; /* query is always available */
     }
+    if (test_bit(S390_FEAT_MSA_EXT_8, features)) {
+        s390_fill_feat_block(features, S390_FEAT_TYPE_KMA, prop.kma);
+        prop.kma[0] |= 0x80; /* query is always available */
+    }
     return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
 }
 
@@ -2487,6 +2559,7 @@ static int kvm_to_feat[][2] = {
     { KVM_S390_VM_CPU_FEAT_CMMA, S390_FEAT_SIE_CMMA },
     { KVM_S390_VM_CPU_FEAT_PFMFI, S390_FEAT_SIE_PFMFI},
     { KVM_S390_VM_CPU_FEAT_SIGPIF, S390_FEAT_SIE_SIGPIF},
+    { KVM_S390_VM_CPU_FEAT_KSS, S390_FEAT_SIE_KSS},
 };
 
 static int query_cpu_feat(S390FeatBitmap features)
@@ -2606,8 +2679,15 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
     /* with cpu model support, CMM is only indicated if really available */
     if (kvm_s390_cmma_available()) {
         set_bit(S390_FEAT_CMM, model->features);
+    } else {
+        /* no cmm -> no cmm nt */
+        clear_bit(S390_FEAT_CMM_NT, model->features);
     }
 
+    /* set zpci and aen facilities */
+    set_bit(S390_FEAT_ZPCI, model->features);
+    set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);
+
     if (s390_known_cpu_type(cpu_type)) {
         /* we want the exact model, even if some features are missing */
         model->def = s390_find_cpu_def(cpu_type, ibc_gen(unblocked_ibc),
@@ -2641,7 +2721,7 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
 
     if (!model) {
         /* compatibility handling if cpu models are disabled */
-        if (kvm_s390_cmma_available() && !mem_path) {
+        if (kvm_s390_cmma_available()) {
             kvm_s390_enable_cmma();
         }
         return;
@@ -2672,13 +2752,8 @@ void kvm_s390_apply_cpu_model(const S390CPUModel *model, Error **errp)
         error_setg(errp, "KVM: Error configuring CPU subfunctions: %d", rc);
         return;
     }
-    /* enable CMM via CMMA - disable on hugetlbfs */
+    /* enable CMM via CMMA */
     if (test_bit(S390_FEAT_CMM, model->features)) {
-        if (mem_path) {
+        kvm_s390_enable_cmma();
-            warn_report("CMM will not be enabled because it is not "
-                        "compatible to hugetlbfs.");
-        } else {
-            kvm_s390_enable_cmma();
-        }
     }
 }

target/s390x/machine.c

@@ -174,6 +174,22 @@ const VMStateDescription vmstate_exval = {
     }
 };
 
+static bool gscb_needed(void *opaque)
+{
+    return kvm_s390_get_gs();
+}
+
+const VMStateDescription vmstate_gscb = {
+    .name = "cpu/gscb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = gscb_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.gscb, S390CPU, 4),
+        VMSTATE_END_OF_LIST()
+        }
+};
+
 const VMStateDescription vmstate_s390_cpu = {
     .name = "cpu",
     .post_load = cpu_post_load,
@@ -207,6 +223,7 @@ const VMStateDescription vmstate_s390_cpu = {
         &vmstate_vregs,
         &vmstate_riccb,
         &vmstate_exval,
+        &vmstate_gscb,
         NULL
     },
 };