---
# System prepended metadata

title: 20220203_releng-bringup-multi-arch-FCOS-VMs

---

# 20220203_releng-bringup-multi-arch-FCOS-VMs


Issue: https://pagure.io/fedora-infrastructure/issue/8370

Image to use: https://fedorapeople.org/groups/fcos-images/builds/latest/ppc64le/fedora-coreos-35.20220131.dev.0-qemu.ppc64le.qcow2.xz
Argument to virt-install for the Ignition config

```
--disk path=$PWD/config.ign,format=raw,readonly=on,serial=ignition
```

tasks/virt_instance_create.yml is the task that makes vm's
using the virt-instance-create-cmd in inventory/group_vars/all


aarch64 machine specs (for comparison)
16vcpu
32GiB RAM
100GiB disk

static networking details:
10.3.171.40/24
10.3.171.254


Host kernel: 4.18.0-305.3.1.el8_4.ppc64le
VM (FCOS) kernel: kernel-5.15.17-200.fc35.ppc64le

virt-install command line: 

```
virt-install --import \
    --name buildvm-ppc64le-fcos01.iad2.fedoraproject.org \
    --memory=32768,maxmemory=32768                       \
    --vcpus=16,maxvcpus=16                               \
    --feature nested-hv=on                               \
    --network bridge=br0,model=virtio,mac=RANDOM         \
    --autostart                                          \
    --memballoon virtio                                  \
    --watchdog default                                   \
    --rng /dev/random                                    \
    --noautoconsole                                      \
    --disk path=$PWD/fcos-ppc64le-builder.ign,format=raw,readonly=on,serial=ignition \
    --disk bus=virtio,path=/dev/vg_guests/buildvm-ppc64le-fcos01.iad2.fedoraproject.org,cache=unsafe,io=threads
```


Libvirt XML


```
<domain type='kvm' id='24'>
  <name>buildvm-ppc64le-fcos01.iad2.fedoraproject.org</name>
  <uuid>ed30c95e-b7c0-4c25-a6ba-b739459f101b</uuid>
  <memory unit='KiB'>33554432</memory>
  <currentMemory unit='KiB'>33554432</currentMemory>
  <vcpu placement='static'>16</vcpu>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='ppc64le' machine='pseries-rhel8.3.0'>hvm</type>
    <boot dev='hd'/>
  </os>
  <features>
    <nested-hv state='on'/>
  </features>
  <cpu mode='custom' match='exact' check='none'>
    <model fallback='forbid'>POWER9</model>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='unsafe' io='threads'/>
      <source dev='/dev/vg_guests/buildvm-ppc64le-fcos01.iad2.fedoraproject.org' index='2'/>
      <backingStore/>
      <target dev='vda' bus='virtio'/>
      <alias name='virtio-disk0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='file' device='disk'>
      <driver name='qemu' type='raw'/>
      <source file='/tmp/fcos-ppc64le-builder.ign' index='1'/>
      <backingStore/>
      <target dev='vdb' bus='virtio'/>
      <readonly/>
      <serial>ignition</serial>
      <alias name='virtio-disk1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </disk>
    <controller type='usb' index='0' model='qemu-xhci' ports='15'>
      <alias name='usb'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'>
      <model name='spapr-pci-host-bridge'/>
      <target index='0'/>
      <alias name='pci.0'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <alias name='virtio-serial0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </controller>
    <interface type='bridge'>
      <mac address='52:54:00:c4:d2:aa'/>
      <source bridge='br0'/>
      <target dev='vnet23'/>
      <model type='virtio'/>
      <alias name='net0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </interface>
    <serial type='pty'>
      <source path='/dev/pts/11'/>
      <target type='spapr-vio-serial' port='0'>
        <model name='spapr-vty'/>
      </target>
      <alias name='serial0'/>
      <address type='spapr-vio' reg='0x30000000'/>
    </serial>
    <console type='pty' tty='/dev/pts/11'>
      <source path='/dev/pts/11'/>
      <target type='serial' port='0'/>
      <alias name='serial0'/>
      <address type='spapr-vio' reg='0x30000000'/>
    </console>
    <channel type='unix'>
      <source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-24-buildvm-ppc64le-fcos/org.qemu.guest_agent.0'/>
      <target type='virtio' name='org.qemu.guest_agent.0' state='disconnected'/>
      <alias name='channel0'/>
      <address type='virtio-serial' controller='0' bus='0' port='1'/>
    </channel>
    <input type='tablet' bus='usb'>
      <alias name='input0'/>
      <address type='usb' bus='0' port='1'/>
    </input>
    <input type='keyboard' bus='usb'>
      <alias name='input1'/>
      <address type='usb' bus='0' port='2'/>
    </input>
    <graphics type='vnc' port='5910' autoport='yes' listen='127.0.0.1'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='vga' vram='16384' heads='1' primary='yes'/>
      <alias name='video0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
    </video>
    <watchdog model='i6300esb' action='reset'>
      <alias name='watchdog0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
    </watchdog>
    <memballoon model='virtio'>
      <alias name='balloon0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </memballoon>
    <rng model='virtio'>
      <backend model='random'>/dev/random</backend>
      <alias name='rng0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
    </rng>
    <panic model='pseries'/>
  </devices>
  <seclabel type='dynamic' model='selinux' relabel='yes'>
    <label>system_u:system_r:svirt_t:s0:c131,c913</label>
    <imagelabel>system_u:object_r:svirt_image_t:s0:c131,c913</imagelabel>
  </seclabel>
  <seclabel type='dynamic' model='dac' relabel='yes'>
    <label>+107:+107</label>
    <imagelabel>+107:+107</imagelabel>
  </seclabel>
</domain>
```


Failure seen when running COSA:

```
KVM: Failed to create TCE64 table for liobn 0x71000002
KVM: Failed to create TCE64 table for liobn 0x80000000
KVM: unknown exit, hardware reason ffffffffffffffc9
NIP 0000000000000100   LR 0000000000000000 CTR 0000000000000000 XER 0000000000000000 CPU#0
MSR 8000000000001000 HID0 0000000000000000  HF 6c000004 iidx 3 didx 3
TB 00000000 00000000 DECR 0
GPR00 0000000000000000 0000000000000000 0000000000000000 000000007fe00000
GPR04 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR08 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR12 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR16 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR20 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR24 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR28 0000000000000000 0000000000000000 0000000000000000 0000000000000000
CR 00000000  [ -  -  -  -  -  -  -  -  ]             RES ffffffffffffffff
 SRR0 0000000000000000  SRR1 0000000000000000    PVR 00000000004e1202 VRSAVE 0000000000000000
SPRG0 0000000000000000 SPRG1 0000000000000000  SPRG2 0000000000000000  SPRG3 0000000000000000
SPRG4 0000000000000000 SPRG5 0000000000000000  SPRG6 0000000000000000  SPRG7 0000000000000000
HSRR0 0000000000000000 HSRR1 0000000000000000
 CFAR 0000000000000000
 LPCR 0000000000560413
 PTCR 0000000000000000   DAR 0000000000000000  DSISR 0000000000000000
```

Failure seen in journal when running `virt-ls`

```
Feb 04 16:19:39 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: KVMPPC-UVMEM: No support for secure guests
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: vcpu 000000004bd9d345 (0):
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: pc  = 0000000000000100  msr = 8000000000001000  trap = ffffffc9
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 0 = 0000000000000000  r16 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 1 = 0000000000000000  r17 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 2 = 0000000000000000  r18 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 3 = 000000003fe00000  r19 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 4 = 0000000000000000  r20 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 5 = 0000000000000000  r21 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 6 = 0000000000000000  r22 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 7 = 0000000000000000  r23 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 8 = 0000000000000000  r24 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 9 = 0000000000000000  r25 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r10 = 0000000000000000  r26 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r11 = 0000000000000000  r27 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r12 = 0000000000000000  r28 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r13 = 0000000000000000  r29 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r14 = 0000000000000000  r30 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r15 = 0000000000000000  r31 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: ctr = 0000000000000000  lr  = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: srr0 = 0000000000000000 srr1 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: sprg0 = 0000000000000000 sprg1 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: sprg2 = 0000000000000000 sprg3 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: cr = 00000000  xer = 0000000000000000  dsisr = 00000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: dar = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: fault dar = 0000000000000000 dsisr = 0c68f000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: SLB (0 entries):
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: lpcr = 0040000000560413 sdr1 = 0000000000000000 last_inst = ffffffff
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: trap=0xffffffc9 | pc=0x100 | msr=0x8000000000001000
```


Initial failure before we set  `<features> <nested-hv state='on'/> </features>`


```
Running: rpm-ostree compose tree --repo=/srv/tmp/repo --touch-if-changed /srv/tmp/treecompose.changed --cachedir=/srv/cache --unified-core /srv/tmp/override/coreos-assembler-override-manifest.yaml --download-only --ex-lockfile=/srv/src/config/manifest-lock.overrides.yaml
sudo: pam_open_session: Permission denied
sudo: policy plugin failed session initialization
info: Missing sudo privs; using virt
Formatting 'cache2.qcow2.tmp', fmt=qcow2 cluster_size=65536 extended_l2=off compression_type=zlib size=10737418240 lazy_refcounts=off refcount_bits=16
ioctl(KVM_CREATE_VM) failed: 22 Invalid argument
PPC KVM module is not loaded. Try modprobe kvm_hv.
qemu-system-ppc64: failed to initialize kvm: Invalid argument
Error: failed to establish qmp connection: dial unix /var/tmp/mantle-qemu907103112/qmp-1643923761452348243.sock: connect: connection refused
2022-02-03T21:29:50Z cli: failed to establish qmp connection: dial unix /var/tmp/mantle-qemu907103112/qmp-1643923761452348243.sock: connect: connection refused
fatal: Failed to run 'kola qemuexec'
```


