← Back to team overview

yahoo-eng-team team mailing list archive

[Bug 2127229] [NEW] Only one disk was assigned the bootindex in Nova.

 

Public bug reported:


# create a ceph rbd volume
# openstack volume snap create --volume volume-test snap-test
# openstack server create \
--block-device source_type=snapshot,uuid=727f39d0-0ab9-456d-8e5d-c0834e8d1bba,boot_index=0 \
--block-device source_type=snapshot,uuid=727f39d0-0ab9-456d-8e5d-c0834e8d1bba,boot_index=1 \    
--flavor m1.small --network private testvm-simple-2
# openstack --version
openstack 8.2.0


/usr/bin/qemu-system-x86_64 -name guest=instance-00000013,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-13-instance-00000013/master-key.aes"} -machine pc-i440fx-6.2,usb=off,dump-guest-core=off,memory-backend=pc.ram -accel kvm -cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,stibp=on,arch-capabilities=on,svme-addr-chk=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,xsaves=off -m 2048 -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":2147483648} -overcommit mem-lock=off -smp 1,sockets=1,dies=1,cores=1,threads=1 -uuid 0a6ada0d-361f-4fc3-a225-62c4c251c14d -smbios type=1,manufacturer=OpenStack Foundation,product=OpenStack Nova,version=29.2.0,serial=0a6ada0d-361f-4fc3-a225-62c4c251c14d,uuid=0a6ada0d-361f-4fc3-a225-62c4c251c14d,family=Virtual Machine -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=32,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -boot strict=on -object {"qom-type":"secret","id":"libvirt-2-storage-auth-secret0","data":"L/rZMpy420IYmwUN5JmCygUBzEtspc/0GvsAkB0Er+Y=","keyid":"masterKey0","iv":"3MU86I7gccF1/D8ufBs+aA==","format":"base64"} -blockdev {"driver":"rbd","pool":"cinder-ceph","image":"volume-734ddb4c-3240-4465-8d61-9cfd21d91b5b","server":[{"host":"10.149.94.113","port":"6789"}],"user":"cinder-ceph","auth-client-required":["cephx","none"],"key-secret":"libvirt-2-storage-auth-secret0","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"} -device virtio-blk-pci,bus=pci.0,addr=0x3,drive=libvirt-2-format,id=ua-734ddb4c-3240-4465-8d61-9cfd21d91b5b,bootindex=1,write-cache=on,serial=734ddb4c-3240-4465-8d61-9cfd21d91b5b -object {"qom-type":"secret","id":"libvirt-1-storage-auth-secret0","data":"LCkFuHY1yYQwjuzhfyHvqYC8KBc+HAmBaRDXmQlZ5Rc=","keyid":"masterKey0","iv":"t/C68mBNKgofqWyII9/zIw==","format":"base64"} -blockdev {"driver":"rbd","pool":"cinder-ceph","image":"volume-bbefb78b-7cfb-464f-9cc2-8874a3b4379e","server":[{"host":"10.149.94.113","port":"6789"}],"user":"cinder-ceph","auth-client-required":["cephx","none"],"key-secret":"libvirt-1-storage-auth-secret0","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"} -device virtio-blk-pci,bus=pci.0,addr=0x4,drive=libvirt-1-format,id=ua-bbefb78b-7cfb-464f-9cc2-8874a3b4379e,write-cache=on,serial=bbefb78b-7cfb-464f-9cc2-8874a3b4379e -netdev tap,fd=34,id=hostnet0,vhost=on,vhostfd=36 -device virtio-net-pci,host_mtu=1442,netdev=hostnet0,id=net0,mac=fa:16:3e:8b:9d:0c,bus=pci.0,addr=0x2 -add-fd set=3,fd=33 -chardev pty,id=charserial0,logfile=/dev/fdset/3,logappend=on -device isa-serial,chardev=charserial0,id=serial0 -audiodev {"id":"audio1","driver":"none"} -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"} -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x6 -device vmcoreinfo -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

I create a VM as described, but only its vda disk has the bootindex set, not vdb. 
This leads to a problem in a corner case: GRUB cannot recognize vdb after a chainload from vda. 
Testing confirms this.

There are only 3 scenarios in which grub on vda can detect vdb, as
follows:

- When boot strict=on is not set, as long as it is not the case where vda lacks bootindex while vdb has bootindex.
- When boot strict=on is set, but both vda and vdb have the bootindex parameter.
- When boot strict=on is set, but neither vda nor vdb has the bootindex parameter.
 
Test result: https://paste.ubuntu.com/p/y9pDVkZjMJ/
Bug https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/2125827

So, the fix is to either remove "boot strict=on" or set a bootindex on both disks. 
Otherwise, GRUB won't see vdb. 

The problem is that OpenStack seems to default to "boot strict=on", and
I can't find a way to turn that off while only vda has a bootindex.
That's what's breaking the GRUB chainload.

** Affects: nova
     Importance: Undecided
         Status: New

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/2127229

Title:
  Only one disk was assigned the bootindex in Nova.

Status in OpenStack Compute (nova):
  New

Bug description:

  # create a ceph rbd volume
  # openstack volume snap create --volume volume-test snap-test
  # openstack server create \
  --block-device source_type=snapshot,uuid=727f39d0-0ab9-456d-8e5d-c0834e8d1bba,boot_index=0 \
  --block-device source_type=snapshot,uuid=727f39d0-0ab9-456d-8e5d-c0834e8d1bba,boot_index=1 \    
  --flavor m1.small --network private testvm-simple-2
  # openstack --version
  openstack 8.2.0

  
  /usr/bin/qemu-system-x86_64 -name guest=instance-00000013,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-13-instance-00000013/master-key.aes"} -machine pc-i440fx-6.2,usb=off,dump-guest-core=off,memory-backend=pc.ram -accel kvm -cpu EPYC-Rome,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,stibp=on,arch-capabilities=on,svme-addr-chk=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,pschange-mc-no=on,xsaves=off -m 2048 -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":2147483648} -overcommit mem-lock=off -smp 1,sockets=1,dies=1,cores=1,threads=1 -uuid 0a6ada0d-361f-4fc3-a225-62c4c251c14d -smbios type=1,manufacturer=OpenStack Foundation,product=OpenStack Nova,version=29.2.0,serial=0a6ada0d-361f-4fc3-a225-62c4c251c14d,uuid=0a6ada0d-361f-4fc3-a225-62c4c251c14d,family=Virtual Machine -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=32,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -boot strict=on -object {"qom-type":"secret","id":"libvirt-2-storage-auth-secret0","data":"L/rZMpy420IYmwUN5JmCygUBzEtspc/0GvsAkB0Er+Y=","keyid":"masterKey0","iv":"3MU86I7gccF1/D8ufBs+aA==","format":"base64"} -blockdev {"driver":"rbd","pool":"cinder-ceph","image":"volume-734ddb4c-3240-4465-8d61-9cfd21d91b5b","server":[{"host":"10.149.94.113","port":"6789"}],"user":"cinder-ceph","auth-client-required":["cephx","none"],"key-secret":"libvirt-2-storage-auth-secret0","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"} -device virtio-blk-pci,bus=pci.0,addr=0x3,drive=libvirt-2-format,id=ua-734ddb4c-3240-4465-8d61-9cfd21d91b5b,bootindex=1,write-cache=on,serial=734ddb4c-3240-4465-8d61-9cfd21d91b5b -object {"qom-type":"secret","id":"libvirt-1-storage-auth-secret0","data":"LCkFuHY1yYQwjuzhfyHvqYC8KBc+HAmBaRDXmQlZ5Rc=","keyid":"masterKey0","iv":"t/C68mBNKgofqWyII9/zIw==","format":"base64"} -blockdev {"driver":"rbd","pool":"cinder-ceph","image":"volume-bbefb78b-7cfb-464f-9cc2-8874a3b4379e","server":[{"host":"10.149.94.113","port":"6789"}],"user":"cinder-ceph","auth-client-required":["cephx","none"],"key-secret":"libvirt-1-storage-auth-secret0","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"} -device virtio-blk-pci,bus=pci.0,addr=0x4,drive=libvirt-1-format,id=ua-bbefb78b-7cfb-464f-9cc2-8874a3b4379e,write-cache=on,serial=bbefb78b-7cfb-464f-9cc2-8874a3b4379e -netdev tap,fd=34,id=hostnet0,vhost=on,vhostfd=36 -device virtio-net-pci,host_mtu=1442,netdev=hostnet0,id=net0,mac=fa:16:3e:8b:9d:0c,bus=pci.0,addr=0x2 -add-fd set=3,fd=33 -chardev pty,id=charserial0,logfile=/dev/fdset/3,logappend=on -device isa-serial,chardev=charserial0,id=serial0 -audiodev {"id":"audio1","driver":"none"} -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"} -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x6 -device vmcoreinfo -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

  I create a VM as described, but only its vda disk has the bootindex set, not vdb. 
  This leads to a problem in a corner case: GRUB cannot recognize vdb after a chainload from vda. 
  Testing confirms this.

  There are only 3 scenarios in which grub on vda can detect vdb, as
  follows:

  - When boot strict=on is not set, as long as it is not the case where vda lacks bootindex while vdb has bootindex.
  - When boot strict=on is set, but both vda and vdb have the bootindex parameter.
  - When boot strict=on is set, but neither vda nor vdb has the bootindex parameter.
   
  Test result: https://paste.ubuntu.com/p/y9pDVkZjMJ/
  Bug https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/2125827

  So, the fix is to either remove "boot strict=on" or set a bootindex on both disks. 
  Otherwise, GRUB won't see vdb. 

  The problem is that OpenStack seems to default to "boot strict=on",
  and I can't find a way to turn that off while only vda has a
  bootindex. That's what's breaking the GRUB chainload.

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/2127229/+subscriptions