From cb7e367a560ff09e3179b783e08fa92c35bf2eb6 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 23 Jun 2021 06:46:24 +0000 Subject: [PATCH 01/10] server: fix failed to apply userdata when enable static nat --- .../java/com/cloud/network/rules/RulesManagerImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java b/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java index de567749043f..634c568aa5f7 100644 --- a/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java @@ -640,10 +640,13 @@ protected void applyUserData(long vmId, Network network, Nic guestNic) throws Re if (element == null) { s_logger.error("Can't find network element for " + Service.UserData.getName() + " provider needed for UserData update"); } else { - boolean result = element.saveUserData(network, nicProfile, vmProfile); - if (!result) { + try { + if (!element.saveUserData(network, nicProfile, vmProfile)) { s_logger.error("Failed to update userdata for vm " + vm + " and nic " + guestNic); } + } catch (Exception e) { + s_logger.error("Failed to update userdata for vm " + vm + " and nic " + guestNic + " due to " + e.getMessage(), e); + } } } From b0cbfc72d5b9c1c59f7ae93d1a5de508794f17a2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 23 Jun 2021 15:31:38 +0000 Subject: [PATCH 02/10] server: fix cannot expunge vm as applyUserdata fails --- .../com/cloud/network/rules/RulesManagerImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java b/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java index 634c568aa5f7..b743863edc71 100644 --- a/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java @@ -630,17 +630,17 @@ private boolean enableStaticNat(long ipId, long vmId, long networkId, boolean is } protected void applyUserData(long vmId, Network network, Nic guestNic) throws ResourceUnavailableException { - UserVmVO vm = _vmDao.findById(vmId); - VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); - NicProfile nicProfile = new NicProfile(guestNic, network, null, null, null, - _networkModel.isSecurityGroupSupportedInNetwork(network), - _networkModel.getNetworkTag(template.getHypervisorType(), network)); - VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm); UserDataServiceProvider element = _networkModel.getUserDataUpdateProvider(network); if (element == null) { s_logger.error("Can't find network element for " + Service.UserData.getName() + " provider needed for UserData update"); } else { + UserVmVO vm = _vmDao.findById(vmId); try { + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); + NicProfile nicProfile = new NicProfile(guestNic, network, null, null, null, + _networkModel.isSecurityGroupSupportedInNetwork(network), + _networkModel.getNetworkTag(template.getHypervisorType(), network)); + VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm); if (!element.saveUserData(network, nicProfile, vmProfile)) { s_logger.error("Failed to update userdata for vm " + vm + " and nic " + guestNic); } From 9465283ba7926c770735040bc5084bfe7469cbc4 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 24 Jun 2021 06:26:28 +0000 Subject: [PATCH 03/10] configdrive: fix ISO is not recognized when plug a new nic --- scripts/storage/secondary/createvolume.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/storage/secondary/createvolume.sh b/scripts/storage/secondary/createvolume.sh index 12f73ebc1096..4e7af005d5c2 100755 --- a/scripts/storage/secondary/createvolume.sh +++ b/scripts/storage/secondary/createvolume.sh @@ -135,7 +135,8 @@ create_from_file() { local tmpltname=$3 [ -n "$verbose" ] && echo "Moving to $tmpltfs/$tmpltname...could take a while" >&2 - mv $tmpltimg /$tmpltfs/$tmpltname + cp -rf $tmpltimg /$tmpltfs/$tmpltname + rm -rf $tmpltimg } From 3182171f6379d3c5cc3a756a3ded68e9830177f9 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 25 Jun 2021 07:44:36 +0000 Subject: [PATCH 04/10] configdrive: detach and attach configdrive ISO as it is changed when plug a new nic or migrate vm --- .../resource/LibvirtComputingResource.java | 28 +++++++++++++++++++ .../wrapper/LibvirtMigrateCommandWrapper.java | 5 ++++ .../wrapper/LibvirtPlugNicCommandWrapper.java | 4 +++ 3 files changed, 37 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index a7d5ee35ac16..bd734c1186b3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -303,6 +303,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected String _agentHooksVmOnStopScript = "libvirt-vm-state-change.groovy"; protected String _agentHooksVmOnStopMethod = "onStop"; + private static final String CONFIG_DRIVE_ISO_DISK_LABEL = "hdd"; + private static final int CONFIG_DRIVE_ISO_DEVICE_ID = 4; protected File _qemuSocketsPath; private final String _qemuGuestAgentSocketName = "org.qemu.guest_agent.0"; @@ -2756,6 +2758,32 @@ protected KVMStoragePoolManager getPoolManager() { return _storagePoolMgr; } + public void detachAndAttachConfigDriveISO(final Connect conn, final String vmName) { + // detach and re-attach configdrive ISO + List disks = getDisks(conn, vmName); + DiskDef configdrive = null; + for (DiskDef disk : disks) { + if (disk.getDeviceType() == DiskDef.DeviceType.CDROM && disk.getDiskLabel() == CONFIG_DRIVE_ISO_DISK_LABEL) { + configdrive = disk; + } + } + if (configdrive != null) { + try { + String result = attachOrDetachISO(conn, vmName, configdrive.getDiskPath(), false, CONFIG_DRIVE_ISO_DEVICE_ID); + if (result != null) { + s_logger.warn("Detach ConfigDrive ISO with result: " + result); + } + result = attachOrDetachISO(conn, vmName, configdrive.getDiskPath(), true, CONFIG_DRIVE_ISO_DEVICE_ID); + if (result != null) { + s_logger.warn("Attach ConfigDrive ISO with result: " + result); + } + } catch (final LibvirtException | InternalErrorException | URISyntaxException e) { + final String msg = "Detach and attach ConfigDrive ISO failed due to " + e.toString(); + s_logger.warn(msg, e); + } + } + } + public synchronized String attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, final Integer diskSeq) throws LibvirtException, URISyntaxException, InternalErrorException { final DiskDef iso = new DiskDef(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 841eadf13c28..60261a47837f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -76,6 +76,7 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; import com.google.common.base.Strings; @ResourceWrapper(handles = MigrateCommand.class) @@ -180,6 +181,10 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. dconn = libvirtUtilitiesHelper.retrieveQemuConnection(destinationUri); + if (to.getType() == VirtualMachine.Type.User) { + libvirtComputingResource.detachAndAttachConfigDriveISO(conn, vmName); + } + //run migration in thread so we can monitor it s_logger.info("Live migration of instance " + vmName + " initiated to destination host: " + dconn.getURI()); final ExecutorService executor = Executors.newFixedThreadPool(1); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java index 553a71a30dfe..ca78c718886e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java @@ -72,6 +72,10 @@ public Answer execute(final PlugNicCommand command, final LibvirtComputingResour libvirtComputingResource.applyDefaultNetworkRulesOnNic(conn, vmName, vmId, nic, false, false); } + if (vmType == VirtualMachine.Type.User) { + libvirtComputingResource.detachAndAttachConfigDriveISO(conn, vmName); + } + return new PlugNicAnswer(command, true, "success"); } catch (final LibvirtException e) { final String msg = " Plug Nic failed due to " + e.toString(); From bb5b0df7e250d0b2904e34c4839ac4e39a46072b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 25 Jun 2021 08:53:13 +0000 Subject: [PATCH 05/10] configdrive test: (1) password file does not exists in recreated ISO; (2) vm hostname should be changed after migration --- test/integration/component/test_configdrive.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/integration/component/test_configdrive.py b/test/integration/component/test_configdrive.py index 38a9a7d2ca88..f5f5162a8a2d 100644 --- a/test/integration/component/test_configdrive.py +++ b/test/integration/component/test_configdrive.py @@ -53,7 +53,8 @@ Hypervisor, Template) from marvin.lib.common import (get_domain, get_template, - get_zone, get_test_template, + get_zone, + get_test_template, is_config_suitable) from marvin.lib.utils import random_gen # Import System Modules @@ -974,6 +975,7 @@ def plug_nic(self, vm, network): vm.add_nic(self.api_client, network.id) self.debug("Added NIC in VM with ID - %s and network with ID - %s" % (vm.id, network.id)) + vm.password_test = ConfigDriveUtils.PasswordTest(expect_pw=False) def unplug_nic(self, vm, network): nic = self._find_nic(vm, network) @@ -1702,6 +1704,7 @@ def migrate_VM(self, vm): "%s to Host: %s" % (vm.id, host.id)) try: vm.migrate(self.api_client, hostid=host.id) + vm.password_test = ConfigDriveUtils.PasswordTest(expect_pw=False) except Exception as e: self.fail("Failed to migrate instance, %s" % e) self.debug("Migrated VM with ID: " @@ -1917,7 +1920,8 @@ def test_configdrive_isolated_network(self): # ===================================================================== self.debug("+++ Scenario: " "update userdata and reset password after migrate") - self.migrate_VM(vm1) + host = self.migrate_VM(vm1) + vm1.hostname = host.name self.then_config_drive_is_as_expected(vm1, public_ip_1, metadata=True) self.debug("Updating userdata after migrating VM - %s" % vm1.name) self.update_and_validate_userdata(vm1, "hello after migrate", @@ -2112,7 +2116,8 @@ def test_configdrive_vpc_network(self): # ===================================================================== self.debug("+++ Scenario: " "update userdata and reset password after migrate") - self.migrate_VM(vm) + host = self.migrate_VM(vm) + vm.hostname = host.name self.then_config_drive_is_as_expected(vm, public_ip_1, metadata=True) self.update_and_validate_userdata(vm, "hello migrate", public_ip_1) @@ -2362,6 +2367,7 @@ def test_configdrive_isolated_network_hypervisor_hostname_exposed(self): self.debug("+++Deploy VM in the created Isolated network " "with user data provider as configdrive") + self.given_template_password_enabled_is(True) vm1 = self.when_I_deploy_a_vm(network1) public_ip_1 = self.when_I_create_a_static_nat_ip_to(vm1, network1) @@ -2476,6 +2482,7 @@ def test_configdrive_vpc_network_verify_metadata(self): # ===================================================================== self.debug("+++ Scenario: " "Deploy VM in the Tier 1 with user data") + self.given_template_password_enabled_is(True) vm = self.when_I_deploy_a_vm(network1) public_ip_1 = self.when_I_create_a_static_nat_ip_to(vm, network1) From 1029e3bdbc6e9550bc76848318b343f7d78572c8 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 25 Jun 2021 12:51:27 +0000 Subject: [PATCH 06/10] configdrive: use centos55 template with sshkey and configdrive support --- test/integration/component/test_configdrive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/component/test_configdrive.py b/test/integration/component/test_configdrive.py index f5f5162a8a2d..4767b90335b2 100644 --- a/test/integration/component/test_configdrive.py +++ b/test/integration/component/test_configdrive.py @@ -105,12 +105,12 @@ def __init__(self): self.services = { "test_templates": { "kvm": { - "name": "Centos-5.5-configdrive", - "displaytext": "ConfigDrive enabled CentOS", + "name": "Centos-5.5-sshkey-and-configdrive", + "displaytext": "SSHkey and ConfigDrive enabled CentOS", "format": "qcow2", "hypervisor": "kvm", "ostype": "CentOS 5.5 (64-bit)", - "url": "http://people.apache.org/~fmaximus/centos55-extended.qcow2.bz2", + "url": "http://people.apache.org/~weizhou/centos55-sshkey-configdrive.qcow2.bz2", "requireshvm": "False", "ispublic": "True", "isextractable": "True" From 1164c3db93a81a8f69a597044fee5cc961e479a2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 25 Jun 2021 14:43:17 +0000 Subject: [PATCH 07/10] configdrive: disklabel is 'config-2' for configdrive ISO --- setup/bindir/cloud-set-guest-sshkey-configdrive.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bindir/cloud-set-guest-sshkey-configdrive.in b/setup/bindir/cloud-set-guest-sshkey-configdrive.in index 31dc6df92db3..c1385c6a0d15 100644 --- a/setup/bindir/cloud-set-guest-sshkey-configdrive.in +++ b/setup/bindir/cloud-set-guest-sshkey-configdrive.in @@ -44,7 +44,7 @@ function prepare_mount if [ -e $DefaultDisk ]; then Disk=$DefaultDisk else - BLOCK_DEVICE=$(blkid -t LABEL='config' /dev/hd? /dev/sd? /dev/xvd? /dev/vd? -o device) + BLOCK_DEVICE=$(blkid -t LABEL='config-2' /dev/hd? /dev/sd? /dev/xvd? /dev/vd? -o device) if [ -n $BLOCK_DEVICE ]; then Disk=$BLOCK_DEVICE else From 4392bcb5b60dad69c90bee1a2e7c57ffd695150b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 25 Jun 2021 14:43:27 +0000 Subject: [PATCH 08/10] configdrive: use copy for configdrive ISO and move for other template/volume/iso --- scripts/storage/secondary/createvolume.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/storage/secondary/createvolume.sh b/scripts/storage/secondary/createvolume.sh index 4e7af005d5c2..14bbee4d4dff 100755 --- a/scripts/storage/secondary/createvolume.sh +++ b/scripts/storage/secondary/createvolume.sh @@ -135,9 +135,18 @@ create_from_file() { local tmpltname=$3 [ -n "$verbose" ] && echo "Moving to $tmpltfs/$tmpltname...could take a while" >&2 + mv $tmpltimg /$tmpltfs/$tmpltname + +} + +copy_from_file() { + local tmpltfs=$1 + local tmpltimg=$2 + local tmpltname=$3 + + [ -n "$verbose" ] && echo "Copying to $tmpltfs/$tmpltname...could take a while" >&2 cp -rf $tmpltimg /$tmpltfs/$tmpltname rm -rf $tmpltimg - } tflag= @@ -229,7 +238,11 @@ fi imgsize=$(ls -l $tmpltimg2| awk -F" " '{print $5}') -create_from_file $tmpltfs $tmpltimg2 $tmpltname +if [ "$descr" = "configDrive" ] && [ "$Sflag" = "" ];then + copy_from_file $tmpltfs $tmpltimg2 $tmpltname +else + create_from_file $tmpltfs $tmpltimg2 $tmpltname +fi touch /$tmpltfs/volume.properties rollback_if_needed $tmpltfs $? "Failed to create volume.properties file" From 089448edd75b0bb5a7d8a5b433b68afaa57c0624 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 25 Jun 2021 15:15:27 +0000 Subject: [PATCH 09/10] configdrive: use public-keys.txt --- setup/bindir/cloud-set-guest-sshkey-configdrive.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bindir/cloud-set-guest-sshkey-configdrive.in b/setup/bindir/cloud-set-guest-sshkey-configdrive.in index c1385c6a0d15..7294871f3293 100644 --- a/setup/bindir/cloud-set-guest-sshkey-configdrive.in +++ b/setup/bindir/cloud-set-guest-sshkey-configdrive.in @@ -31,7 +31,7 @@ mountdir=$(mktemp -d) # If lable name is other than config, please change the below line as required DefaultDisk=/dev/disk/by-label/config-2 -SSHKey_File=$mountdir/cloudstack/metadata/public_keys.txt +SSHKey_File=$mountdir/cloudstack/metadata/public-keys.txt keys_received=0 function prepare_mount From 845808b6b0573db6e74e218c4a4b8a1f24a11ce1 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 29 Jun 2021 07:36:36 +0000 Subject: [PATCH 10/10] configdrive test: fix (1) update_template ; (2) ssh into vm by keypair --- test/integration/component/test_configdrive.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/integration/component/test_configdrive.py b/test/integration/component/test_configdrive.py index 4767b90335b2..fb2fb43e5c1c 100644 --- a/test/integration/component/test_configdrive.py +++ b/test/integration/component/test_configdrive.py @@ -654,8 +654,7 @@ def given_template_password_enabled_is(self, new_state): orig_state = self.template.passwordenabled self.debug("Updating guest VM template to password enabled " "from %s to %s" % (orig_state, new_state)) - if orig_state != new_state: - self.update_template(passwordenabled=new_state) + self.update_template(passwordenabled=new_state) self.assertEqual(self.template.passwordenabled, new_state, "Guest VM template is not password enabled") return orig_state @@ -851,7 +850,7 @@ def then_config_drive_is_as_expected(self, vm, self.debug("SSHing into the VM %s" % vm.name) - ssh = self.ssh_into_VM(vm, public_ip, reconnect=reconnect) + ssh = self.ssh_into_VM(vm, public_ip, reconnect=reconnect, keypair=vm.key_pair) d = {x.name: x for x in ssh.logger.handlers} ssh.logger.handlers = list(d.values()) @@ -1532,12 +1531,14 @@ def ssh_into_VM(self, vm, public_ip, reconnect=True, self.debug("SSH into VM with ID - %s on public IP address - %s" % (vm.id, public_ip.ipaddress.ipaddress)) tries = 1 if negative_test else 3 + private_key_file_location = keypair.private_key_file if keypair else None @retry(tries=tries) def retry_ssh(): ssh_client = vm.get_ssh_client( ipaddress=public_ip.ipaddress.ipaddress, reconnect=reconnect, + keyPairFileLocation=private_key_file_location, retries=3 if negative_test else 30 ) self.debug("Successful to SSH into VM with ID - %s on " @@ -1959,7 +1960,7 @@ def test_configdrive_isolated_network(self): # ===================================================================== self.debug("+++ Scenario: " "Update Userdata on a VM that is not password enabled") - self.update_template(passwordenabled=False) + self.given_template_password_enabled_is(False) vm1 = self.when_I_deploy_a_vm_with_keypair_in(network1) public_ip_1 = \ @@ -2155,7 +2156,7 @@ def test_configdrive_vpc_network(self): self.debug("+++ Scenario: " "Update Userdata on a VM that is not password enabled") - self.update_template(passwordenabled=False) + self.given_template_password_enabled_is(False) vm = self.when_I_deploy_a_vm(network1, keypair=self.keypair.name) @@ -2290,7 +2291,7 @@ def test_configdrive_shared_network(self): self.delete(vm1, expunge=True) self.given_config_drive_provider_is("Enabled") - self.update_template(passwordenabled=False) + self.given_template_password_enabled_is(False) vm1 = self.create_VM( [shared_network.network],