From 6e5d78a8a78feec31a6dfa893137b65fed5f7677 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Thu, 22 Jan 2026 12:46:16 +0530 Subject: [PATCH 001/117] Fix NPE on adding new columns in the tables (#12464) * Fix NPE on adding new columns in the tables * Remove assert --- .../java/com/cloud/utils/db/GenericDaoBase.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 301803aab9b6..c3a4d2c2487c 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -89,6 +89,7 @@ import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.Element; +import org.springframework.util.ClassUtils; /** * GenericDaoBase is a simple way to implement DAOs. It DOES NOT @@ -2047,16 +2048,22 @@ public boolean unremove(ID id) { @DB() protected void setField(final Object entity, final ResultSet rs, ResultSetMetaData meta, final int index) throws SQLException { - Attribute attr = _allColumns.get(new Pair(meta.getTableName(index), meta.getColumnName(index))); + String tableName = meta.getTableName(index); + String columnName = meta.getColumnName(index); + Attribute attr = _allColumns.get(new Pair<>(tableName, columnName)); if (attr == null) { // work around for mysql bug to return original table name instead of view name in db view case Table tbl = entity.getClass().getSuperclass().getAnnotation(Table.class); if (tbl != null) { - attr = _allColumns.get(new Pair(tbl.name(), meta.getColumnLabel(index))); + attr = _allColumns.get(new Pair<>(tbl.name(), meta.getColumnLabel(index))); } } - assert (attr != null) : "How come I can't find " + meta.getCatalogName(index) + "." + meta.getColumnName(index); - setField(entity, attr.field, rs, index); + if(attr == null) { + logger.warn(String.format("Failed to find attribute in the entity %s to map column %s.%s (%s)", + ClassUtils.getUserClass(entity).getSimpleName(), tableName, columnName)); + } else { + setField(entity, attr.field, rs, index); + } } @Override From b5e9178078f0efac75fdb3eb5b07459228da471d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 22 Jan 2026 10:56:03 +0100 Subject: [PATCH 002/117] UI: fix issues when deploy VNF applicance on network with SG (#12436) --- ui/public/locales/en.json | 2 +- ui/src/config/section/network.js | 5 ++++- ui/src/views/compute/DeployVnfAppliance.vue | 2 +- ui/src/views/network/VnfAppliancesTab.vue | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index aaf499d2f954..64437a4d07c1 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2527,7 +2527,7 @@ "label.vnf.app.action.reinstall": "Reinstall VNF Appliance", "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from", "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.", -"label.vnf.configure.management": "Configure Firewall and Port Forwarding rules for VNF's management interfaces", +"label.vnf.configure.management": "Configure network rules for VNF's management interfaces", "label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", "label.vnf.detail.add": "Add VNF detail", "label.vnf.detail.remove": "Remove VNF detail", diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index 30aae3a8deb3..fbc044ff5006 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -356,7 +356,10 @@ export default { permission: ['listVnfAppliances'], resourceType: 'UserVm', params: () => { - return { details: 'servoff,tmpl,nics', isvnf: true } + return { + details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics', + isvnf: true + } }, columns: () => { const fields = ['name', 'state', 'ipaddress'] diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index 1117413d7102..fec1139ab9b6 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -1305,7 +1305,7 @@ export default { for (const deviceId of managementDeviceIds) { if (this.vnfNicNetworks && this.vnfNicNetworks[deviceId] && ((this.vnfNicNetworks[deviceId].type === 'Isolated' && this.vnfNicNetworks[deviceId].vpcid === undefined) || - (this.vnfNicNetworks[deviceId].type === 'Shared' && this.zone.securitygroupsenabled))) { + (this.vnfNicNetworks[deviceId].type === 'Shared' && this.vnfNicNetworks[deviceId].service.filter(svc => svc.name === 'SecurityGroupProvider')))) { return true } } diff --git a/ui/src/views/network/VnfAppliancesTab.vue b/ui/src/views/network/VnfAppliancesTab.vue index 0db85323d15d..139516187c46 100644 --- a/ui/src/views/network/VnfAppliancesTab.vue +++ b/ui/src/views/network/VnfAppliancesTab.vue @@ -120,7 +120,7 @@ export default { methods: { fetchData () { var params = { - details: 'servoff,tmpl,nics', + details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics', isVnf: true, listAll: true } From cd5bb09d0d19e4e01baeaad7a6cbe14ab2db28bc Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:29:41 +0530 Subject: [PATCH 003/117] Fix potential leaks in executePipedCommands (#12478) --- .../java/com/cloud/utils/script/Script.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index 6c62c9106484..ffda782edda2 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -40,9 +40,11 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.cloudstack.utils.security.KeyStoreUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -708,13 +710,31 @@ public static int executeCommandForExitValue(String... command) { return executeCommandForExitValue(0, command); } + private static void cleanupProcesses(AtomicReference> processesRef) { + List processes = processesRef.get(); + if (CollectionUtils.isNotEmpty(processes)) { + for (Process process : processes) { + if (process == null) { + continue; + } + LOGGER.trace(String.format("Cleaning up process [%s] from piped commands.", process.pid())); + IOUtils.closeQuietly(process.getErrorStream()); + IOUtils.closeQuietly(process.getOutputStream()); + IOUtils.closeQuietly(process.getInputStream()); + process.destroyForcibly(); + } + } + } + public static Pair executePipedCommands(List commands, long timeout) { if (timeout <= 0) { timeout = DEFAULT_TIMEOUT; } + final AtomicReference> processesRef = new AtomicReference<>(); Callable> commandRunner = () -> { List builders = commands.stream().map(ProcessBuilder::new).collect(Collectors.toList()); List processes = ProcessBuilder.startPipeline(builders); + processesRef.set(processes); Process last = processes.get(processes.size()-1); try (BufferedReader reader = new BufferedReader(new InputStreamReader(last.getInputStream()))) { String line; @@ -741,6 +761,8 @@ public static Pair executePipedCommands(List commands result.second(ERR_TIMEOUT); } catch (InterruptedException | ExecutionException e) { LOGGER.error("Error executing piped commands", e); + } finally { + cleanupProcesses(processesRef); } return result; } From d1eb2822d9d5b346840851cf21611345454ed734 Mon Sep 17 00:00:00 2001 From: Vishesh <8760112+vishesh92@users.noreply.github.com> Date: Thu, 22 Jan 2026 18:59:35 +0530 Subject: [PATCH 004/117] Remove redundant Exceptions from logs for vm schedules (#12428) --- .../vm/schedule/dao/VMScheduledJobDao.java | 2 ++ .../vm/schedule/dao/VMScheduledJobDaoImpl.java | 15 +++++++++++++++ .../cloudstack/vm/schedule/VMSchedulerImpl.java | 8 +++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java index 7b8c01aae6ad..835ac696f26c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java @@ -31,4 +31,6 @@ public interface VMScheduledJobDao extends GenericDao { int expungeJobsForSchedules(List scheduleId, Date dateAfter); int expungeJobsBefore(Date currentTimestamp); + + VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java index 50a2b12fd774..2f08a41b92e4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java @@ -39,6 +39,8 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase expungeJobForScheduleSearch; + private final SearchBuilder scheduleAndTimestampSearch; + static final String SCHEDULED_TIMESTAMP = "scheduled_timestamp"; static final String VM_SCHEDULE_ID = "vm_schedule_id"; @@ -58,6 +60,11 @@ public VMScheduledJobDaoImpl() { expungeJobForScheduleSearch.and(VM_SCHEDULE_ID, expungeJobForScheduleSearch.entity().getVmScheduleId(), SearchCriteria.Op.IN); expungeJobForScheduleSearch.and(SCHEDULED_TIMESTAMP, expungeJobForScheduleSearch.entity().getScheduledTime(), SearchCriteria.Op.GTEQ); expungeJobForScheduleSearch.done(); + + scheduleAndTimestampSearch = createSearchBuilder(); + scheduleAndTimestampSearch.and(VM_SCHEDULE_ID, scheduleAndTimestampSearch.entity().getVmScheduleId(), SearchCriteria.Op.EQ); + scheduleAndTimestampSearch.and(SCHEDULED_TIMESTAMP, scheduleAndTimestampSearch.entity().getScheduledTime(), SearchCriteria.Op.EQ); + scheduleAndTimestampSearch.done(); } /** @@ -92,4 +99,12 @@ public int expungeJobsBefore(Date date) { sc.setParameters(SCHEDULED_TIMESTAMP, date); return expunge(sc); } + + @Override + public VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp) { + SearchCriteria sc = scheduleAndTimestampSearch.create(); + sc.setParameters(VM_SCHEDULE_ID, scheduleId); + sc.setParameters(SCHEDULED_TIMESTAMP, scheduledTimestamp); + return findOneBy(sc); + } } diff --git a/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java index 7410fb1c2655..56d794fa5c2c 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java @@ -162,7 +162,13 @@ public Date scheduleNextJob(VMScheduleVO vmSchedule, Date timestamp) { } Date scheduledDateTime = Date.from(ts.toInstant()); - VMScheduledJobVO scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime); + VMScheduledJobVO scheduledJob = vmScheduledJobDao.findByScheduleAndTimestamp(vmSchedule.getId(), scheduledDateTime); + if (scheduledJob != null) { + logger.trace("Job is already scheduled for schedule {} at {}", vmSchedule, scheduledDateTime); + return scheduledDateTime; + } + + scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime); try { vmScheduledJobDao.persist(scheduledJob); ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), actionEventMap.get(vmSchedule.getAction()), From 6846619a6f1f27bb8fe67be8161e4ca839c6c4fc Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 22 Jan 2026 10:32:46 -0300 Subject: [PATCH 005/117] Fix update network offering domainids size limitation (#12431) --- .../api/command/admin/network/UpdateNetworkOfferingCmd.java | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java index 9af10262b2d5..8910966ba2e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java @@ -78,6 +78,7 @@ public class UpdateNetworkOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.STRING, + length = 4096, description = "The ID of the containing domain(s) as comma separated string, public for public offerings") private String domainIds; From 6a9835904cb35983ab88e539fe0a5b4c8ce9931b Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 22 Jan 2026 10:57:46 -0300 Subject: [PATCH 006/117] Fix for zoneids parameters length on updateAPIs (#12440) --- .../api/command/admin/offering/UpdateDiskOfferingCmd.java | 1 + .../api/command/admin/offering/UpdateServiceOfferingCmd.java | 1 + .../cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java | 1 + 3 files changed, 3 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java index 2f07f85f9836..c93b5d41a1c5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java @@ -75,6 +75,7 @@ public class UpdateDiskOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java index 0dc97659b9d8..26c7d87ab45c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java @@ -69,6 +69,7 @@ public class UpdateServiceOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java index b8a8077b30b5..44bc88c8daf5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java @@ -65,6 +65,7 @@ public class UpdateVPCOfferingCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; From bce3e54a7e46216917acfe2f1ba1e2a9c9b12128 Mon Sep 17 00:00:00 2001 From: Daman Arora <61474540+Damans227@users.noreply.github.com> Date: Thu, 22 Jan 2026 09:02:46 -0500 Subject: [PATCH 007/117] improve error handling for template upload notifications (#12412) Co-authored-by: Daman Arora --- ui/src/utils/plugins.js | 13 +++++++------ ui/src/views/image/RegisterOrUploadTemplate.vue | 6 +----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index a07f8178604f..0ec957c87292 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -218,18 +218,19 @@ export const notifierPlugin = { if (error.response.status) { msg = `${i18n.global.t('message.request.failed')} (${error.response.status})` } - if (error.message) { - desc = error.message - } - if (error.response.headers && 'x-description' in error.response.headers) { + if (error.response.headers?.['x-description']) { desc = error.response.headers['x-description'] - } - if (desc === '' && error.response.data) { + } else if (error.response.data) { const responseKey = _.findKey(error.response.data, 'errortext') if (responseKey) { desc = error.response.data[responseKey].errortext + } else if (typeof error.response.data === 'string') { + desc = error.response.data } } + if (!desc && error.message) { + desc = error.message + } } let countNotify = store.getters.countNotify countNotify++ diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index 76df7b246aa1..3ada9f6fd531 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -638,11 +638,7 @@ export default { this.$emit('refresh-data') this.closeAction() }).catch(e => { - this.$notification.error({ - message: this.$t('message.upload.failed'), - description: `${this.$t('message.upload.template.failed.description')} - ${e}`, - duration: 0 - }) + this.$notifyError(e) }) }, fetchCustomHypervisorName () { From 8db065a14eb41bab0fb3420e66ee96722f1ed6ad Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Fri, 23 Jan 2026 21:04:52 +0530 Subject: [PATCH 008/117] limit iso filename to have 251 chars at max (#12430) --- .../api/BaseUpdateTemplateOrIsoCmd.java | 2 +- .../api/command/user/iso/RegisterIsoCmd.java | 2 +- .../cloud/upgrade/DatabaseUpgradeChecker.java | 2 + .../upgrade/dao/Upgrade42020to42030.java | 64 +++++++++++++++++++ .../META-INF/db/schema-42020to42030.sql | 22 +++++++ 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java create mode 100644 engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java index 38cf765dd1aa..696a500860e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java @@ -42,7 +42,7 @@ public abstract class BaseUpdateTemplateOrIsoCmd extends BaseCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, required = true, description = "The ID of the image file") private Long id; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the image file") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, length = 251, description = "The name of the image file") private String templateName; @Parameter(name = ApiConstants.OS_TYPE_ID, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index f499c01ce582..2de0f96f2716 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -70,7 +70,7 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd { @Parameter(name = ApiConstants.IS_EXTRACTABLE, type = CommandType.BOOLEAN, description = "True if the ISO or its derivatives are extractable; default is false") private Boolean extractable; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "The name of the ISO") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, length = 251, description = "The name of the ISO") private String isoName; @Parameter(name = ApiConstants.OS_TYPE_ID, diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index afb7a8d69e6d..a8a166fbf275 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -33,6 +33,7 @@ import javax.inject.Inject; +import com.cloud.upgrade.dao.Upgrade42020to42030; import com.cloud.utils.FileUtil; import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang3.StringUtils; @@ -236,6 +237,7 @@ public DatabaseUpgradeChecker() { .next("4.19.0.0", new Upgrade41900to41910()) .next("4.19.1.0", new Upgrade41910to42000()) .next("4.20.0.0", new Upgrade42000to42010()) + .next("4.20.2.0", new Upgrade42020to42030()) .build(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java new file mode 100644 index 000000000000..68100e164018 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import java.io.InputStream; +import java.sql.Connection; + +import com.cloud.utils.exception.CloudRuntimeException; + +public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + + @Override + public String[] getUpgradableVersionRange() { + return new String[]{"4.20.2.0", "4.20.3.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.20.3.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return false; + } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-42020to42030.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public InputStream[] getCleanupScripts() { + return null; + } + + @Override + public void updateSystemVmTemplates(Connection conn) { + } +} diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql new file mode 100644 index 000000000000..598fdb7adc46 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql @@ -0,0 +1,22 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.20.2.0 to 4.20.3.0 +--; + +ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048); From c8cadcb56e553bdbbc141365061bd3542f43612e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 26 Jan 2026 14:01:14 +0530 Subject: [PATCH 009/117] NPE fix while deleting storage pool when pool has detached volumes (#12451) * NPE fix while deleting storage pool when pool has detached volumes * review * unit tests * Added log for volumes not attached to any VMs * update filter, log and test * updated volume dao method names returning non destroyed volumes * build fix --------- Co-authored-by: dahn --- .../java/com/cloud/storage/dao/VolumeDao.java | 6 +- .../com/cloud/storage/dao/VolumeDaoImpl.java | 6 +- .../datastore/PrimaryDataStoreImpl.java | 2 +- .../cloudstack/sioc/SiocManagerImpl.java | 2 +- .../driver/DateraPrimaryDataStoreDriver.java | 2 +- .../provider/DateraHostListener.java | 4 +- .../SolidFirePrimaryDataStoreDriver.java | 2 +- .../provider/SolidFireHostListener.java | 4 +- .../StorPoolPrimaryDataStoreDriver.java | 2 +- .../cloud/resource/ResourceManagerImpl.java | 4 +- .../java/com/cloud/server/StatsCollector.java | 2 +- .../com/cloud/storage/StorageManagerImpl.java | 16 ++++-- .../storage/StoragePoolAutomationImpl.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 2 +- .../resource/ResourceManagerImplTest.java | 12 ++-- .../cloud/storage/StorageManagerImplTest.java | 56 ++++++++++++++++++- 16 files changed, 90 insertions(+), 34 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 4936af3caab5..83f027195185 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -48,7 +48,7 @@ public interface VolumeDao extends GenericDao, StateDao findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); - List findByInstanceIdAndPoolId(long instanceId, long poolId); + List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); List findByInstanceIdDestroyed(long vmId); @@ -70,11 +70,11 @@ public interface VolumeDao extends GenericDao, StateDao findCreatedByInstance(long id); - List findByPoolId(long poolId); + List findNonDestroyedVolumesByPoolId(long poolId); VolumeVO findByPoolIdName(long poolId, String name); - List findByPoolId(long poolId, Volume.Type volumeType); + List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType); List findByPoolIdAndState(long poolid, Volume.State state); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 5ef64b046646..a72b4a258457 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -135,7 +135,7 @@ public List findByInstanceAndDeviceId(long instanceId, long deviceId) } @Override - public List findByPoolId(long poolId) { + public List findNonDestroyedVolumesByPoolId(long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); @@ -144,7 +144,7 @@ public List findByPoolId(long poolId) { } @Override - public List findByInstanceIdAndPoolId(long instanceId, long poolId) { + public List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("instanceId", instanceId); sc.setParameters("poolId", poolId); @@ -161,7 +161,7 @@ public VolumeVO findByPoolIdName(long poolId, String name) { } @Override - public List findByPoolId(long poolId, Volume.Type volumeType) { + public List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 6a10c26cc0bc..d864bf8cd8c7 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -126,7 +126,7 @@ public VolumeInfo getVolume(long id) { @Override public List getVolumes() { - List volumes = volumeDao.findByPoolId(getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(getId()); List volumeInfos = new ArrayList(); for (VolumeVO volume : volumes) { volumeInfos.add(VolumeObject.getVolumeObject(this, volume)); diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index e93b8df39e95..b01af35725f9 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -123,7 +123,7 @@ public void updateSiocInfo(long zoneId, long storagePoolId, int sharesPerGB, int int limitIopsTotal = 0; - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null && volumes.size() > 0) { Set instanceIds = new HashSet<>(); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index dcf84525748f..62393610499a 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -563,7 +563,7 @@ public long getUsedBytes(StoragePool storagePool) { private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { long usedSpaceBytes = 0; - List lstVolumes = _volumeDao.findByPoolId(storagePool.getId(), null); + List lstVolumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (lstVolumes != null) { for (VolumeVO volume : lstVolumes) { diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java index a0dc23da4861..08bc89737f26 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java @@ -247,7 +247,7 @@ private List getStoragePaths(long clusterId, long storagePoolId) { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -317,7 +317,7 @@ private List> getTargets(long clusterId, long storagePoolId) StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 6cc76d99d9ee..1e927e20168e 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -433,7 +433,7 @@ private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { public long getUsedIops(StoragePool storagePool) { long usedIops = 0; - List volumes = volumeDao.findByPoolId(storagePool.getId(), null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java index 052191128f1c..c961c9267395 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java @@ -199,7 +199,7 @@ private List getStoragePaths(long clusterId, long storagePoolId) { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -230,7 +230,7 @@ private List> getTargets(long clusterId, long storagePoolId) StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index 6ca67cb59235..619beee3ec6a 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -1276,7 +1276,7 @@ public Pair getVolumeStats(StoragePool storagePool, String volumeId) return volumeStats; } } else { - List volumes = volumeDao.findByPoolId(storagePool.getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId()); for (VolumeVO volume : volumes) { if (volume.getPath() != null && volume.getPath().equals(volumeId)) { long size = volume.getSize(); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index e62e89eb0efc..a77ecfcb7fe1 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1026,8 +1026,8 @@ private void addVolumesToList(List volumes, List volumesToAd } protected void destroyLocalStoragePoolVolumes(long poolId) { - List rootDisks = volumeDao.findByPoolId(poolId); - List dataVolumes = volumeDao.findByPoolId(poolId, Volume.Type.DATADISK); + List rootDisks = volumeDao.findNonDestroyedVolumesByPoolId(poolId); + List dataVolumes = volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK); List volumes = new ArrayList<>(); addVolumesToList(volumes, rootDisks); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 7e83d452bb9f..1e0138f7cf90 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1646,7 +1646,7 @@ protected void runInContext() { List pools = _storagePoolDao.listAll(); for (StoragePoolVO pool : pools) { - List volumes = _volsDao.findByPoolId(pool.getId(), null); + List volumes = _volsDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); for (VolumeVO volume : volumes) { if (!List.of(ImageFormat.QCOW2, ImageFormat.VHD, ImageFormat.OVA, ImageFormat.RAW).contains(volume.getFormat()) && !List.of(Storage.StoragePoolType.PowerFlex, Storage.StoragePoolType.FiberChannel).contains(pool.getPoolType())) { diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 8392c85527d0..13b7fbb00c20 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1558,17 +1558,21 @@ private boolean deleteDataStoreInternal(StoragePoolVO sPool, boolean forced) { protected String getStoragePoolNonDestroyedVolumesLog(long storagePoolId) { StringBuilder sb = new StringBuilder(); - List nonDestroyedVols = volumeDao.findByPoolId(storagePoolId, null); + List nonDestroyedVols = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); VMInstanceVO volInstance; List logMessageInfo = new ArrayList<>(); sb.append("["); for (VolumeVO vol : nonDestroyedVols) { - volInstance = _vmInstanceDao.findById(vol.getInstanceId()); - if (volInstance != null) { - logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + if (vol.getInstanceId() != null) { + volInstance = _vmInstanceDao.findById(vol.getInstanceId()); + if (volInstance != null) { + logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + } else { + logMessageInfo.add(String.format("Volume [%s] (attached VM with ID [%d] doesn't exists)", vol.getUuid(), vol.getInstanceId())); + } } else { - logMessageInfo.add(String.format("Volume [%s]", vol.getUuid())); + logMessageInfo.add(String.format("Volume [%s] (not attached to any VM)", vol.getUuid())); } } sb.append(String.join(", ", logMessageInfo)); @@ -2640,7 +2644,7 @@ private void handleRemoveChildStoragePoolFromDatastoreCluster(Set childD for (String childDatastoreUUID : childDatastoreUUIDs) { StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childDatastoreUUID); - List allVolumes = volumeDao.findByPoolId(dataStoreVO.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(dataStoreVO.getId()); allVolumes.removeIf(volumeVO -> volumeVO.getInstanceId() == null); allVolumes.removeIf(volumeVO -> volumeVO.getState() != Volume.State.Ready); for (VolumeVO volume : allVolumes) { diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 612582640f42..667af5a876f1 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -91,7 +91,7 @@ public boolean maintain(DataStore store) { boolean restart = !CollectionUtils.isEmpty(upPools); // 2. Get a list of all the ROOT volumes within this storage pool - List allVolumes = volumeDao.findByPoolId(pool.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId()); // 3. Enqueue to the work queue enqueueMigrationsForVolumes(allVolumes, pool); // 4. Process the queue diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index b00358caaa93..3e045f5a9057 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2261,7 +2261,7 @@ public HashMap getVolumeStatistics(long clusterId, Str private List getVolumesByHost(HostVO host, StoragePool pool){ List vmsPerHost = _vmInstanceDao.listByHostId(host.getId()); return vmsPerHost.stream() - .flatMap(vm -> _volsDao.findByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> + .flatMap(vm -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> vol.getState() == Volume.State.Ready ? (vol.getFormat() == ImageFormat.OVA ? vol.getChainInfo() : vol.getPath()) : null).filter(Objects::nonNull)) .collect(Collectors.toList()); } diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 414d41145f7b..5b7353bded6c 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -198,8 +198,8 @@ public void setup() throws Exception { rootDisks = Arrays.asList(rootDisk1, rootDisk2); dataDisks = Collections.singletonList(dataDisk); - when(volumeDao.findByPoolId(poolId)).thenReturn(rootDisks); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(rootDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); } @After @@ -564,22 +564,22 @@ public void testDestroyLocalStoragePoolVolumesBothRootDisksAndDataDisks() { @Test public void testDestroyLocalStoragePoolVolumesOnlyRootDisks() { - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(rootDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesOnlyDataDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(dataDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesNoDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, never()).updateAndRemoveVolume(any(VolumeVO.class)); } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 4a28e044d9c5..5f02c89339a0 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -531,7 +531,7 @@ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallation } @Test - public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumesReturnLog() { + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMAttachedLogs() { Mockito.doReturn(1L).when(storagePoolVOMock).getId(); Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); @@ -539,7 +539,7 @@ public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumesReturnLog Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); - Mockito.when(_volumeDao.findByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); @@ -548,6 +548,58 @@ public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumesReturnLog Assert.assertEquals(expected, log); } + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMLogForOneVolume() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (attached to VM [%s])]", volume1VOMock.getUuid(), volume2VOMock.getUuid(), vmInstanceVOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_NotAttachedLogs() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(null).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (not attached to any VM)]", volume1VOMock.getUuid(), volume2VOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMNotExistsLog() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(null).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (attached VM with ID [%d] doesn't exists), Volume [%s] (attached VM with ID [%d] doesn't exists)]", + volume1VOMock.getUuid(), volume1VOMock.getInstanceId(), volume2VOMock.getUuid(), volume2VOMock.getInstanceId()); + + Assert.assertEquals(expected, log); + } + private ChangeStoragePoolScopeCmd mockChangeStoragePooolScopeCmd(String newScope) { ChangeStoragePoolScopeCmd cmd = new ChangeStoragePoolScopeCmd(); ReflectionTestUtils.setField(cmd, "id", 1L); From 4adb7195701f2b28b86456f739ad59ec9f369abf Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 26 Jan 2026 04:18:12 -0500 Subject: [PATCH 010/117] Allow modification of user vm details if user.vm.readonly.details is empty (#10456) --- .../apache/cloudstack/query/QueryService.java | 2 +- .../framework/config/ConfigKey.java | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 828f9d5e064a..5181ebe2b763 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -118,7 +118,7 @@ public interface QueryService { ConfigKey UserVMReadOnlyDetails = new ConfigKey<>(String.class, "user.vm.readonly.details", "Advanced", "dataDiskController, rootDiskController", - "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); + "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null, ""); ConfigKey SortKeyAscending = new ConfigKey<>("Advanced", Boolean.class, "sortkey.algorithm", "true", "Sort algorithm - ascending or descending - to use. For entities that use sort key(template, disk offering, service offering, " + diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 00cf56345c8d..27b04ddf8937 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -120,10 +120,18 @@ public String toString() { static ConfigDepotImpl s_depot = null; - static public void init(ConfigDepotImpl depot) { + private String _defaultValueIfEmpty = null; + + public static void init(ConfigDepotImpl depot) { s_depot = depot; } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, + String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options, String defaultValueIfEmpty) { + this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, displayText, parent, group, subGroup, kind, options); + this._defaultValueIfEmpty = defaultValueIfEmpty; + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Scope scope) { this(type, name, category, defaultValue, description, isDynamic, scope, null); } @@ -216,7 +224,19 @@ public boolean isSameKeyAs(Object obj) { public T value() { if (_value == null || isDynamic()) { String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; - _value = valueOf((value == null) ? defaultValue() : value); + + String effective; + if (value != null) { + if (value.isEmpty() && _defaultValueIfEmpty != null) { + effective = _defaultValueIfEmpty; + } else { + effective = value; + } + } else { + effective = _defaultValueIfEmpty != null ? _defaultValueIfEmpty : defaultValue(); + } + + _value = valueOf(effective); } return _value; @@ -231,6 +251,10 @@ protected T valueInScope(Scope scope, Long id) { if (value == null) { return value(); } + + if (value.isEmpty() && _defaultValueIfEmpty != null) { + return valueOf(_defaultValueIfEmpty); + } return valueOf(value); } From 0958dfc13864315e83ae906bf4f90328ccd1557c Mon Sep 17 00:00:00 2001 From: Artem Sidorenko Date: Mon, 26 Jan 2026 10:21:47 +0100 Subject: [PATCH 011/117] Fix: proper permissions for systemvm template registrations on hardened systems (#12098) Related to https://github.com/apache/cloudstack/issues/10029#issuecomment-2531599607 We have umask 0077, so cloud-install-sys-tmplt is creating by default paths like below ``` $ ls -l /mnt/secondary/template/tmpl/ total 16 drwx------. 3 root root 4096 Nov 19 13:58 1 drwxrwxrwx. 7 root root 4096 Oct 31 09:42 2 drwxrwxrwx. 3 root root 4096 Oct 30 15:59 4 drwxr-xr-x. 2 root root 4096 Oct 31 10:21 5 $ ls -l /mnt/secondary/template/tmpl/1/ total 4 drwx------. 2 root root 4096 Nov 19 13:59 3 $ ls -l /mnt/secondary/template/tmpl/1/3/ total 549848 -rw-------. 1 root root 563032576 Nov 19 13:59 d23a1e19-c563-4f69-85ca-8721cf02082c.qcow2 -rw-------. 1 root root 287 Nov 19 13:59 template.properties ``` This results to the permissions problems later on, when trying to access the image Signed-off-by: Artem Sidorenko --- scripts/storage/secondary/cloud-install-sys-tmplt | 1 + scripts/storage/secondary/setup-sysvm-tmplt | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/storage/secondary/cloud-install-sys-tmplt b/scripts/storage/secondary/cloud-install-sys-tmplt index ad976c502c69..fc09dc968fff 100755 --- a/scripts/storage/secondary/cloud-install-sys-tmplt +++ b/scripts/storage/secondary/cloud-install-sys-tmplt @@ -44,6 +44,7 @@ failed() { } #set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments mflag= fflag= ext="vhd" diff --git a/scripts/storage/secondary/setup-sysvm-tmplt b/scripts/storage/secondary/setup-sysvm-tmplt index 06f0586fe342..63006cc4e4c2 100755 --- a/scripts/storage/secondary/setup-sysvm-tmplt +++ b/scripts/storage/secondary/setup-sysvm-tmplt @@ -19,6 +19,7 @@ # Usage: e.g. failed $? "this is an error" set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments failed() { local returnval=$1 From d010e9fcf29822a312618eee5d7a4c8f0eb69d2e Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Mon, 26 Jan 2026 15:03:30 +0530 Subject: [PATCH 012/117] Notify user if template upgrade is not required (#12483) --- .../cloud/network/router/VirtualNetworkApplianceManagerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e171b68399bf..7d0a4f208386 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -3358,6 +3358,7 @@ private List rebootRouters(final List routers) { jobIds.add(jobId); } else { logger.debug("Router: {} is already at the latest version. No upgrade required", router); + throw new CloudRuntimeException("Router is already at the latest version. No upgrade required"); } } return jobIds; From 315cd52fd1ac7fe461b9ceca6651cddca0382087 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 26 Jan 2026 16:23:47 +0530 Subject: [PATCH 013/117] snapshot: fix listSnapshots for volume which got delete and whose storage pool got deleted (#12433) This fixes the case when the storage pool is removed as well the KVM host and the subsequent volumes on the host. When that happened, listing snapshots (for recovery purposes) cause NPE as the pool_id was null, but last_pool_id for the related destroyed volume wasn't null. This adds a fallback logic. Signed-off-by: Rohit Yadav --- .../storage/snapshot/StorageSystemSnapshotStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java index a19397d03e32..560bb4b2fc12 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java @@ -951,7 +951,7 @@ public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperat VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId); - long volumeStoragePoolId = volumeVO.getPoolId(); + long volumeStoragePoolId = (volumeVO.getPoolId() != null ? volumeVO.getPoolId() : volumeVO.getLastPoolId()); if (SnapshotOperation.REVERT.equals(op)) { boolean baseVolumeExists = volumeVO.getRemoved() == null; From 63bdc2b990314f5443961a24b7522397a65bc81f Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Mon, 26 Jan 2026 16:25:55 +0530 Subject: [PATCH 014/117] Add log for null templateVO (#12406) --- .../cloudstack/storage/image/TemplateDataFactoryImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index c6430bcf9f93..3e1504beb3ad 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -296,6 +296,9 @@ public TemplateInfo getReadyBypassedTemplateOnManagedStorage(long templateId, Te @Override public boolean isTemplateMarkedForDirectDownload(long templateId) { VMTemplateVO templateVO = imageDataDao.findById(templateId); + if (templateVO == null) { + throw new CloudRuntimeException(String.format("Template not found with ID: %s", templateId)); + } return templateVO.isDirectDownload(); } } From 097c3a018bae6faf6cdac2db09c56507b980fa4f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 26 Jan 2026 11:56:14 +0100 Subject: [PATCH 015/117] ConfigDrive: use file absolute path instead of canonical path to create ISO (#11623) * ConfigDrive: use file absolute path instead of canonical path to create ISO * el8: add xorrisofs as option --- .../storage/configdrive/ConfigDriveBuilder.java | 4 ++-- .../storage/configdrive/ConfigDriveBuilderTest.java | 12 ++++++------ packaging/el8/cloud.spec | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 0b81a25b1cd0..15febbe972c4 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -231,9 +231,9 @@ static String getProgramToGenerateIso() throws IOException { throw new CloudRuntimeException("Cannot create ISO for config drive using any know tool. Known paths [/usr/bin/genisoimage, /usr/bin/mkisofs, /usr/local/bin/mkisofs]"); } if (!isoCreator.canExecute()) { - throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getCanonicalPath()); + throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getAbsolutePath()); } - return isoCreator.getCanonicalPath(); + return isoCreator.getAbsolutePath(); } /** diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index c04ff0a16015..03ceac843997 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -435,7 +435,7 @@ public void getProgramToGenerateIsoTestGenIsoExistsAndIsExecutable() throws Exce Mockito.verify(genIsoFileMock, Mockito.times(2)).exists(); Mockito.verify(genIsoFileMock).canExecute(); - Mockito.verify(genIsoFileMock).getCanonicalPath(); + Mockito.verify(genIsoFileMock).getAbsolutePath(); } } @@ -475,11 +475,11 @@ public void getProgramToGenerateIsoTestNotGenIsoMkIsoInLinux() throws Exception Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(2)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getAbsolutePath(); } } @@ -509,15 +509,15 @@ public void getProgramToGenerateIsoTestMkIsoMac() throws Exception { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getAbsolutePath(); } } diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 3d4851122669..507a6e64173f 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -76,7 +76,7 @@ Requires: sudo Requires: /sbin/service Requires: /sbin/chkconfig Requires: /usr/bin/ssh-keygen -Requires: (genisoimage or mkisofs) +Requires: (genisoimage or mkisofs or xorrisofs) Requires: ipmitool Requires: %{name}-common = %{_ver} Requires: (iptables-services or iptables) From 36edd92e480d8b738335c6500b2e90c4d3f91fb9 Mon Sep 17 00:00:00 2001 From: Henrique Sato Date: Mon, 26 Jan 2026 07:58:42 -0300 Subject: [PATCH 016/117] Fix snapshot physical size after migration (#12166) --- .../cloudstack/storage/image/SecondaryStorageServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java index 641a2a40dcd5..f739fecf9bf1 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -280,7 +280,7 @@ protected Void migrateDataCallBack(AsyncCallbackDispatcher Date: Mon, 26 Jan 2026 19:22:22 +0800 Subject: [PATCH 017/117] =?UTF-8?q?fix=20Sensitive=20Data=20Exposure=20Thr?= =?UTF-8?q?ough=20Exception=20Logging=20in=20OVM=20Hypervis=E2=80=A6=20(#1?= =?UTF-8?q?2032)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix Sensitive Data Exposure Through Exception Logging in OVM Hypervisor Configuration * extra ‘)’ in log. Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> * remove non-descriptive part Co-authored-by: Suresh Kumar Anaparti --------- Co-authored-by: chenyoulong20g@ict.ac.cn Co-authored-by: dahn Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Co-authored-by: Suresh Kumar Anaparti --- .../src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java index 9d958a9894a4..a65e4d778d37 100644 --- a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java +++ b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java @@ -362,7 +362,7 @@ protected void setupServer() throws IOException { sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s)", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "sh /usr/bin/configureOvm.sh postSetup")) { From bbc23a74683052228d254ba0e4958f4c19254f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernardo=20De=20Marco=20Gon=C3=A7alves?= Date: Mon, 26 Jan 2026 09:14:40 -0300 Subject: [PATCH 018/117] fix install path for systemvm templates when introducing new sec storage (#11605) --- .../cloudstack/storage/image/TemplateServiceImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 1bb954da4109..c18be7c73355 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -1318,9 +1318,10 @@ public void addSystemVMTemplatesToSecondary(DataStore store) { if (_vmTemplateStoreDao.isTemplateMarkedForDirectDownload(tmplt.getId())) { continue; } - tmpltStore = - new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null, - TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH + tmplt.getId() + '/', tmplt.getUrl()); + String templateDirectoryPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR; + String installPath = templateDirectoryPath + tmplt.getAccountId() + File.separator + tmplt.getId() + File.separator; + tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, + null, null, null, installPath, tmplt.getUrl()); tmpltStore.setSize(0L); tmpltStore.setPhysicalSize(0); // no size information for // pre-seeded system vm templates From 7536516e41636c9ae77ddda89c4f9827bfe55ba5 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Mon, 26 Jan 2026 18:12:43 +0530 Subject: [PATCH 019/117] add missing label text for label.aclname (#12511) --- ui/public/locales/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 64437a4d07c1..8bb7dba9bf54 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -47,6 +47,7 @@ "label.acl.rules": "ACL rules", "label.acl.reason.description": "Enter the reason behind an ACL rule.", "label.aclid": "ACL", +"label.aclname": "ACL name", "label.acl.rule.name": "ACL rule name", "label.acquire.new.ip": "Acquire new IP", "label.acquire.new.secondary.ip": "Acquire new secondary IP", From d50899427a70185bd0933bf386ae52aa0eab396b Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 26 Jan 2026 14:17:38 +0100 Subject: [PATCH 020/117] merge forward error --- .../src/main/java/com/cloud/resource/ResourceManagerImpl.java | 2 +- .../test/java/com/cloud/resource/ResourceManagerImplTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index ab3162e7a4b9..110353c4b482 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -2366,7 +2366,7 @@ public void updateStoragePoolConnectionsOnHosts(Long poolId, List storag List conflictingHostIds = new ArrayList<>(CollectionUtils.intersection(hostIdsToDisconnect, hostIdsUsingTheStoragePool)); if (CollectionUtils.isNotEmpty(conflictingHostIds)) { Map> hostVolumeMap = new HashMap<>(); - List volumesInPool = volumeDao.findByPoolId(poolId); + List volumesInPool = volumeDao.findNonDestroyedVolumesByPoolId(poolId); Map vmInstanceCache = new HashMap<>(); for (Long hostId : conflictingHostIds) { diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 6ed0774a4238..7e60c111ab2f 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -944,7 +944,7 @@ public void testUpdateStoragePoolConnectionsOnHosts_ConflictWithHostIdsAndVolume Mockito.when(volume2.getInstanceId()).thenReturn(101L); List volumesInPool = Arrays.asList(volume1, volume2); - Mockito.doReturn(volumesInPool).when(volumeDao).findByPoolId(poolId); + Mockito.doReturn(volumesInPool).when(volumeDao).findNonDestroyedVolumesByPoolId(poolId); VMInstanceVO vmInstance1 = Mockito.mock(VMInstanceVO.class); VMInstanceVO vmInstance2 = Mockito.mock(VMInstanceVO.class); From 88181ebe722c95bc628851c25b8b6fad06a14780 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Mon, 26 Jan 2026 23:59:31 +1000 Subject: [PATCH 021/117] Standardize and auto add license headers to all cfg files with pre-commit (#12230) --- .pre-commit-config.yaml | 10 +++++++ setup/dev/s3.cfg | 29 +++++++++--------- systemvm/debian/etc/haproxy/haproxy.cfg | 17 +++++++++++ .../devcloud-kvm-advanced-fusion.cfg | 30 +++++++++---------- tools/devcloud-kvm/devcloud-kvm-advanced.cfg | 30 +++++++++---------- tools/devcloud-kvm/devcloud-kvm.cfg | 30 +++++++++---------- tools/devcloud4/advanced/marvin.cfg | 30 +++++++++---------- tools/devcloud4/basic/marvin.cfg | 30 +++++++++---------- 8 files changed, 112 insertions(+), 94 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26adafcbf268..49829caf125e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,16 @@ repos: args: ['644'] files: \.md$ stages: [manual] + - id: insert-license + name: add license for all cfg files + description: automatically adds a licence header to all cfg files that don't have a license header + files: \.cfg$ + args: + - --comment-style + - '|#|' + - --license-filepath + - .github/workflows/license-templates/LICENSE.txt + - --fuzzy-match-generates-todo - id: insert-license name: add license for all Markdown files files: \.md$ diff --git a/setup/dev/s3.cfg b/setup/dev/s3.cfg index de28e5b2698c..ce414f584cf1 100644 --- a/setup/dev/s3.cfg +++ b/setup/dev/s3.cfg @@ -1,20 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # TODO: Change ACCESS_KEY/ SECRET_KEY to your credentials on the object store diff --git a/systemvm/debian/etc/haproxy/haproxy.cfg b/systemvm/debian/etc/haproxy/haproxy.cfg index 21964f297c25..68a4cd7cd585 100644 --- a/systemvm/debian/etc/haproxy/haproxy.cfg +++ b/systemvm/debian/etc/haproxy/haproxy.cfg @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + global log 127.0.0.1:3914 local0 info chroot /var/lib/haproxy diff --git a/tools/devcloud-kvm/devcloud-kvm-advanced-fusion.cfg b/tools/devcloud-kvm/devcloud-kvm-advanced-fusion.cfg index b1a3418e5d30..ce3ec91bbcfb 100644 --- a/tools/devcloud-kvm/devcloud-kvm-advanced-fusion.cfg +++ b/tools/devcloud-kvm/devcloud-kvm-advanced-fusion.cfg @@ -1,21 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. { "zones": [ diff --git a/tools/devcloud-kvm/devcloud-kvm-advanced.cfg b/tools/devcloud-kvm/devcloud-kvm-advanced.cfg index a3a41da874f0..60ad8b58b9ff 100644 --- a/tools/devcloud-kvm/devcloud-kvm-advanced.cfg +++ b/tools/devcloud-kvm/devcloud-kvm-advanced.cfg @@ -1,21 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. { "zones": [ diff --git a/tools/devcloud-kvm/devcloud-kvm.cfg b/tools/devcloud-kvm/devcloud-kvm.cfg index ffd23504ffec..5ac417a13dc7 100644 --- a/tools/devcloud-kvm/devcloud-kvm.cfg +++ b/tools/devcloud-kvm/devcloud-kvm.cfg @@ -1,20 +1,20 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # This is a stock devcloud config converted from the file # tools/devcloud/devcloud.cfg. diff --git a/tools/devcloud4/advanced/marvin.cfg b/tools/devcloud4/advanced/marvin.cfg index 222dc65d0452..7b6e656e6204 100644 --- a/tools/devcloud4/advanced/marvin.cfg +++ b/tools/devcloud4/advanced/marvin.cfg @@ -1,21 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. { "zones": [ diff --git a/tools/devcloud4/basic/marvin.cfg b/tools/devcloud4/basic/marvin.cfg index 1c8ee547b265..9b7d73c381b1 100644 --- a/tools/devcloud4/basic/marvin.cfg +++ b/tools/devcloud4/basic/marvin.cfg @@ -1,21 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. { "zones": [ From 63c8b5fc5627fd0de6d05609625e826de1a6f677 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 26 Jan 2026 15:23:27 +0100 Subject: [PATCH 022/117] api/server: support deploy-as-is template as VNF template (#12499) --- .../user/vm/DeployVnfApplianceCmd.java | 2 +- .../storage/template/VnfTemplateManager.java | 4 +++- .../storage/template/VnfTemplateUtils.java | 18 ++++++++++++++++ .../cloud/template/TemplateManagerImpl.java | 9 ++++++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 2 +- .../template/VnfTemplateManagerImpl.java | 21 ++++++++++++++++++- .../template/TemplateManagerImplTest.java | 8 +++++++ .../com/cloud/vm/UserVmManagerImplTest.java | 6 +++--- .../template/VnfTemplateManagerImplTest.java | 8 +++---- ui/public/locales/en.json | 2 +- ui/src/views/compute/DeployVnfAppliance.vue | 14 +++++++++++-- .../views/compute/wizard/VnfNicsSelection.vue | 5 +++++ 12 files changed, 85 insertions(+), 14 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java index 4d50dd9c39bf..92ddfd5b2357 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java @@ -43,7 +43,7 @@ public class DeployVnfApplianceCmd extends DeployVMCmd implements UserCmd { @Parameter(name = ApiConstants.VNF_CONFIGURE_MANAGEMENT, type = CommandType.BOOLEAN, required = false, - description = "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. " + + description = "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. " + "Network rules are configured if management network is an isolated network or shared network with security groups.") private Boolean vnfConfigureManagement; diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java index 6571346ad654..3df59811561b 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.framework.config.ConfigKey; import java.util.List; +import java.util.Map; public interface VnfTemplateManager { @@ -42,11 +43,12 @@ public interface VnfTemplateManager { void updateVnfTemplate(long templateId, UpdateVnfTemplateCmd cmd); - void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds); + void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap); SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd); void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, UserVm vm, DeployVnfApplianceCmd cmd) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException; + } diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java index e997a50cec03..16ff2abb564a 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.storage.template; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.VNF; import com.cloud.storage.Storage; @@ -124,6 +125,9 @@ public static void validateVnfNics(List nicsList) { public static void validateApiCommandParams(BaseCmd cmd, VirtualMachineTemplate template) { if (cmd instanceof RegisterVnfTemplateCmd) { RegisterVnfTemplateCmd registerCmd = (RegisterVnfTemplateCmd) cmd; + if (registerCmd.isDeployAsIs() && CollectionUtils.isNotEmpty(registerCmd.getVnfNics())) { + throw new InvalidParameterValueException("VNF nics cannot be specified when register a deploy-as-is Template. Please wait until Template settings are read from OVA."); + } validateApiCommandParams(registerCmd.getVnfDetails(), registerCmd.getVnfNics(), registerCmd.getTemplateType()); } else if (cmd instanceof UpdateVnfTemplateCmd) { UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd; @@ -149,4 +153,18 @@ public static void validateVnfCidrList(List cidrList) { } } } + + public static void validateDeployAsIsTemplateVnfNics(List ovfNetworks, List vnfNics) { + if (CollectionUtils.isEmpty(vnfNics)) { + return; + } + if (CollectionUtils.isEmpty(ovfNetworks)) { + throw new InvalidParameterValueException("The list of networks read from OVA is empty. Please wait until the template is fully downloaded and processed."); + } + for (VNF.VnfNic vnfNic : vnfNics) { + if (vnfNic.getDeviceId() < ovfNetworks.size() && !vnfNic.isRequired()) { + throw new InvalidParameterValueException(String.format("The VNF nic [device ID: %s ] is required as it is defined in the OVA template.", vnfNic.getDeviceId())); + } + } + } } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index ba8e57141803..2c7d2d593e3e 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -122,6 +122,7 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; @@ -131,6 +132,7 @@ import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.deploy.DeployDestination; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -313,6 +315,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, protected SnapshotHelper snapshotHelper; @Inject VnfTemplateManager vnfTemplateManager; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; @@ -2172,6 +2176,11 @@ private VMTemplateVO updateTemplateOrIso(BaseUpdateTemplateOrIsoCmd cmd) { templateType = validateTemplateType(cmd, isAdmin, template.isCrossZones()); if (cmd instanceof UpdateVnfTemplateCmd) { VnfTemplateUtils.validateApiCommandParams(cmd, template); + UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd; + if (template.isDeployAsIs() && CollectionUtils.isNotEmpty(updateCmd.getVnfNics())) { + List ovfNetworks = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); + VnfTemplateUtils.validateDeployAsIsTemplateVnfNics(ovfNetworks, updateCmd.getVnfNics()); + } vnfTemplateManager.updateVnfTemplate(template.getId(), (UpdateVnfTemplateCmd) cmd); } templateTag = ((UpdateTemplateCmd)cmd).getTemplateTag(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 3e045f5a9057..815ac4f70fe8 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -6127,7 +6127,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE throw new InvalidParameterValueException("Unable to use template " + templateId); } if (TemplateType.VNF.equals(template.getTemplateType())) { - vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds()); + vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds(), cmd.getVmNetworkMap()); } else if (cmd instanceof DeployVnfApplianceCmd) { throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template"); } diff --git a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java index ef0f6f6b226d..0ebff237a44e 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java @@ -201,7 +201,14 @@ public ConfigKey[] getConfigKeys() { } @Override - public void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds) { + public void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap) { + if (template.isDeployAsIs()) { + if (CollectionUtils.isNotEmpty(networkIds)) { + throw new InvalidParameterValueException("VNF nics mappings should be empty for deploy-as-is templates"); + } + validateVnfApplianceNetworksMap(template, vmNetworkMap); + return; + } if (CollectionUtils.isEmpty(networkIds)) { throw new InvalidParameterValueException("VNF nics list is empty"); } @@ -213,6 +220,18 @@ public void validateVnfApplianceNics(VirtualMachineTemplate template, List } } + private void validateVnfApplianceNetworksMap(VirtualMachineTemplate template, Map vmNetworkMap) { + if (MapUtils.isEmpty(vmNetworkMap)) { + throw new InvalidParameterValueException("VNF networks map is empty"); + } + List vnfNics = vnfTemplateNicDao.listByTemplateId(template.getId()); + for (VnfTemplateNicVO vnfNic : vnfNics) { + if (vnfNic.isRequired() && vmNetworkMap.size() <= vnfNic.getDeviceId()) { + throw new InvalidParameterValueException("VNF nic is required but not found: " + vnfNic); + } + } + } + protected Set getOpenPortsForVnfAppliance(VirtualMachineTemplate template) { Set ports = new HashSet<>(); VnfTemplateDetailVO accessMethodsDetail = vnfTemplateDetailsDao.findDetail(template.getId(), VNF.AccessDetail.ACCESS_METHODS.name().toLowerCase()); diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index 98b1c05dba83..9680fe5e1fd4 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -23,6 +23,7 @@ import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.domain.dao.DomainDao; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; @@ -204,6 +205,8 @@ public class TemplateManagerImplTest { AccountManager _accountMgr; @Inject VnfTemplateManager vnfTemplateManager; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject HeuristicRuleHelper heuristicRuleHelperMock; @@ -956,6 +959,11 @@ public VnfTemplateManager vnfTemplateManager() { return Mockito.mock(VnfTemplateManager.class); } + @Bean + public TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao() { + return Mockito.mock(TemplateDeployAsIsDetailsDao.class); + } + @Bean public SnapshotHelper snapshotHelper() { return Mockito.mock(SnapshotHelper.class); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index ac1ecaa456b0..570c57cb68d1 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -1079,7 +1079,7 @@ public void createVirtualMachine() throws ResourceUnavailableException, Insuffic when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class)); ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class); when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock); @@ -1091,7 +1091,7 @@ public void createVirtualMachine() throws ResourceUnavailableException, Insuffic UserVm result = userVmManagerImpl.createVirtualMachine(deployVMCmd); assertEquals(userVmVoMock, result); - Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null); + Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null, null); Mockito.verify(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any()); @@ -1335,7 +1335,7 @@ public void createVirtualMachineWithCloudRuntimeException() throws ResourceUnava when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class)); ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class); when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock); diff --git a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java index c3fa0d62604c..b9565ebb2922 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java @@ -228,25 +228,25 @@ public void testPersistVnfTemplateUpdateWithoutDetails() { @Test public void testValidateVnfApplianceNicsWithRequiredNics() { List networkIds = Arrays.asList(200L, 201L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test public void testValidateVnfApplianceNicsWithAllNics() { List networkIds = Arrays.asList(200L, 201L, 202L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test(expected = InvalidParameterValueException.class) public void testValidateVnfApplianceNicsWithEmptyList() { List networkIds = new ArrayList<>(); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test(expected = InvalidParameterValueException.class) public void testValidateVnfApplianceNicsWithMissingNetworkId() { List networkIds = Arrays.asList(200L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 8bb7dba9bf54..b2465fa325f4 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2529,7 +2529,7 @@ "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from", "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.", "label.vnf.configure.management": "Configure network rules for VNF's management interfaces", -"label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", +"label.vnf.configure.management.tooltip": "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", "label.vnf.detail.add": "Add VNF detail", "label.vnf.detail.remove": "Remove VNF detail", "label.vnf.details": "VNF Details", diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index fec1139ab9b6..9b09de5a1868 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -372,6 +372,7 @@
@@ -1293,7 +1294,8 @@ export default { return tabList }, showVnfNicsSection () { - return this.networks && this.networks.length > 0 && this.vm.templateid && this.templateVnfNics && this.templateVnfNics.length > 0 + return ((this.networks && this.networks.length > 0) || (this.templateNics && this.templateNics.length > 0)) && + this.vm.templateid && this.templateVnfNics && this.templateVnfNics.length > 0 }, showVnfConfigureManagement () { const managementDeviceIds = [] @@ -1303,6 +1305,11 @@ export default { } } for (const deviceId of managementDeviceIds) { + if (this.templateNics && this.templateNics[deviceId] && + ((this.templateNics[deviceId].selectednetworktype === 'Isolated' && this.templateNics[deviceId].selectednetworkvpcid === undefined) || + (this.templateNics[deviceId].selectednetworktype === 'Shared' && this.templateNics[deviceId].selectednetworkwithsg))) { + return true + } if (this.vnfNicNetworks && this.vnfNicNetworks[deviceId] && ((this.vnfNicNetworks[deviceId].type === 'Isolated' && this.vnfNicNetworks[deviceId].vpcid === undefined) || (this.vnfNicNetworks[deviceId].type === 'Shared' && this.vnfNicNetworks[deviceId].service.filter(svc => svc.name === 'SecurityGroupProvider')))) { @@ -2005,7 +2012,7 @@ export default { // All checked networks should be used and only once. // Required NIC must be associated to a network // DeviceID must be consequent - if (this.templateVnfNics && this.templateVnfNics.length > 0) { + if (this.templateVnfNics && this.templateVnfNics.length > 0 && (!this.templateNics || this.templateNics.length === 0)) { let nextDeviceId = 0 const usedNetworkIds = [] const keys = Object.keys(this.vnfNicNetworks) @@ -2629,6 +2636,9 @@ export default { var network = this.options.networks[Math.min(i, this.options.networks.length - 1)] nic.selectednetworkid = network.id nic.selectednetworkname = network.name + nic.selectednetworktype = network.type + nic.selectednetworkvpcid = network.vpcid + nic.selectednetworkwithsg = network.service.filter(svc => svc.name === 'SecurityGroupProvider').length > 0 this.nicToNetworkSelection.push({ nic: nic.id, network: network.id }) } } diff --git a/ui/src/views/compute/wizard/VnfNicsSelection.vue b/ui/src/views/compute/wizard/VnfNicsSelection.vue index fdd5276b4f6e..40bdc1c676a8 100644 --- a/ui/src/views/compute/wizard/VnfNicsSelection.vue +++ b/ui/src/views/compute/wizard/VnfNicsSelection.vue @@ -50,6 +50,7 @@ @@ -140,6 +141,13 @@ export default { handleSearch (value) { this.filter = value this.fetchData() + }, + handleConfigRefresh (name, updatedRecord) { + if (!name || !updatedRecord) return + const index = this.items.findIndex(item => item.name === name) + if (index !== -1) { + this.items.splice(index, 1, updatedRecord) + } } } } diff --git a/ui/src/views/setting/ConfigurationHierarchy.vue b/ui/src/views/setting/ConfigurationHierarchy.vue index 80b464e657cc..815a048bc257 100644 --- a/ui/src/views/setting/ConfigurationHierarchy.vue +++ b/ui/src/views/setting/ConfigurationHierarchy.vue @@ -34,7 +34,7 @@ {{ record.description }} @@ -83,6 +83,9 @@ export default { return 'light-row' } return 'dark-row' + }, + handleConfigRefresh (name, updatedRecord) { + this.$emit('refresh-config', name, updatedRecord) } } } diff --git a/ui/src/views/setting/ConfigurationTab.vue b/ui/src/views/setting/ConfigurationTab.vue index 75905cbd1748..65b256c94c95 100644 --- a/ui/src/views/setting/ConfigurationTab.vue +++ b/ui/src/views/setting/ConfigurationTab.vue @@ -58,7 +58,8 @@ :count="count" :page="page" :pagesize="pagesize" - @change-page="changePage" /> + @change-page="changePage" + @refresh-config="handleConfigRefresh" /> + :config="config" + @refresh-config="handleConfigRefresh" /> @@ -322,6 +324,13 @@ export default { '#' + this.$route.path ) } + }, + handleConfigRefresh (name, updatedRecord) { + if (!name || !updatedRecord) return + const index = this.config.findIndex(item => item.name === name) + if (index !== -1) { + this.config.splice(index, 1, updatedRecord) + } } } } diff --git a/ui/src/views/setting/ConfigurationTable.vue b/ui/src/views/setting/ConfigurationTable.vue index da05b9342a0a..7edc1b1aad62 100644 --- a/ui/src/views/setting/ConfigurationTable.vue +++ b/ui/src/views/setting/ConfigurationTable.vue @@ -32,7 +32,10 @@ {{record.displaytext }} {{ ' (' + record.name + ')' }}
{{ record.description }} @@ -113,6 +116,9 @@ export default { return 'config-light-row' } return 'config-dark-row' + }, + handleConfigRefresh (name, updatedRecord) { + this.$emit('refresh-config', name, updatedRecord) } } } diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue index 662e5ef142e5..531d4e0ea61a 100644 --- a/ui/src/views/setting/ConfigurationValue.vue +++ b/ui/src/views/setting/ConfigurationValue.vue @@ -187,7 +187,7 @@ @onClick="$resetConfigurationValueConfirm(configrecord, resetConfigurationValue)" v-if="editableValueKey === null" icon="reload-outlined" - :disabled="(!('resetConfiguration' in $store.getters.apis) || configDisabled || valueLoading)" /> + :disabled="(!('resetConfiguration' in $store.getters.apis) || configDisabled || valueLoading || configrecord.value === configrecord.defaultvalue)" /> @@ -273,6 +273,7 @@ export default { this.editableValueKey = null }, updateConfigurationValue (configrecord) { + let configRecordEntry = this.configrecord this.valueLoading = true this.editableValueKey = null var newValue = this.editableValue @@ -294,7 +295,8 @@ export default { params[this.scopeKey] = this.resource?.id } postAPI('updateConfiguration', params).then(json => { - this.editableValue = this.getEditableValue(json.updateconfigurationresponse.configuration) + configRecordEntry = json.updateconfigurationresponse.configuration + this.editableValue = this.getEditableValue(configRecordEntry) this.actualValue = this.editableValue this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') @@ -318,10 +320,11 @@ export default { }) }).finally(() => { this.valueLoading = false - this.$emit('refresh') + this.$emit('refresh', configrecord.name, configRecordEntry) }) }, resetConfigurationValue (configrecord) { + let configRecordEntry = this.configrecord this.valueLoading = true this.editableValueKey = null const params = { @@ -332,7 +335,8 @@ export default { params[this.scopeKey] = this.resource?.id } postAPI('resetConfiguration', params).then(json => { - this.editableValue = this.getEditableValue(json.resetconfigurationresponse.configuration) + configRecordEntry = json.resetconfigurationresponse.configuration + this.editableValue = this.getEditableValue(configRecordEntry) this.actualValue = this.editableValue var newValue = this.editableValue if (configrecord.type === 'Range') { @@ -360,7 +364,7 @@ export default { }) }).finally(() => { this.valueLoading = false - this.$emit('refresh') + this.$emit('refresh', configrecord.name, configRecordEntry) }) }, getEditableValue (configrecord) { From dd0b863e22579caa77e399855df690d350c80d3e Mon Sep 17 00:00:00 2001 From: Edward-x <30854794+YLChen-007@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:41:23 +0800 Subject: [PATCH 047/117] sensitive information leak to log (#12018) * sensitive information leak to log * Update agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java * Update core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java * Update engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java * Update utils/src/main/java/com/cloud/utils/UriUtils.java Co-authored-by: dahn * Update plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> * Sync with 4.20 and fix conflict in BaremetalPingPxeResource * Apply suggestions from code review Co-authored-by: Suresh Kumar Anaparti --------- Co-authored-by: chenyoulong20g@ict.ac.cn Co-authored-by: dahn Co-authored-by: dahn Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Co-authored-by: Suresh Kumar Anaparti --- .../resource/consoleproxy/ConsoleProxyResource.java | 2 +- .../networkservice/BaremetalDnsmasqResource.java | 4 ++-- .../networkservice/BaremetalKickStartPxeResource.java | 6 +++--- .../networkservice/BaremetalPingPxeResource.java | 10 +++++----- utils/src/main/java/com/cloud/utils/UriUtils.java | 8 ++++++-- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java index b0b1e487a263..83b11418f2c5 100644 --- a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java +++ b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java @@ -331,7 +331,7 @@ private void launchConsoleProxy(final byte[] ksBits, final String ksPassword, fi final Object resource = this; logger.info("Building class loader for com.cloud.consoleproxy.ConsoleProxy"); if (consoleProxyMain == null) { - logger.info("Running com.cloud.consoleproxy.ConsoleProxy with encryptor password={}", encryptorPassword); + logger.info("Running com.cloud.consoleproxy.ConsoleProxy"); consoleProxyMain = new Thread(new ManagedContextRunnable() { @Override protected void runInContext() { diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java index 51acfe93d39e..8e7efedfca3b 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java @@ -46,10 +46,10 @@ public boolean configure(String name, Map params) throws Configu com.trilead.ssh2.Connection sshConnection = null; try { super.configure(name, params); - logger.debug(String.format("Trying to connect to DHCP server(IP=%1$s, username=%2$s, password=%3$s)", _ip, _username, _password)); + logger.debug(String.format("Trying to connect to DHCP server(IP=%1$s, username=%2$s", _ip, _username)); sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new ConfigurationException(String.format("Cannot connect to DHCP server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to DHCP server(IP=%1$s, username=%2$s", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "[ -f '/usr/sbin/dnsmasq' ]")) { diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java index 3775f4effc17..88c4dea96b3d 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java @@ -130,8 +130,8 @@ private Answer execute(VmDataCommand cmd) { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { - logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + logger.debug("SSH Failed to authenticate with user {} credentials", _username); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = String.format("python /usr/bin/baremetal_user_data.py '%s'", arg); @@ -167,7 +167,7 @@ private Answer execute(PrepareKickstartPxeServerCommand cmd) { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String copyTo = String.format("%s/%s", _tftpDir, cmd.getTemplateUuid()); diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java index 96b2dbfeb935..a54cd4a1a118 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java @@ -101,7 +101,7 @@ public boolean configure(String name, Map params) throws Configu sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, "******")); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=******", _ip, _username)); } String cmd = String.format("[ -f /%1$s/pxelinux.0 ] && [ -f /%2$s/kernel ] && [ -f /%3$s/initrd.gz ] ", _tftpDir, _tftpDir, _tftpDir); @@ -150,8 +150,8 @@ protected PreparePxeServerAnswer execute(PreparePxeServerCommand cmd) { try { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { - logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + logger.debug("SSH Failed to authenticate with user {} credentials", _username); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = @@ -179,7 +179,7 @@ protected Answer execute(PrepareCreateTemplateCommand cmd) { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = @@ -237,7 +237,7 @@ private Answer execute(VmDataCommand cmd) { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = String.format("python /usr/bin/baremetal_user_data.py '%s'", arg); diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 961c121597f5..4722e3c540ac 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -500,8 +500,12 @@ public static InputStream getInputStreamFromUrl(String url, String user, String if ((user != null) && (password != null)) { httpclient.getParams().setAuthenticationPreemptive(true); Credentials defaultcreds = new UsernamePasswordCredentials(user, password); - httpclient.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); - LOGGER.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); + httpclient.getState().setCredentials( + new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); + LOGGER.info("Added username={} along with password for host {}:{}" + , user + , hostAndPort.first() + , hostAndPort.second()); } // Execute the method. GetMethod method = new GetMethod(url); From 66665b883c1392ebb0af09544681971dcc00e046 Mon Sep 17 00:00:00 2001 From: Tonitzpp <134986282+Tonitzpp@users.noreply.github.com> Date: Wed, 28 Jan 2026 01:42:57 -0300 Subject: [PATCH 048/117] Changed error message when snapshot is not on secondary when trying to perform download (#12462) Co-authored-by: toni.zamparetti --- .../java/com/cloud/storage/snapshot/SnapshotManagerImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index ff9989acac3e..1db486584462 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -578,8 +578,9 @@ public String extractSnapshot(ExtractSnapshotCmd cmd) { } if (ObjectUtils.anyNull(chosenStore, snapshotDataStoreReference)) { - logger.error("Snapshot [{}] not found in any secondary storage.", snapshot); - throw new InvalidParameterValueException("Snapshot not found."); + String errorMessage = String.format("Snapshot [%s] not found in any secondary storage. The snapshot may be on primary storage, where it cannot be downloaded.", snapshot.getUuid()); + logger.error(errorMessage); + throw new InvalidParameterValueException(errorMessage); } snapshotSrv.syncVolumeSnapshotsToRegionStore(snapshot.getVolumeId(), chosenStore); From 062b98a51eca7da6b6083cbb31c2fb0608448386 Mon Sep 17 00:00:00 2001 From: cheng102e <38267524+cheng102e@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:45:11 +0800 Subject: [PATCH 049/117] fix: clean magic value, and update if-else to switch (#8848) * fix: clean magic value, and update if-else to switch * fix: return the (String args[]) * review --------- Co-authored-by: jiejc1 Co-authored-by: Suresh Kumar Anaparti --- .../src/main/java/common/Client.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/services/console-proxy/rdpconsole/src/main/java/common/Client.java b/services/console-proxy/rdpconsole/src/main/java/common/Client.java index 742f5c9f0cd4..972d5d753e8f 100644 --- a/services/console-proxy/rdpconsole/src/main/java/common/Client.java +++ b/services/console-proxy/rdpconsole/src/main/java/common/Client.java @@ -210,7 +210,6 @@ private void help() { public void runClient(String[] args) { try { - Protocol protocol = parseOptions(args); if (protocol == Protocol.NONE) return; @@ -299,21 +298,28 @@ private Element setMainElementAndAddressBasedOnProtocol(Protocol protocol, SSLSt private Protocol parseOptions(String[] args) { String protocolName = (args.length > 0) ? args[0] : ""; - Protocol protocol = Protocol.NONE; + Protocol protocol; Option[] options; - if (protocolName.equals("vnc")) { - protocol = Protocol.VNC; - options = join(commonOptions, vncOptions); - } else if (protocolName.equals("rdp")) { - protocol = Protocol.RDP; - options = join(commonOptions, rdpOptions); - } else if (protocolName.equals("hyperv")) { - protocol = Protocol.HYPERV; - options = join(commonOptions, hyperVOptions); - } else { - help(); - return Protocol.NONE; + try { + protocol = Protocol.valueOf(protocolName); + } catch (IllegalArgumentException e) { + protocol = Protocol.NONE; + } + + switch (protocol) { + case VNC: + options = join(commonOptions, vncOptions); + break; + case RDP: + options = join(commonOptions, rdpOptions); + break; + case HYPERV: + options = join(commonOptions, hyperVOptions); + break; + default: + help(); + return Protocol.NONE; } // Parse all options for given protocol From 21d5c10850111cd2e76d783b95b090da623ae024 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 28 Jan 2026 10:55:59 +0530 Subject: [PATCH 050/117] Apply reordered ACL list to VR router (#12525) This PR address #9398 --- .../network/element/VpcVirtualRouterElement.java | 10 +++++++++- .../cloud/network/vpc/NetworkACLServiceImpl.java | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java index 3d613fca18ea..f393ef8a129d 100644 --- a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java @@ -550,7 +550,15 @@ public boolean applyNetworkACLs(final Network network, final List networks, List networkACLItems) { - return true; + boolean result = true; + try { + for (Network network : networks) { + result = result && applyNetworkACLs(network, networkACLItems); + } + } catch (ResourceUnavailableException ex) { + result = false; + } + return result; } @Override diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java index ecb164018ac0..7460ae87d44c 100644 --- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java @@ -109,6 +109,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ private NsxProviderDao nsxProviderDao; @Inject private NetrisProviderDao netrisProviderDao; + @Inject + private VpcManager vpcManager; private String supportedProtocolsForAclRules = "tcp,udp,icmp,all"; @@ -1037,13 +1039,20 @@ public NetworkACLItem moveNetworkAclRuleToNewPosition(MoveNetworkAclItemCmd move if (Objects.isNull(vpc)) { return networkACLItem; } + List networks = _networkDao.listByAclId(lockedAcl.getId()); + if (networks.isEmpty()) { + return networkACLItem; + } + final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId()); final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId()); final NetrisProviderVO netrisProvider = netrisProviderDao.findByZoneId(dc.getId()); - List networks = _networkDao.listByAclId(lockedAcl.getId()); - if (ObjectUtils.anyNotNull(nsxProvider, netrisProvider) && !networks.isEmpty()) { + boolean isVpcNetworkACLProvider = vpcManager.isProviderSupportServiceInVpc(vpc.getId(), Network.Service.NetworkACL, Network.Provider.VPCVirtualRouter); + + if (ObjectUtils.anyNotNull(nsxProvider, netrisProvider) || isVpcNetworkACLProvider) { allAclRules = getAllAclRulesSortedByNumber(lockedAcl.getId()); - Network.Provider networkProvider = nsxProvider != null ? Network.Provider.Nsx : Network.Provider.Netris; + Network.Provider networkProvider = isVpcNetworkACLProvider ? Network.Provider.VPCVirtualRouter + : (nsxProvider != null ? Network.Provider.Nsx : Network.Provider.Netris); _networkAclMgr.reorderAclRules(vpc, networks, allAclRules, networkProvider); } return networkACLItem; From 572aa1956493d58a0efa283d2f75d876d92a5ad9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Jan 2026 11:01:53 +0530 Subject: [PATCH 051/117] ui: show usage server restart message on usage config change (#11969) Fixes #10853 --------- Signed-off-by: Abhishek Kumar --- ui/public/locales/en.json | 1 + ui/src/components/view/ListView.vue | 10 +--------- ui/src/utils/plugins.js | 11 +++++++++++ ui/src/views/setting/ConfigurationValue.vue | 20 ++------------------ 4 files changed, 15 insertions(+), 27 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 275b8dbb0fe5..747154964345 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -3735,6 +3735,7 @@ "message.resource.not.found": "Resource not found.", "message.restart.mgmt.server": "Please restart your management server(s) for your new settings to take effect.", "message.restart.network": "All services provided by this Network will be interrupted. Please confirm that you want to restart this Network.", +"message.restart.usage.server": "Please restart your usage server(s) for your new settings to take effect.", "message.restart.vm.to.update.settings": "Update in fields other than name and display name will require the Instance to be restarted.", "message.restart.vpc": "Please confirm that you want to restart the VPC.", "message.restart.vpc.remark": "Please confirm that you want to restart the VPC

Remark: making a non-redundant VPC redundant will force a clean up. The Networks will not be available for a couple of minutes.

", diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 168e355cbc80..79ec5a182071 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -1234,15 +1234,7 @@ export default { this.editableValueKey = null this.$store.dispatch('RefreshFeatures') this.$messageConfigSuccess(`${this.$t('message.setting.updated')} ${record.name}`, record) - if (json.updateconfigurationresponse && - json.updateconfigurationresponse.configuration && - !json.updateconfigurationresponse.configuration.isdynamic && - ['Admin'].includes(this.$store.getters.userInfo.roletype)) { - this.$notification.warning({ - message: this.$t('label.status'), - description: this.$t('message.restart.mgmt.server') - }) - } + this.$notifyConfigurationValueChange(json?.updateconfigurationresponse?.configuration || null) }).catch(error => { console.error(error) this.$message.error(this.$t('message.error.save.setting')) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 306eb9d1f594..729cef84d021 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -550,6 +550,17 @@ export const dialogUtilPlugin = { onOk: () => callback(configRecord) }) } + + app.config.globalProperties.$notifyConfigurationValueChange = function (configRecord) { + if (!configRecord || configRecord.isdynamic || store.getters.userInfo?.roletype !== 'Admin') { + return + } + const server = configRecord.group === 'Usage Server' ? 'usage' : 'mgmt' + this.$notification.warning({ + message: this.$t('label.status'), + description: this.$t('message.restart.' + server + '.server') + }) + } } } diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue index 662e5ef142e5..31c0798a7178 100644 --- a/ui/src/views/setting/ConfigurationValue.vue +++ b/ui/src/views/setting/ConfigurationValue.vue @@ -299,15 +299,7 @@ export default { this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') this.$messageConfigSuccess(`${this.$t('message.setting.updated')} ${configrecord.name}`, configrecord) - if (json.updateconfigurationresponse && - json.updateconfigurationresponse.configuration && - !json.updateconfigurationresponse.configuration.isdynamic && - ['Admin'].includes(this.$store.getters.userInfo.roletype)) { - this.$notification.warning({ - message: this.$t('label.status'), - description: this.$t('message.restart.mgmt.server') - }) - } + this.$notifyConfigurationValueChange(json?.updateconfigurationresponse?.configuration || null) }).catch(error => { this.editableValue = this.actualValue console.error(error) @@ -341,15 +333,7 @@ export default { this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') this.$messageConfigSuccess(`${this.$t('label.setting')} ${configrecord.name} ${this.$t('label.reset.config.value')}`, configrecord) - if (json.resetconfigurationresponse && - json.resetconfigurationresponse.configuration && - !json.resetconfigurationresponse.configuration.isdynamic && - ['Admin'].includes(this.$store.getters.userInfo.roletype)) { - this.$notification.warning({ - message: this.$t('label.status'), - description: this.$t('message.restart.mgmt.server') - }) - } + this.$notifyConfigurationValueChange(json?.resetconfigurationresponse?.configuration || null) }).catch(error => { this.editableValue = this.actualValue console.error(error) From 2bfc9cb8eb621cead6b6c8067a878126f1b0c7b5 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 28 Jan 2026 06:47:14 +0100 Subject: [PATCH 052/117] CKS: skip default egress policy check for vpc network offerings (#11998) This PR fixes #11995 Steps to reproduce the issue - create a vpc - create a vpc tier with default offering `DefaultIsolatedNetworkOfferingForVpcNetworks` - register CKS ISO - create CKS on the vpc tier expected: succeed actual: failed with error `Kubernetes service has not been configured properly to provision Kubernetes clusters` --- .../cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index cf4bbce098af..d19470f8bab2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -477,7 +477,7 @@ private boolean isKubernetesServiceNetworkOfferingConfigured(DataCenter zone, Lo logger.warn("Network offering: {} does not have necessary services to provision Kubernetes cluster", networkOffering); return false; } - if (!networkOffering.isEgressDefaultPolicy()) { + if (!networkOffering.isForVpc() && !networkOffering.isEgressDefaultPolicy()) { logger.warn("Network offering: {} has egress default policy turned off should be on to provision Kubernetes cluster", networkOffering); return false; } From 70d4c9d1baa5f6e7696b6311e7eeaaa2c4f0de3b Mon Sep 17 00:00:00 2001 From: Fabricio Duarte Date: Wed, 28 Jan 2026 02:48:31 -0300 Subject: [PATCH 053/117] Consider secondary storage selectors during cold volume migration (#10957) The secondary storage selectors allow operators to specify, for instance, that volumes should go to a specific secondary storage A. Thus, when uploading a volume, it will always be downloaded to secondary storage A. The cold volume migration moves volumes to a secondary storage before moving them to the destination primary storage. This process does not consider the secondary storage selectors. However, some companies want to dedicate specific secondary storages for cold migration. To address this, this PR makes the cold volume migration process consider the secondary storage selectors. --- .../motion/AncientDataMotionStrategy.java | 13 ++++++++++++- .../storage/heuristics/HeuristicRuleHelper.java | 16 ++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index b59ee2c61660..4cb09fb81f71 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -45,10 +45,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.logging.log4j.Logger; @@ -104,6 +106,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { @Inject SnapshotDao snapshotDao; + @Inject + HeuristicRuleHelper heuristicRuleHelper; + @Override public StrategyPriority canHandle(DataObject srcData, DataObject destData) { return StrategyPriority.DEFAULT; @@ -374,7 +379,13 @@ protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData) } // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 - ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId()); + Long zoneId = destScope.getScopeId(); + ImageStoreEntity imageStore = (ImageStoreEntity) heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, destData); + if (imageStore == null) { + logger.debug("Secondary storage selector did not direct volume migration to a specific secondary storage; using secondary storage with the most free capacity."); + imageStore = (ImageStoreEntity) dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + } + if (imageStore == null || !imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { String errMsg = "can't find a nfs (or cifs) image store to satisfy the need for a staging store"; Answer answer = new Answer(null, false, errMsg); diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java index 21a34de0d23b..2e0780e7fe8d 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java @@ -117,8 +117,8 @@ protected void buildPresetVariables(JsInterpreter jsInterpreter, HeuristicType h accountId = ((SnapshotInfo) obj).getAccountId(); break; case VOLUME: - presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj)); - accountId = ((VolumeVO) obj).getAccountId(); + presetVariables.setVolume(setVolumePresetVariable((com.cloud.storage.Volume) obj)); + accountId = ((com.cloud.storage.Volume) obj).getAccountId(); break; } presetVariables.setAccount(setAccountPresetVariable(accountId)); @@ -191,14 +191,14 @@ protected Template setTemplatePresetVariable(VMTemplateVO templateVO) { return template; } - protected Volume setVolumePresetVariable(VolumeVO volumeVO) { - Volume volume = new Volume(); + protected Volume setVolumePresetVariable(com.cloud.storage.Volume volumeVO) { + Volume volumePresetVariable = new Volume(); - volume.setName(volumeVO.getName()); - volume.setFormat(volumeVO.getFormat()); - volume.setSize(volumeVO.getSize()); + volumePresetVariable.setName(volumeVO.getName()); + volumePresetVariable.setFormat(volumeVO.getFormat()); + volumePresetVariable.setSize(volumeVO.getSize()); - return volume; + return volumePresetVariable; } protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) { From 4761935145e100622df083dedffec612482b16d2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 28 Jan 2026 06:59:31 +0100 Subject: [PATCH 054/117] server: add options for kvm.guest.os.machine.type (#12414) --- server/src/main/java/com/cloud/api/query/QueryManagerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 3b2329483954..d42dbaec6de3 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -5398,6 +5398,7 @@ private void fillVMOrTemplateDetailOptions(final Map> optio options.put(VmDetailConstants.VIRTUAL_TPM_VERSION, Arrays.asList("1.2", "2.0")); options.put(VmDetailConstants.GUEST_CPU_MODE, Arrays.asList("custom", "host-model", "host-passthrough")); options.put(VmDetailConstants.GUEST_CPU_MODEL, Collections.emptyList()); + options.put(VmDetailConstants.KVM_GUEST_OS_MACHINE_TYPE, Collections.emptyList()); options.put(VmDetailConstants.KVM_SKIP_FORCE_DISK_CONTROLLER, Arrays.asList("true", "false")); } From 0dcbe57a47875d45ee7effa92451a331c5bccc10 Mon Sep 17 00:00:00 2001 From: Edward-x <30854794+YLChen-007@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:56:44 +0800 Subject: [PATCH 055/117] Fix that Sensitive information logged in SshHelper.sshExecute method (#12026) * Sensitive information logged in SshHelper.sshExecute method * Fix that Sensitive information logged in SshHelper.sshExecute method2 * Fix sensitive information handling in SshHelper and its tests --------- Co-authored-by: chenyoulong20g@ict.ac.cn --- .../java/com/cloud/utils/ssh/SshHelper.java | 73 ++++++++++++++++++- .../com/cloud/utils/ssh/SshHelperTest.java | 60 +++++++++++++++ 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java index 87221ab5ac8e..caf2b28c52ff 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java @@ -23,6 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -40,6 +42,23 @@ public class SshHelper { private static final int DEFAULT_CONNECT_TIMEOUT = 180000; private static final int DEFAULT_KEX_TIMEOUT = 60000; private static final int DEFAULT_WAIT_RESULT_TIMEOUT = 120000; + private static final String MASKED_VALUE = "*****"; + + private static final Pattern[] SENSITIVE_COMMAND_PATTERNS = new Pattern[] { + Pattern.compile("(?i)(\\s+-p\\s+['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-p\\s+)([^\\s]+)"), + Pattern.compile("(?i)(\\s+-p=['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-p=)([^\\s]+)"), + Pattern.compile("(?i)(--password=['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(--password=)([^\\s]+)"), + Pattern.compile("(?i)(--password\\s+['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(--password\\s+)([^\\s]+)"), + Pattern.compile("(?i)(\\s+-u\\s+['\"][^,'\":]+[,:])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-u\\s+[^\\s,:]+[,:])([^\\s]+)"), + Pattern.compile("(?i)(\\s+-s\\s+['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-s\\s+)([^\\s]+)"), + + }; protected static Logger LOGGER = LogManager.getLogger(SshHelper.class); @@ -145,7 +164,7 @@ public static void scpTo(String host, int port, String user, File pemKeyFile, St } public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, String[] localFiles, String fileMode, - int connectTimeoutInMs, int kexTimeoutInMs) throws Exception { + int connectTimeoutInMs, int kexTimeoutInMs) throws Exception { com.trilead.ssh2.Connection conn = null; com.trilead.ssh2.SCPClient scpClient = null; @@ -291,13 +310,16 @@ public static Pair sshExecute(String host, int port, String use } if (sess.getExitStatus() == null) { - //Exit status is NOT available. Returning failure result. - LOGGER.error(String.format("SSH execution of command %s has no exit status set. Result output: %s", command, result)); + // Exit status is NOT available. Returning failure result. + LOGGER.error(String.format("SSH execution of command %s has no exit status set. Result output: %s", + sanitizeForLogging(command), sanitizeForLogging(result))); return new Pair(false, result); } if (sess.getExitStatus() != null && sess.getExitStatus().intValue() != 0) { - LOGGER.error(String.format("SSH execution of command %s has an error status code in return. Result output: %s", command, result)); + LOGGER.error(String.format( + "SSH execution of command %s has an error status code in return. Result output: %s", + sanitizeForLogging(command), sanitizeForLogging(result))); return new Pair(false, result); } return new Pair(true, result); @@ -366,4 +388,47 @@ protected static void throwSshExceptionIfStdoutOrStdeerIsNull(InputStream stdout throw new SshException(msg); } } + + private static String sanitizeForLogging(String value) { + if (value == null) { + return null; + } + String masked = maskSensitiveValue(value); + String cleaned = com.cloud.utils.StringUtils.cleanString(masked); + if (StringUtils.isBlank(cleaned)) { + return masked; + } + return cleaned; + } + + private static String maskSensitiveValue(String value) { + String masked = value; + for (Pattern pattern : SENSITIVE_COMMAND_PATTERNS) { + masked = replaceWithMask(masked, pattern); + } + return masked; + } + + private static String replaceWithMask(String value, Pattern pattern) { + Matcher matcher = pattern.matcher(value); + if (!matcher.find()) { + return value; + } + + StringBuffer buffer = new StringBuffer(); + do { + StringBuilder replacement = new StringBuilder(); + replacement.append(matcher.group(1)); + if (matcher.groupCount() >= 3) { + replacement.append(MASKED_VALUE); + replacement.append(matcher.group(matcher.groupCount())); + } else { + replacement.append(MASKED_VALUE); + } + matcher.appendReplacement(buffer, Matcher.quoteReplacement(replacement.toString())); + } while (matcher.find()); + + matcher.appendTail(buffer); + return buffer.toString(); + } } diff --git a/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java b/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java index 61d746bc12db..8a14f60527b6 100644 --- a/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java +++ b/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import org.junit.Assert; import org.junit.Test; @@ -140,4 +141,63 @@ public void openConnectionSessionTest() throws IOException, InterruptedException Mockito.verify(conn).openSession(); } + + @Test + public void sanitizeForLoggingMasksShortPasswordFlag() throws Exception { + String command = "/opt/cloud/bin/script -v 10.0.0.1 -p superSecret"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain flag", sanitized.contains("-p *****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("superSecret")); + } + + @Test + public void sanitizeForLoggingMasksQuotedPasswordFlag() throws Exception { + String command = "/opt/cloud/bin/script -v 10.0.0.1 -p \"super Secret\""; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain quoted flag", sanitized.contains("-p *****")); + Assert.assertFalse("Sanitized command should not contain original password", + sanitized.contains("super Secret")); + } + + @Test + public void sanitizeForLoggingMasksLongPasswordAssignments() throws Exception { + String command = "tool --password=superSecret"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain assignment", sanitized.contains("--password=*****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("superSecret")); + } + + @Test + public void sanitizeForLoggingMasksUsernamePasswordPairs() throws Exception { + String command = "/opt/cloud/bin/vpn_l2tp.sh -u alice,topSecret"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain username and mask password", + sanitized.contains("-u alice,*****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("topSecret")); + } + + @Test + public void sanitizeForLoggingMasksUsernamePasswordPairsWithColon() throws Exception { + String command = "curl -u alice:topSecret https://example.com"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain username and mask password", + sanitized.contains("-u alice:*****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("topSecret")); + } + + @Test + public void sanitizeForLoggingHandlesNullValues() throws Exception { + Assert.assertNull(invokeSanitizeForLogging(null)); + } + + private String invokeSanitizeForLogging(String value) throws Exception { + Method method = SshHelper.class.getDeclaredMethod("sanitizeForLogging", String.class); + method.setAccessible(true); + return (String) method.invoke(null, value); + } } From 1b2ae13df74ef3f22f8e5b40b4cf45a10863205f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Jan 2026 12:40:34 +0530 Subject: [PATCH 056/117] ui: add cache for oslogo request using osId (#11422) When OsLogo component is used in the items of a list having same OS type it was causing listOsTypes API call multiple time. This change allows caching request and response value for 30 seconds. Caching behaviour is controlled using `useCache` flag. Signed-off-by: Abhishek Kumar --- ui/src/components/widgets/OsLogo.vue | 78 ++++++++++--------- .../compute/wizard/OsBasedImageRadioGroup.vue | 3 +- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/ui/src/components/widgets/OsLogo.vue b/ui/src/components/widgets/OsLogo.vue index 643953012c18..f19aac56a1a6 100644 --- a/ui/src/components/widgets/OsLogo.vue +++ b/ui/src/components/widgets/OsLogo.vue @@ -31,6 +31,9 @@ - - diff --git a/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue b/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue index 2518ed0c0420..45ea347553c3 100644 --- a/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue +++ b/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue @@ -42,7 +42,8 @@ class="radio-group__os-logo" size="2x" :osId="item.ostypeid" - :os-name="item.osName" /> + :os-name="item.osName" + :use-cache="true" />   {{ item.displaytext }} From 7001d43dbfa7a0e5ca68d527754570596efcb180 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:09:29 +0530 Subject: [PATCH 057/117] Bump org.codehaus.mojo:properties-maven-plugin from 1.0-alpha-2 to 1.2.1 (#12508) --- developer/pom.xml | 2 +- tools/devcloud-kvm/pom.xml | 2 +- tools/devcloud4/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/developer/pom.xml b/developer/pom.xml index e2fd782fd25f..0a0979ee0379 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -66,7 +66,7 @@ org.codehaus.mojo properties-maven-plugin - 1.0-alpha-2 + 1.2.1 initialize diff --git a/tools/devcloud-kvm/pom.xml b/tools/devcloud-kvm/pom.xml index a8cd23db9799..35cf828a27b6 100644 --- a/tools/devcloud-kvm/pom.xml +++ b/tools/devcloud-kvm/pom.xml @@ -56,7 +56,7 @@ org.codehaus.mojo properties-maven-plugin - 1.0-alpha-2 + 1.2.1 initialize diff --git a/tools/devcloud4/pom.xml b/tools/devcloud4/pom.xml index 1af63b439ad7..385b49ad88c7 100644 --- a/tools/devcloud4/pom.xml +++ b/tools/devcloud4/pom.xml @@ -56,7 +56,7 @@ org.codehaus.mojo properties-maven-plugin - 1.0-alpha-2 + 1.2.1 initialize From 434e472ef814897a985f87b92b1d46e9cfc23eeb Mon Sep 17 00:00:00 2001 From: Tonitzpp <134986282+Tonitzpp@users.noreply.github.com> Date: Wed, 28 Jan 2026 06:10:43 -0300 Subject: [PATCH 058/117] Change to display if public IPs are reserved in the tab (#12461) Co-authored-by: toni.zamparetti --- ui/src/views/infra/network/IpRangesTabPublic.vue | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ui/src/views/infra/network/IpRangesTabPublic.vue b/ui/src/views/infra/network/IpRangesTabPublic.vue index 81f5656799e5..dc6ef671e7ce 100644 --- a/ui/src/views/infra/network/IpRangesTabPublic.vue +++ b/ui/src/views/infra/network/IpRangesTabPublic.vue @@ -47,6 +47,9 @@ + @@ -128,10 +131,6 @@
{{ $t('label.domain') }}
{{ selectedItem.domain }}
-
-
{{ $t('label.system.vms') }}
-
{{ selectedItem.forsystemvms }}
-
@@ -449,6 +448,10 @@ export default { key: 'endip', title: this.$t('label.endip') }, + { + key: 'systemvms', + title: this.$t('label.reserved.system.ip') + }, { key: 'actions', title: this.$t('label.actions') From 0e7f74839ae264f08382bfcb6692af4f5f073653 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 28 Jan 2026 10:48:27 +0100 Subject: [PATCH 059/117] Add configuration for archiving stale issues (#12293) --- .github/workflows/stale.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e90c75979b6d..c957392c5046 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -41,3 +41,10 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' + days-before-close: -1 + - uses: actions/stale@v10 + with: + stale-issue-label: 'archive' + days-before-stale: 240 + exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' + days-before-close: -1 From 6932cacabc187cf3d76e53c7979ed10067aff2f2 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Wed, 28 Jan 2026 16:00:30 +0530 Subject: [PATCH 060/117] Allow copy of templates from secondary storages of other zone when adding a new secondary storage (#12296) * Allow copy of templates from secondary storages of other zone when adding a new secondary storage * Add API param and UI changes on add secondary storage page * Make copy template across zones non blocking * Code fixes * unused imports * Add copy template flag in zone wizard and remove NFS checks * Fix UI * Label fixes * code optimizations * code refactoring * missing changes * Combine template copy and download into a single asynchronous operation * unused import and fixed conflicts * unused code * update config message * Fix configuration setting value on add secondary storage page * Removed unused code * Update unit tests --- .../admin/host/AddSecondaryStorageCmd.java | 24 ++- .../service/StorageOrchestrationService.java | 3 +- .../api/storage/TemplateService.java | 4 +- .../com/cloud/storage/StorageManager.java | 5 +- .../orchestration/StorageOrchestrator.java | 45 +++-- .../storage/image/TemplateServiceImpl.java | 157 +++++++++++++--- .../image/TemplateServiceImplTest.java | 171 +++++++++++++++++- .../cloud/storage/ImageStoreDetailsUtil.java | 11 ++ .../com/cloud/storage/StorageManagerImpl.java | 2 +- .../cloud/template/TemplateManagerImpl.java | 14 +- ui/public/locales/en.json | 4 +- ui/src/views/infra/AddSecondaryStorage.vue | 82 ++++++++- .../infra/zone/ZoneWizardAddResources.vue | 25 ++- .../views/infra/zone/ZoneWizardLaunchZone.vue | 5 + 14 files changed, 487 insertions(+), 65 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java index 9a7eff7e2e59..585fd1b87a88 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java @@ -29,6 +29,11 @@ import com.cloud.exception.DiscoveryException; import com.cloud.storage.ImageStore; import com.cloud.user.Account; +import org.apache.commons.collections.MapUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; @APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage") protected Long zoneId; + @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true") + protected Map details; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -56,6 +64,20 @@ public Long getZoneId() { return zoneId; } + public Map getDetails() { + Map detailsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(details)) { + Collection props = details.values(); + for (Object prop : props) { + HashMap detail = (HashMap) prop; + for (Map.Entry entry: detail.entrySet()) { + detailsMap.put(entry.getKey(),entry.getValue()); + } + } + } + return detailsMap; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -68,7 +90,7 @@ public long getEntityOwnerId() { @Override public void execute(){ try{ - ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null); + ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails()); ImageStoreResponse storeResponse = null; if (result != null ) { storeResponse = _responseGenerator.createImageStoreResponse(result); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java index 8be2015bfef6..4af0c806060b 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java @@ -22,7 +22,6 @@ import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; @@ -31,5 +30,5 @@ public interface StorageOrchestrationService { MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List templateIdList, List snapshotIdList); - Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore); + Future orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index a8861d5acc68..269eb4f1c213 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -80,4 +80,6 @@ public TemplateInfo getTemplate() { List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId); AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore); -} + + void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); + } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index de0cb34d63ee..4ce1f4a96385 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -220,8 +220,9 @@ public interface StorageManager extends StorageService { "storage.pool.host.connect.workers", "1", "Number of worker threads to be used to connect hosts to a primary storage", true); - ConfigKey COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages", - "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.", + ConfigKey COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages", + "Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " + + "while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.", true, ConfigKey.Scope.Zone, null); /** diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 37a1f8dc196e..933b4e0c5ce6 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -36,6 +36,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.TemplateManager; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -45,6 +48,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; @@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra VolumeDataStoreDao volumeDataStoreDao; @Inject DataMigrationUtility migrationHelper; + @Inject + TemplateManager templateManager; + @Inject + VMTemplateDao templateDao; + @Inject + TemplateDataFactory templateDataFactory; + @Inject + DataCenterDao dcDao; + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, "image.store.imbalance.threshold", @@ -304,8 +317,9 @@ public MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreI } @Override - public Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) { - return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore)); + public Future orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) { + Long dstZoneId = destStore.getScope().getScopeId(); + return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore)); } protected Pair migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List files, MigrationPolicy migrationPolicy, int skipped) { @@ -624,13 +638,13 @@ public DataObjectResult call() { } } - private class CopyTemplateTask implements Callable { - private TemplateInfo sourceTmpl; - private DataStore destStore; - private String logid; + private class CopyTemplateFromSecondaryStorageTask implements Callable { + private final long srcTemplateId; + private final DataStore destStore; + private final String logid; - public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) { - this.sourceTmpl = sourceTmpl; + CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) { + this.srcTemplateId = srcTemplateId; this.destStore = destStore; this.logid = ThreadContext.get(LOGCONTEXTID); } @@ -639,17 +653,16 @@ public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) { public TemplateApiResult call() { ThreadContext.put(LOGCONTEXTID, logid); TemplateApiResult result; - AsyncCallFuture future = templateService.copyTemplateToImageStore(sourceTmpl, destStore); + long destZoneId = destStore.getScope().getScopeId(); + TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image); try { - result = future.get(); - } catch (ExecutionException | InterruptedException e) { - logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", - sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore); result = new TemplateApiResult(sourceTmpl); - result.setResult(e.getMessage()); + } finally { + tryCleaningUpExecutor(destZoneId); + ThreadContext.clearAll(); } - tryCleaningUpExecutor(destStore.getScope().getScopeId()); - ThreadContext.clearAll(); + return result; } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index bee62955051a..5fc9bbac3522 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -31,6 +31,8 @@ import javax.inject.Inject; +import com.cloud.exception.StorageUnavailableException; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -67,9 +69,11 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.ThreadContext; import org.springframework.stereotype.Component; import com.cloud.agent.api.Answer; @@ -567,10 +571,7 @@ public void handleTemplateSync(DataStore store) { } if (availHypers.contains(tmplt.getHypervisorType())) { - boolean copied = isCopyFromOtherStoragesEnabled(zoneId) && tryCopyingTemplateToImageStore(tmplt, store); - if (!copied) { - tryDownloadingTemplateToImageStore(tmplt, store); - } + storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(tmplt.getId(), store); } else { logger.info("Skip downloading template {} since current data center does not have hypervisor {}", tmplt, tmplt.getHypervisorType()); } @@ -617,6 +618,16 @@ public void handleTemplateSync(DataStore store) { } + @Override + public void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore) { + VMTemplateVO template = _templateDao.findById(templateId); + long zoneId = destStore.getScope().getScopeId(); + boolean copied = imageStoreDetailsUtil.isCopyTemplatesFromOtherStoragesEnabled(destStore.getId(), zoneId) && tryCopyingTemplateToImageStore(template, destStore); + if (!copied) { + tryDownloadingTemplateToImageStore(template, destStore); + } + } + protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { if (tmplt.getUrl() == null) { logger.info("Not downloading template [{}] to image store [{}], as it has no URL.", tmplt.getUniqueName(), @@ -634,28 +645,134 @@ protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore } protected boolean tryCopyingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { - Long zoneId = destStore.getScope().getScopeId(); - List storesInZone = _storeMgr.getImageStoresByZoneIds(zoneId); - for (DataStore sourceStore : storesInZone) { - Map existingTemplatesInSourceStore = listTemplate(sourceStore); - if (existingTemplatesInSourceStore == null || !existingTemplatesInSourceStore.containsKey(tmplt.getUniqueName())) { - logger.debug("Template [{}] does not exist on image store [{}]; searching on another one.", - tmplt.getUniqueName(), sourceStore.getName()); + if (searchAndCopyWithinZone(tmplt, destStore)) { + return true; + } + + Long destZoneId = destStore.getScope().getScopeId(); + logger.debug("Template [{}] not found in any image store of zone [{}]. Checking other zones.", + tmplt.getUniqueName(), destZoneId); + + return searchAndCopyAcrossZones(tmplt, destStore, destZoneId); + } + + private boolean searchAndCopyAcrossZones(VMTemplateVO tmplt, DataStore destStore, Long destZoneId) { + List allZoneIds = _dcDao.listAllIds(); + for (Long otherZoneId : allZoneIds) { + if (otherZoneId.equals(destZoneId)) { continue; } - TemplateObject sourceTmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), sourceStore); - if (sourceTmpl.getInstallPath() == null) { - logger.warn("Can not copy template [{}] from image store [{}], as it returned a null install path.", tmplt.getUniqueName(), - sourceStore.getName()); + + List storesInOtherZone = _storeMgr.getImageStoresByZoneIds(otherZoneId); + logger.debug("Checking zone [{}] for template [{}]...", otherZoneId, tmplt.getUniqueName()); + + if (CollectionUtils.isEmpty(storesInOtherZone)) { + logger.debug("Zone [{}] has no image stores. Skipping.", otherZoneId); continue; } - storageOrchestrator.orchestrateTemplateCopyToImageStore(sourceTmpl, destStore); - return true; + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInOtherZone); + if (sourceTmpl == null) { + logger.debug("Template [{}] not found with a valid install path in any image store of zone [{}].", + tmplt.getUniqueName(), otherZoneId); + continue; + } + + logger.info("Template [{}] found in zone [{}]. Initiating cross-zone copy to zone [{}].", + tmplt.getUniqueName(), otherZoneId, destZoneId); + + return copyTemplateAcrossZones(destStore, sourceTmpl); } - logger.debug("Can't copy template [{}] from another image store.", tmplt.getUniqueName()); + + logger.debug("Template [{}] was not found in any zone. Cannot perform zone-to-zone copy.", tmplt.getUniqueName()); return false; } + protected TemplateObject findUsableTemplate(VMTemplateVO tmplt, List imageStores) { + for (DataStore store : imageStores) { + + Map templates = listTemplate(store); + if (templates == null || !templates.containsKey(tmplt.getUniqueName())) { + continue; + } + + TemplateObject tmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), store); + if (tmpl.getInstallPath() == null) { + logger.debug("Template [{}] found in image store [{}] but install path is null. Skipping.", + tmplt.getUniqueName(), store.getName()); + continue; + } + return tmpl; + } + return null; + } + + private boolean searchAndCopyWithinZone(VMTemplateVO tmplt, DataStore destStore) { + Long destZoneId = destStore.getScope().getScopeId(); + List storesInSameZone = _storeMgr.getImageStoresByZoneIds(destZoneId); + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInSameZone); + if (sourceTmpl == null) { + return false; + } + + TemplateApiResult result; + AsyncCallFuture future = copyTemplateToImageStore(sourceTmpl, destStore); + try { + result = future.get(); + } catch (ExecutionException | InterruptedException e) { + logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", + sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } + return result.isSuccess(); + } + + private boolean copyTemplateAcrossZones(DataStore destStore, TemplateObject sourceTmpl) { + Long dstZoneId = destStore.getScope().getScopeId(); + DataCenterVO dstZone = _dcDao.findById(dstZoneId); + + if (dstZone == null) { + logger.warn("Destination zone [{}] not found for template [{}].", dstZoneId, sourceTmpl.getUniqueName()); + return false; + } + + TemplateApiResult result; + try { + VMTemplateVO template = _templateDao.findById(sourceTmpl.getId()); + try { + DataStore sourceStore = sourceTmpl.getDataStore(); + long userId = CallContext.current().getCallingUserId(); + boolean success = _tmpltMgr.copy(userId, template, sourceStore, dstZone); + + result = new TemplateApiResult(sourceTmpl); + if (!success) { + result.setResult("Cross-zone template copy failed"); + } + } catch (StorageUnavailableException | ResourceAllocationException e) { + logger.error("Exception while copying template [{}] from zone [{}] to zone [{}]", + template, + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZone.getId(), + e); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } finally { + ThreadContext.clearAll(); + } + } catch (Exception e) { + logger.error("Failed to copy template [{}] from zone [{}] to zone [{}].", + sourceTmpl.getUniqueName(), + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZoneId, + e); + return false; + } + + return result.isSuccess(); + } + @Override public AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore) { TemplateObject sourceTmpl = (TemplateObject) source; @@ -699,10 +816,6 @@ protected Void copyTemplateToImageStoreCallback(AsyncCallbackDispatcher