From 482809bbfd944a1eed0eabfdaee57ae3cb50b048 Mon Sep 17 00:00:00 2001
From: sp16743
Date: Mon, 26 Oct 2026 19:28:17 +0530
Subject: [PATCH 1/4] Fix ONTAP LUN clone visibility using GET retry
---
.../storage/service/UnifiedSANStrategy.java | 56 ++++++++++++++++++-
.../storage/utils/OntapStorageConstants.java | 3 +
.../service/UnifiedSANStrategyTest.java | 3 +
3 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
index 21e58a00a8bc..a49d94ab793b 100644
--- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
+++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
@@ -261,6 +261,9 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) {
*
* The resulting LUN shares blocks with its source and is created in constant time.
* It is a sis-clone, so it stays readable after the source LUN is deleted.
+ *
+ * The create response includes the LUN uuid. That uuid is confirmed with repeated
+ * {@code GET /api/storage/luns/{uuid}} calls while ONTAP returns 404.
*/
@Override
public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume) {
@@ -279,12 +282,15 @@ public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume)
lunRequest.getClone().getSource().getUuid());
try {
String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword());
- OntapResponse clonedLun = sanFeignClient.createLun(authHeader, true, lunRequest);
- if (clonedLun == null || CollectionUtils.isEmpty(clonedLun.getRecords())) {
+ OntapResponse created = sanFeignClient.createLun(authHeader, true, lunRequest);
+ if (created == null || created.getRecords() == null || created.getRecords().isEmpty()
+ || created.getRecords().get(0) == null
+ || StringUtils.isBlank(created.getRecords().get(0).getUuid())) {
logger.error("cloneCloudStackVolume: LUN clone returned no records for Lun {}", lunRequest.getName());
throw new CloudRuntimeException("Failed to clone Lun: " + lunRequest.getName());
}
- Lun lun = clonedLun.getRecords().get(0);
+ String lunUuid = created.getRecords().get(0).getUuid();
+ Lun lun = getLunByUuidUntilVisible(authHeader, lunUuid, lunRequest.getName());
validateCreatedLun(lun, lunRequest.getName(), "cloneCloudStackVolume");
logger.debug("cloneCloudStackVolume: LUN cloned successfully. Lun: {}", lun);
@@ -487,6 +493,50 @@ protected void waitForLunCopy(String authHeader, String lunUuid, long deadline)
}
}
+ /**
+ * Confirms a clone uuid by reading that LUN directly.
+ *
+ * {@code GET /api/storage/luns/{uuid}} returns 404 until the clone is visible. Those
+ * 404s are retried. Any other GET failure is returned to the caller.
+ */
+ private Lun getLunByUuidUntilVisible(String authHeader, String lunUuid, String lunName) {
+ for (int attempt = 1; attempt <= OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES; attempt++) {
+ try {
+ Lun lun = sanFeignClient.getLunByUUID(authHeader, lunUuid);
+ if (lun != null) {
+ if (StringUtils.isBlank(lun.getUuid())) {
+ lun.setUuid(lunUuid);
+ }
+ if (lun.getName() == null) {
+ lun.setName(lunName);
+ }
+ logger.debug("cloneCloudStackVolume: LUN uuid [{}] is visible on attempt {}/{}",
+ lunUuid, attempt, OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES);
+ return lun;
+ }
+ logger.info("cloneCloudStackVolume: LUN uuid [{}] is not visible yet (attempt {}/{})",
+ lunUuid, attempt, OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES);
+ } catch (FeignException e) {
+ if (e.status() != 404) {
+ logger.error("cloneCloudStackVolume: GET LUN uuid [{}] failed, Status: {}, Exception: {}",
+ lunUuid, e.status(), e.getMessage());
+ throw new CloudRuntimeException("Failed to fetch cloned Lun: " + e.getMessage());
+ }
+ logger.info("cloneCloudStackVolume: GET LUN uuid [{}] returned 404 (attempt {}/{})",
+ lunUuid, attempt, OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES);
+ }
+ if (attempt < OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES) {
+ try {
+ Thread.sleep(OntapStorageConstants.ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS);
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new CloudRuntimeException("Interrupted while waiting for cloned Lun " + lunName, interrupted);
+ }
+ }
+ }
+ throw new CloudRuntimeException("Cloned Lun uuid was not visible for: " + lunUuid);
+ }
+
/**
* Ensures ONTAP returned a usable LUN identity from create/clone. Callers in the datastore
* driver rely on non-null name and uuid, so reject incomplete records at the Feign boundary.
diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
index 26325c44cbab..5f1ee4915f82 100644
--- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
+++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
@@ -176,6 +176,9 @@ public class OntapStorageConstants {
public static final int ONTAP_VOLUME_JOB_POLL_INTERVAL_MS = 1000;
public static final int ONTAP_FLEXVOL_JOB_POLL_INTERVAL_MS = 2000;
public static final int ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES = 30;
+ /** GET-by-UUID retries after LUN clone response. */
+ public static final int ONTAP_LUN_CLONE_GET_MAX_RETRIES = 5;
+ public static final int ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS = 1000;
/** vm_snapshot_details key for ONTAP FlexVolume-level VM snapshots. */
public static final String ONTAP_FLEXVOL_SNAPSHOT = "ontapFlexVolSnapshot";
diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java
index a767f483c8f8..13ec49e37b57 100644
--- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java
+++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java
@@ -1016,6 +1016,7 @@ void testCloneCloudStackVolume_Success() {
when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class)))
.thenReturn(response);
+ when(sanFeignClient.getLunByUUID(eq(authHeader), eq("cloned-lun-uuid"))).thenReturn(clonedLun);
CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request);
@@ -1023,6 +1024,7 @@ void testCloneCloudStackVolume_Success() {
assertEquals("cloned-lun-uuid", result.getLun().getUuid());
assertSame(volumeInfo, result.getVolumeInfo());
verify(sanFeignClient).createLun(eq(authHeader), eq(true), any(Lun.class));
+ verify(sanFeignClient).getLunByUUID(eq(authHeader), eq("cloned-lun-uuid"));
}
}
@@ -1403,6 +1405,7 @@ void testCloneCloudStackVolume_SnapshotSource_SendsSnapshotQualifiedName() {
utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password"))
.thenReturn(authHeader);
when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenReturn(response);
+ when(sanFeignClient.getLunByUUID(eq(authHeader), eq("new-lun-uuid"))).thenReturn(createdLun);
CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request);
From 3901003a61c1482ca9d1f128b555aec2b1690e2d Mon Sep 17 00:00:00 2001
From: sp16743
Date: Wed, 4 Nov 2026 21:23:45 +0530
Subject: [PATCH 2/4] Change the cloned LUN read failure message
---
.../apache/cloudstack/storage/service/UnifiedSANStrategy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
index a49d94ab793b..6ca24f17ce2f 100644
--- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
+++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
@@ -534,7 +534,7 @@ private Lun getLunByUuidUntilVisible(String authHeader, String lunUuid, String l
}
}
}
- throw new CloudRuntimeException("Cloned Lun uuid was not visible for: " + lunUuid);
+ throw new CloudRuntimeException("Failed to read cloned Lun " + lunName);
}
/**
From 247ba1724dccfa814c5e1126ecdc2a260f3e8a84 Mon Sep 17 00:00:00 2001
From: sp16743
Date: Thu, 12 Nov 2026 18:28:42 +0530
Subject: [PATCH 3/4] Addressing Comments
---
.../storage/service/UnifiedSANStrategy.java | 54 +++++++++++++++----
.../storage/utils/OntapStorageConstants.java | 2 +-
.../service/UnifiedSANStrategyTest.java | 7 +--
3 files changed, 48 insertions(+), 15 deletions(-)
diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
index 6ca24f17ce2f..265f836bc857 100644
--- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
+++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/service/UnifiedSANStrategy.java
@@ -263,7 +263,7 @@ public void deleteCloudStackVolume(CloudStackVolume cloudstackVolume) {
* It is a sis-clone, so it stays readable after the source LUN is deleted.
*
* The create response includes the LUN uuid. That uuid is confirmed with repeated
- * {@code GET /api/storage/luns/{uuid}} calls while ONTAP returns 404.
+ * {@code GET /api/storage/luns?uuid=} calls until the response contains a record.
*/
@Override
public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume) {
@@ -280,16 +280,17 @@ public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume)
lunRequest.getName(),
lunRequest.getClone().getSource().getName(),
lunRequest.getClone().getSource().getUuid());
+ String lunUuid = null;
try {
String authHeader = OntapStorageUtils.generateAuthHeader(storage.getUsername(), storage.getPassword());
- OntapResponse created = sanFeignClient.createLun(authHeader, true, lunRequest);
- if (created == null || created.getRecords() == null || created.getRecords().isEmpty()
- || created.getRecords().get(0) == null
- || StringUtils.isBlank(created.getRecords().get(0).getUuid())) {
+ OntapResponse clonedLun = sanFeignClient.createLun(authHeader, true, lunRequest);
+ if (clonedLun == null || CollectionUtils.isEmpty(clonedLun.getRecords())
+ || clonedLun.getRecords().get(0) == null
+ || StringUtils.isBlank(clonedLun.getRecords().get(0).getUuid())) {
logger.error("cloneCloudStackVolume: LUN clone returned no records for Lun {}", lunRequest.getName());
throw new CloudRuntimeException("Failed to clone Lun: " + lunRequest.getName());
}
- String lunUuid = created.getRecords().get(0).getUuid();
+ lunUuid = clonedLun.getRecords().get(0).getUuid();
Lun lun = getLunByUuidUntilVisible(authHeader, lunUuid, lunRequest.getName());
validateCreatedLun(lun, lunRequest.getName(), "cloneCloudStackVolume");
logger.debug("cloneCloudStackVolume: LUN cloned successfully. Lun: {}", lun);
@@ -299,12 +300,15 @@ public CloudStackVolume cloneCloudStackVolume(CloudStackVolume cloudstackVolume)
clonedCloudStackVolume.setVolumeInfo(cloudstackVolume.getVolumeInfo());
return clonedCloudStackVolume;
} catch (FeignException e) {
+ bestEffortDeleteClonedLun(lunUuid, lunRequest.getName());
logger.error("FeignException occurred while cloning LUN: {}, Status: {}, Exception: {}",
lunRequest.getName(), e.status(), e.getMessage());
throw new CloudRuntimeException("Failed to clone Lun: " + e.getMessage());
} catch (CloudRuntimeException e) {
+ bestEffortDeleteClonedLun(lunUuid, lunRequest.getName());
throw e;
} catch (Exception e) {
+ bestEffortDeleteClonedLun(lunUuid, lunRequest.getName());
logger.error("Exception occurred while cloning LUN: {}, Exception: {}", lunRequest.getName(), e.getMessage());
throw new CloudRuntimeException("Failed to clone Lun: " + e.getMessage());
}
@@ -494,16 +498,44 @@ protected void waitForLunCopy(String authHeader, String lunUuid, long deadline)
}
/**
- * Confirms a clone uuid by reading that LUN directly.
+ * Removes a clone that was created but could not be confirmed.
*
- * {@code GET /api/storage/luns/{uuid}} returns 404 until the clone is visible. Those
- * 404s are retried. Any other GET failure is returned to the caller.
+ * The LUN is already on ONTAP once create returns a uuid. Leaving it in place makes the
+ * next clone of the same name fail as a duplicate.
+ */
+ private void bestEffortDeleteClonedLun(String lunUuid, String lunName) {
+ if (StringUtils.isBlank(lunUuid)) {
+ return;
+ }
+ try {
+ Lun lun = new Lun();
+ lun.setUuid(lunUuid);
+ lun.setName(lunName);
+ CloudStackVolume deleteRequest = new CloudStackVolume();
+ deleteRequest.setLun(lun);
+ deleteCloudStackVolume(deleteRequest);
+ logger.info("bestEffortDeleteClonedLun: Removed leftover cloned LUN [{}] uuid [{}]",
+ lunName, lunUuid);
+ } catch (Exception cleanupEx) {
+ logger.warn("bestEffortDeleteClonedLun: Failed to remove leftover cloned LUN [{}] uuid [{}]: {}",
+ lunName, lunUuid, cleanupEx.getMessage());
+ }
+ }
+
+ /**
+ * Confirms a clone uuid from the collection GET record.
+ *
+ * {@code GET /api/storage/luns?uuid=} returns no records until the clone is visible.
+ * Empty responses and 404s are retried. Any other GET failure is returned to the caller.
*/
private Lun getLunByUuidUntilVisible(String authHeader, String lunUuid, String lunName) {
+ Map queryParams = Map.of(OntapStorageConstants.UUID, lunUuid);
for (int attempt = 1; attempt <= OntapStorageConstants.ONTAP_LUN_CLONE_GET_MAX_RETRIES; attempt++) {
try {
- Lun lun = sanFeignClient.getLunByUUID(authHeader, lunUuid);
- if (lun != null) {
+ OntapResponse lunResponse = sanFeignClient.getLunResponse(authHeader, queryParams);
+ if (lunResponse != null && !CollectionUtils.isEmpty(lunResponse.getRecords())
+ && lunResponse.getRecords().get(0) != null) {
+ Lun lun = lunResponse.getRecords().get(0);
if (StringUtils.isBlank(lun.getUuid())) {
lun.setUuid(lunUuid);
}
diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
index 5f1ee4915f82..9ed41d1be8c7 100644
--- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
+++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
@@ -177,7 +177,7 @@ public class OntapStorageConstants {
public static final int ONTAP_FLEXVOL_JOB_POLL_INTERVAL_MS = 2000;
public static final int ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES = 30;
/** GET-by-UUID retries after LUN clone response. */
- public static final int ONTAP_LUN_CLONE_GET_MAX_RETRIES = 5;
+ public static final int ONTAP_LUN_CLONE_GET_MAX_RETRIES = 10;
public static final int ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS = 1000;
/** vm_snapshot_details key for ONTAP FlexVolume-level VM snapshots. */
diff --git a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java
index 13ec49e37b57..d58b095de042 100644
--- a/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java
+++ b/plugins/storage/volume/ontap/src/test/java/org/apache/cloudstack/storage/service/UnifiedSANStrategyTest.java
@@ -1016,7 +1016,7 @@ void testCloneCloudStackVolume_Success() {
when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class)))
.thenReturn(response);
- when(sanFeignClient.getLunByUUID(eq(authHeader), eq("cloned-lun-uuid"))).thenReturn(clonedLun);
+ when(sanFeignClient.getLunResponse(eq(authHeader), anyMap())).thenReturn(response);
CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request);
@@ -1024,7 +1024,7 @@ void testCloneCloudStackVolume_Success() {
assertEquals("cloned-lun-uuid", result.getLun().getUuid());
assertSame(volumeInfo, result.getVolumeInfo());
verify(sanFeignClient).createLun(eq(authHeader), eq(true), any(Lun.class));
- verify(sanFeignClient).getLunByUUID(eq(authHeader), eq("cloned-lun-uuid"));
+ verify(sanFeignClient).getLunResponse(eq(authHeader), anyMap());
}
}
@@ -1405,7 +1405,7 @@ void testCloneCloudStackVolume_SnapshotSource_SendsSnapshotQualifiedName() {
utilityMock.when(() -> OntapStorageUtils.generateAuthHeader("admin", "password"))
.thenReturn(authHeader);
when(sanFeignClient.createLun(eq(authHeader), eq(true), any(Lun.class))).thenReturn(response);
- when(sanFeignClient.getLunByUUID(eq(authHeader), eq("new-lun-uuid"))).thenReturn(createdLun);
+ when(sanFeignClient.getLunResponse(eq(authHeader), anyMap())).thenReturn(response);
CloudStackVolume result = unifiedSANStrategy.cloneCloudStackVolume(request);
@@ -1666,6 +1666,7 @@ void testCloneCloudStackVolume_FeignException_ThrowsException() {
assertThrows(CloudRuntimeException.class,
() -> unifiedSANStrategy.cloneCloudStackVolume(request));
+ verify(sanFeignClient, never()).deleteLun(any(), any(), anyMap());
}
}
From f7836bde138e2c98dd10e56d54672bf60ef401d0 Mon Sep 17 00:00:00 2001
From: sp16743
Date: Wed, 18 Nov 2026 00:34:33 +0530
Subject: [PATCH 4/4] Increase ONTAP LUN clone GET poll interval
---
.../apache/cloudstack/storage/utils/OntapStorageConstants.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
index 9ed41d1be8c7..a2326f576a0b 100644
--- a/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
+++ b/plugins/storage/volume/ontap/src/main/java/org/apache/cloudstack/storage/utils/OntapStorageConstants.java
@@ -178,7 +178,7 @@ public class OntapStorageConstants {
public static final int ONTAP_FLEXVOL_RESOLVE_MAX_RETRIES = 30;
/** GET-by-UUID retries after LUN clone response. */
public static final int ONTAP_LUN_CLONE_GET_MAX_RETRIES = 10;
- public static final int ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS = 1000;
+ public static final int ONTAP_LUN_CLONE_GET_POLL_INTERVAL_MS = 2000;
/** vm_snapshot_details key for ONTAP FlexVolume-level VM snapshots. */
public static final String ONTAP_FLEXVOL_SNAPSHOT = "ontapFlexVolSnapshot";