From f0ec5530e1c851cc97eb9593d6bd6ea19d31fd78 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Sun, 8 Nov 2026 00:01:48 +0100 Subject: [PATCH] linstor: record the size of the DRBD device as the volume size Linstor rounds the storage of a volume up (e.g. to the LVM extent size) and DRBD makes all of it usable, so the device a VM sees is usually a few MiB larger than the size CloudStack requested and records: a 512 MiB volume has a 540909568 byte device. Filesystems the guest creates on the whole disk, or grows to its end (cloud-init growpart), use that space, but the copies CloudStack makes are cut at the recorded size: snapshot backups to secondary storage are shrunk to it, and a copy opened at its own size then holds a filesystem larger than the disk. The driver now records the size of the device after creating a volume (new, cloned from a template or from a snapshot) and after a resize, and notifies the hypervisor of that size. Existing volumes are corrected before a snapshot and before a copy to secondary storage. The recorded size only grows, and the difference is counted against the account's primary storage, except on resize, which counts the recorded size itself. The device size is the smallest usable size of the top layer on the diskful resources; diskless resources only report the size of the volume definition. With DrbdOptions/ExactSize the device has the size of the volume definition. --- .../LinstorPrimaryDataStoreDriverImpl.java | 45 +++++++- .../storage/datastore/util/LinstorUtil.java | 63 +++++++++++ ...LinstorPrimaryDataStoreDriverImplTest.java | 104 ++++++++++++++++++ .../datastore/util/LinstorUtilTest.java | 92 ++++++++++++++++ 4 files changed, 302 insertions(+), 2 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 00383c249f88..20397c1f2fcc 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -73,6 +73,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; @@ -80,6 +81,7 @@ import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.user.ResourceLimitService; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; @@ -133,6 +135,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver @Inject private HostDao _hostDao; @Inject private VMTemplateDao _vmTemplateDao; + @Inject private DiskOfferingDao _diskOfferingDao; + @Inject private ResourceLimitService _resourceLimitMgr; private final Map volumeStatsLastUpdate = new HashMap<>(); private final Map> volumeStats = new HashMap<>(); @@ -354,6 +358,33 @@ private void resizeResource(DevelopersApi api, String resourceName, long sizeByt } } + /** + * Records the size of the volume's Linstor device as the size of the volume, if it is larger. Linstor rounds the + * storage of a volume up and the VM sees all of it, so copies of the volume (snapshot backups, templates, + * downloads) have to cover all of it too. + * @param adjustResourceCount whether to count the difference against the account's primary storage + * @return the size of the volume + */ + protected long recordDeviceSize(DevelopersApi api, VolumeVO volume, String rscName, boolean adjustResourceCount) { + final long size = volume.getSize() != null ? volume.getSize() : 0; + try { + final long deviceSize = LinstorUtil.getVolumeDeviceSize(api, rscName); + if (deviceSize > size) { + logger.info("Linstor: the device of {} has {} bytes, recording that instead of {}", volume, deviceSize, size); + volume.setSize(deviceSize); + _volumeDao.update(volume.getId(), volume); + if (adjustResourceCount) { + _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(volume.getAccountId(), + volume.isDisplayVolume(), deviceSize - size, _diskOfferingDao.findById(volume.getDiskOfferingId())); + } + return deviceSize; + } + } catch (ApiException apiEx) { + logger.warn("Linstor: unable to get the device size of {}, keeping {} bytes: {}", rscName, size, apiEx.getBestMessage()); + } + return size; + } + /** * Update resource-definitions resource-group to the correct one if it isn't already the intended. * @param api Linstor api @@ -614,6 +645,7 @@ public void createAsync(DataStore dataStore, DataObject vol, AsyncCompletionCall volume.setPath(vol.getUuid()); _volumeDao.update(volume.getId(), volume); + recordDeviceSize(getLinstorAPI(storagePool), volume, LinstorUtil.RSC_PREFIX + volume.getPath(), true); break; case SNAPSHOT: logger.debug("createAsync - SNAPSHOT"); @@ -975,8 +1007,11 @@ private Answer copyVolume(DataObject srcData, DataObject dstData) { final StoragePoolVO pool = _storagePoolDao.findById(srcVolInfo.getDataStore().getId()); final DevelopersApi api = getLinstorAPI(pool); final String rscName = LinstorUtil.RSC_PREFIX + srcVolInfo.getPath(); + // volumes created before their device size was recorded + final long size = recordDeviceSize(api, _volumeDao.findById(srcVolInfo.getId()), rscName, true); VolumeObjectTO to = (VolumeObjectTO) srcVolInfo.getTO(); + to.setSize(size); // patch source format // Linstor volumes are stored as RAW, but we can't set the correct format as RAW (we use QCOW2) // otherwise create template from snapshot won't work, because this operation @@ -1205,12 +1240,13 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As private CreateCmdResult notifyResize( VolumeObject vol, long oldSize, + long newSize, ResizeVolumePayload resizeParameter) { StoragePool pool = (StoragePool) vol.getDataStore(); ResizeVolumeCommand resizeCmd = - new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, resizeParameter.newSize, resizeParameter.shrinkOk, + new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), oldSize, newSize, resizeParameter.shrinkOk, resizeParameter.instanceName, null); CreateCmdResult result = new CreateCmdResult(null, null); try { @@ -1244,6 +1280,7 @@ public void resize(DataObject data, AsyncCompletionCallback cal final long oldSize = vol.getSize(); String errMsg = null; + long newSize = resizeParameter.newSize; VolumeDefinitionModify dfm = new VolumeDefinitionModify(); dfm.setSizeKib(resizeParameter.newSize / 1024); try @@ -1257,6 +1294,8 @@ public void resize(DataObject data, AsyncCompletionCallback cal volume.setMaxIops(resizeParameter.newMaxIops); volume.setSize(resizeParameter.newSize); _volumeDao.update(volume.getId(), volume); + // the resize counts the recorded size against the account's primary storage + newSize = recordDeviceSize(api, volume, rscName, false); } } catch (ApiException apiExc) { @@ -1270,7 +1309,7 @@ public void resize(DataObject data, AsyncCompletionCallback cal result.setResult(errMsg); } else { // notify guests - result = notifyResize(vol, oldSize, resizeParameter); + result = notifyResize(vol, oldSize, newSize, resizeParameter); } callback.complete(result); @@ -1313,6 +1352,8 @@ public void takeSnapshot(SnapshotInfo snapshotInfo, AsyncCompletionCallback rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), true, null, null, null); + if (rscDfns.isEmpty() || CollectionUtils.isEmpty(rscDfns.get(0).getVolumeDefinitions())) { + return 0; + } + ResourceDefinition rscDfn = rscDfns.get(0); + final long definedSize = rscDfn.getVolumeDefinitions().get(0).getSizeKib() * 1024; + if (rscDfn.getProps() != null && Boolean.parseBoolean(rscDfn.getProps().get(PROP_DRBD_EXACT_SIZE))) { + return definedSize; + } + + List resources = api.viewResources( + Collections.emptyList(), + Collections.singletonList(rscName), + Collections.emptyList(), + null, + null, + null); + Long deviceKib = null; + for (ResourceWithVolumes rsc : resources) { + if (!isRscDiskless(rsc) && !rsc.getVolumes().isEmpty()) { + Long usableKib = getTopLayerUsableSizeKib(rsc.getVolumes().get(0)); + if (usableKib != null && usableKib > 0 && (deviceKib == null || usableKib < deviceKib)) { + deviceKib = usableKib; + } + } + } + return deviceKib != null ? Math.max(definedSize, deviceKib * 1024) : definedSize; + } + /** * Checks if all diskful resource are on a zeroed block device. * @param pool Linstor pool to use diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java index 69a6e90128f7..30f5d9209d42 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -19,8 +19,15 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.DrbdVolume; import com.linbit.linstor.api.model.LayerType; +import com.linbit.linstor.api.model.Properties; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.ResourceWithVolumes; +import com.linbit.linstor.api.model.Volume; +import com.linbit.linstor.api.model.VolumeDefinition; +import com.linbit.linstor.api.model.VolumeLayer; import java.util.Arrays; import java.util.Collections; @@ -42,6 +49,11 @@ import org.mockito.junit.MockitoJUnitRunner; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.ResourceLimitService; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -54,6 +66,15 @@ public class LinstorPrimaryDataStoreDriverImplTest { @Mock private SnapshotDataStoreDao snapshotStoreDao; + @Mock + private VolumeDao volumeDao; + + @Mock + private DiskOfferingDao diskOfferingDao; + + @Mock + private ResourceLimitService resourceLimitMgr; + @InjectMocks private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver; @@ -178,4 +199,87 @@ public void testGetEncryptedLayerList() throws ApiException { layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); } + + /** + * A 512 MiB volume definition whose DRBD device has 540909568 bytes (LVM-thin rounds the storage up). + */ + private void mockLinstorVolume(String rscName) throws ApiException { + VolumeDefinition vlmDfn = new VolumeDefinition(); + vlmDfn.setSizeKib(524288L); + ResourceDefinition rscDfn = new ResourceDefinition(); + rscDfn.setName(rscName); + rscDfn.addVolumeDefinitionsItem(vlmDfn); + rscDfn.setProps(new Properties()); + when(api.resourceDefinitionList(Collections.singletonList(rscName), true, null, null, null)) + .thenReturn(Collections.singletonList(rscDfn)); + + DrbdVolume drbdVolume = new DrbdVolume(); + drbdVolume.setUsableSizeKib(528232L); + VolumeLayer layer = new VolumeLayer(); + layer.setType(LayerType.DRBD); + layer.setData(drbdVolume); + Volume volume = new Volume(); + volume.addLayerDataListItem(layer); + ResourceWithVolumes rsc = new ResourceWithVolumes(); + rsc.setNodeName("nodeA"); + rsc.setVolumes(Collections.singletonList(volume)); + when(api.viewResources(Collections.emptyList(), Collections.singletonList(rscName), Collections.emptyList(), + null, null, null)).thenReturn(Collections.singletonList(rsc)); + } + + private VolumeVO mockVolume(long size) { + VolumeVO volume = mock(VolumeVO.class); + Mockito.lenient().when(volume.getId()).thenReturn(7L); + Mockito.lenient().when(volume.getAccountId()).thenReturn(2L); + Mockito.lenient().when(volume.isDisplayVolume()).thenReturn(true); + Mockito.lenient().when(volume.getDiskOfferingId()).thenReturn(3L); + when(volume.getSize()).thenReturn(size); + return volume; + } + + @Test + public void testRecordDeviceSizeRecordsTheLargerDevice() throws ApiException { + mockLinstorVolume("cs-vol"); + VolumeVO volume = mockVolume(536870912L); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOfferingDao.findById(3L)).thenReturn(diskOffering); + + long size = linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true); + + Assert.assertEquals(540909568L, size); + Mockito.verify(volume).setSize(540909568L); + Mockito.verify(volumeDao).update(7L, volume); + Mockito.verify(resourceLimitMgr).incrementVolumePrimaryStorageResourceCount(2L, true, 4038656L, diskOffering); + } + + @Test + public void testRecordDeviceSizeWithoutResourceCount() throws ApiException { + // a resize counts the recorded size itself + mockLinstorVolume("cs-vol"); + VolumeVO volume = mockVolume(536870912L); + + Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", false)); + Mockito.verify(volumeDao).update(7L, volume); + Mockito.verifyNoInteractions(resourceLimitMgr); + } + + @Test + public void testRecordDeviceSizeKeepsRecordedDeviceSize() throws ApiException { + mockLinstorVolume("cs-vol"); + VolumeVO volume = mockVolume(540909568L); + + Assert.assertEquals(540909568L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true)); + Mockito.verify(volume, Mockito.never()).setSize(Mockito.anyLong()); + Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr); + } + + @Test + public void testRecordDeviceSizeKeepsSizeIfLinstorFails() throws ApiException { + when(api.resourceDefinitionList(Collections.singletonList("cs-vol"), true, null, null, null)) + .thenThrow(new ApiException(500, "controller unreachable")); + VolumeVO volume = mockVolume(536870912L); + + Assert.assertEquals(536870912L, linstorPrimaryDataStoreDriver.recordDeviceSize(api, volume, "cs-vol", true)); + Mockito.verifyNoInteractions(volumeDao, resourceLimitMgr); + } } diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java index fc4153e33644..69cc74c1a961 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -20,14 +20,20 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.DrbdVolume; +import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceState; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.StoragePool; +import com.linbit.linstor.api.model.StorageVolume; import com.linbit.linstor.api.model.Volume; +import com.linbit.linstor.api.model.VolumeDefinition; +import com.linbit.linstor.api.model.VolumeLayer; import java.util.Arrays; import java.util.Collections; @@ -38,8 +44,10 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -199,4 +207,88 @@ public void testGetRscGroupStoragePools() throws ApiException { .collect(Collectors.toList()); Assert.assertEquals(names, Arrays.asList("nodeA::thinpool", "nodeB::thinpool", "nodeC::thinpool")); } + + private void mockResourceDefinition(String rscName, long sizeKib, Properties props) throws ApiException { + VolumeDefinition vlmDfn = new VolumeDefinition(); + vlmDfn.setSizeKib(sizeKib); + ResourceDefinition rscDfn = new ResourceDefinition(); + rscDfn.setName(rscName); + rscDfn.addVolumeDefinitionsItem(vlmDfn); + rscDfn.setProps(props); + when(api.resourceDefinitionList(Collections.singletonList(rscName), true, null, null, null)) + .thenReturn(Collections.singletonList(rscDfn)); + } + + private ResourceWithVolumes mockResource(String node, boolean diskless, LayerType topLayer, long usableSizeKib) { + VolumeLayer layer = new VolumeLayer(); + layer.setType(topLayer); + if (topLayer == LayerType.DRBD) { + DrbdVolume drbdVolume = new DrbdVolume(); + drbdVolume.setUsableSizeKib(usableSizeKib); + layer.setData(drbdVolume); + } else { + StorageVolume storageVolume = new StorageVolume(); + storageVolume.setUsableSizeKib(usableSizeKib); + layer.setData(storageVolume); + } + Volume volume = new Volume(); + volume.addLayerDataListItem(layer); + ResourceWithVolumes rsc = new ResourceWithVolumes(); + rsc.setNodeName(node); + rsc.setFlags(diskless ? Collections.singletonList(ApiConsts.FLAG_DISKLESS) : Collections.emptyList()); + rsc.setVolumes(Collections.singletonList(volume)); + return rsc; + } + + private void mockResources(String rscName, ResourceWithVolumes... resources) throws ApiException { + when(api.viewResources(Collections.emptyList(), Collections.singletonList(rscName), Collections.emptyList(), + null, null, null)).thenReturn(Arrays.asList(resources)); + } + + @Test + public void testGetVolumeDeviceSizeIsTheSmallestDiskfulDrbdDevice() throws ApiException { + // a 512 MiB volume definition on LVM-thin, the diskless resource only reports the volume definition's size + mockResourceDefinition("cs-vol", 524288L, new Properties()); + mockResources("cs-vol", + mockResource("nodeA", false, LayerType.DRBD, 528232L), + mockResource("nodeB", true, LayerType.DRBD, 524288L), + mockResource("nodeC", false, LayerType.DRBD, 528236L)); + + Assert.assertEquals(528232L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + } + + @Test + public void testGetVolumeDeviceSizeWithoutDrbd() throws ApiException { + mockResourceDefinition("cs-vol", 524288L, new Properties()); + mockResources("cs-vol", mockResource("nodeA", false, LayerType.STORAGE, 528384L)); + + Assert.assertEquals(528384L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + } + + @Test + public void testGetVolumeDeviceSizeWithExactSize() throws ApiException { + Properties props = new Properties(); + props.put("DrbdOptions/ExactSize", "True"); + mockResourceDefinition("cs-vol", 524288L, props); + + Assert.assertEquals(524288L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + Mockito.verify(api, Mockito.never()).viewResources(any(), any(), any(), any(), any(), any()); + } + + @Test + public void testGetVolumeDeviceSizeFallsBackToVolumeDefinition() throws ApiException { + // no diskful resource reports a size, e.g. while the resources are being created + mockResourceDefinition("cs-vol", 524288L, new Properties()); + mockResources("cs-vol", mockResource("nodeB", true, LayerType.DRBD, 524288L)); + + Assert.assertEquals(524288L * 1024, LinstorUtil.getVolumeDeviceSize(api, "cs-vol")); + } + + @Test + public void testGetVolumeDeviceSizeOfMissingResourceDefinition() throws ApiException { + when(api.resourceDefinitionList(Collections.singletonList("cs-gone"), true, null, null, null)) + .thenReturn(Collections.emptyList()); + + Assert.assertEquals(0L, LinstorUtil.getVolumeDeviceSize(api, "cs-gone")); + } }