From 3da4e542e4cc3f360bbdf849cf59667904fabf04 Mon Sep 17 00:00:00 2001 From: Brad House Date: Mon, 9 Nov 2026 20:19:11 +0000 Subject: [PATCH 1/2] CKS: check and renew the control plane certificates kubeadm issues the control plane certificates of a CKS cluster for one year, and nothing in CKS ever renewed them or looked at their expiry. CKS also only refreshed the kubeconfig that getKubernetesClusterConfig returns on start, so even after an upgrade, which renews the certificates, it kept handing out the old admin certificate until it expired. - Refresh the stored kubeconfig after an upgrade and a renewal, and whenever the expiry check finds it older than the cluster's admin.conf. - Check the certificate expiry of every control node of running clusters with kubeadm certs check-expiration, at most every cloud.kubernetes.cluster.certificate.check.interval hours (6), on a scanner thread of its own. Raise an alert and an event as the first certificate, and separately the first certificate authority, reach each of cloud.kubernetes.cluster.certificate.expiry.alert.days (60,30). - Add the rotateKubernetesClusterCertificates API. It renews the certificates on one control node at a time, restarts that node's control plane static pods, waits for its API server and the node to be ready, stops at the first node that fails and raises an alert, then refreshes the stored kubeconfig. The cluster is RotatingCertificates meanwhile, so no other operation can start. A cluster in Alert can be rotated too, to retry a failed rotation or to rescue a cluster whose certificates already expired: when a certificate on any control node has expired, every control node is renewed before waiting, since stacked etcd has no quorum until they all are. - Renew automatically cloud.kubernetes.cluster.certificate.auto.renew.days (30) before expiry when cloud.kubernetes.cluster.certificate.auto.renew.enabled (true) is set for the cluster's account, through the same rolling path. - Keep root's kubeconfig on control nodes, which CKS itself uses for kubectl, in step with admin.conf after an upgrade or a renewal. Certificate authorities are reported but not rotated. Clusters older than Kubernetes 1.20, which have no kubeadm certs command, are skipped. External etcd nodes serve plain HTTP and hold no certificates, so only control nodes are renewed. Signed-off-by: Brad House --- .../kubernetes/cluster/KubernetesCluster.java | 8 + .../apache/cloudstack/alert/AlertService.java | 1 + .../cluster/KubernetesClusterEventTypes.java | 2 + .../cluster/KubernetesClusterManagerImpl.java | 390 ++++++++++++- .../cluster/KubernetesClusterService.java | 40 ++ .../KubernetesClusterActionWorker.java | 79 +++ .../KubernetesClusterCertificateWorker.java | 476 ++++++++++++++++ .../KubernetesClusterStartWorker.java | 20 +- .../KubernetesClusterUpgradeWorker.java | 19 +- .../KubernetesClusterCertificateUtil.java | 346 +++++++++++ .../cluster/utils/KubernetesClusterUtil.java | 11 +- ...otateKubernetesClusterCertificatesCmd.java | 114 ++++ .../script/rotate-kubernetes-certificates.sh | 235 ++++++++ .../resources/script/upgrade-kubernetes.sh | 21 + ...netesClusterCertificateManagementTest.java | 536 ++++++++++++++++++ ...ubernetesClusterCertificateWorkerTest.java | 526 +++++++++++++++++ .../KubernetesClusterUpgradeWorkerTest.java | 72 +++ .../KubernetesClusterCertificateUtilTest.java | 236 ++++++++ 18 files changed, 3112 insertions(+), 20 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 80f6a6045c72..a316f842b3c5 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -47,6 +47,7 @@ enum Event { AddNodeRequested, RemoveNodeRequested, UpgradeRequested, + CertificateRotationRequested, OperationSucceeded, OperationFailed, CreateFailed, @@ -62,6 +63,7 @@ enum State { Scaling("Transient state in which resources are either getting scaled up/down"), ScalingStoppedCluster("Transient state in which the service offerings of stopped clusters are getting scaled"), Upgrading("Transient state in which cluster is getting upgraded"), + RotatingCertificates("Transient state in which the control plane certificates of the cluster are getting renewed"), Importing("Transient state in which additional nodes are added as worker nodes to a cluster"), RemovingNodes("Transient state in which additional nodes are removed from a cluster"), Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), @@ -106,6 +108,12 @@ enum State { s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.CertificateRotationRequested, State.RotatingCertificates); + // lets a failed or interrupted rotation be retried, and rescues a cluster whose certificates already expired + s_fsm.addTransition(State.Alert, Event.CertificateRotationRequested, State.RotatingCertificates); + s_fsm.addTransition(State.RotatingCertificates, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.RotatingCertificates, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing); s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing); s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running); diff --git a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java index a9c2abc11ce7..b4ec59d38012 100644 --- a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java +++ b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java @@ -86,6 +86,7 @@ private AlertType(short type, String name, boolean isDefault) { public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_FAILED = new AlertType((short)35, "ALERT.BACKUP.VALIDATION.FAILED", true, true); public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_UNABLE_TO_VALIDATE = new AlertType((short)36, "ALERT.BACKUP.VALIDATION.UNABLE.TO.VALIDATE", true, true); public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_CLEANUP_FAILED = new AlertType((short)37, "ALERT.BACKUP.VALIDATION.CLEANUP_FAILED", true, true); + public static final AlertType ALERT_TYPE_KUBERNETES_CLUSTER_CERTIFICATE = new AlertType((short)38, "ALERT.KUBERNETES.CLUSTER.CERTIFICATE", true, true); public short getType() { return type; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index d11a0fc3dea5..54d49d43059b 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -26,4 +26,6 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE"; + public static final String EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE = "KUBERNETES.CLUSTER.CERTIFICATES.ROTATE"; + public static final String EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY = "KUBERNETES.CLUSTER.CERTIFICATES.EXPIRY"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index eb4ebb78ee48..23bc02d70956 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -39,8 +39,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -86,6 +89,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; @@ -124,6 +128,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; +import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; @@ -152,6 +157,7 @@ import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.deploy.DeployDestination; import com.cloud.event.ActionEvent; +import com.cloud.event.EventVO; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; @@ -167,12 +173,16 @@ import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterCertificateWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterDestroyWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; @@ -312,8 +322,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER.name(), CONTROL.name(), ETCD.name()); + protected static final long CERTIFICATE_SCANNER_INTERVAL_SECONDS = 15 * 60; + protected static final int CERTIFICATE_ROTATION_THREADS = 4; + /** Automatic renewal is not retried within this long of a failed rotation; the failure already raised an alert. */ + protected static final long CERTIFICATE_AUTO_RENEW_RETRY_BACKOFF_MS = TimeUnit.HOURS.toMillis(24); + ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; + ScheduledExecutorService _certificateScanner; + ExecutorService _certificateRotationExecutor; + private final Set queuedCertificateRotations = ConcurrentHashMap.newKeySet(); @Inject public KubernetesClusterDao kubernetesClusterDao; @@ -1078,6 +1096,7 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) { BaseCmd.getCommandNameByClass(StartKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(RotateKubernetesClusterCertificatesCmd.class), BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) ).contains(cmdName); @@ -1955,6 +1974,9 @@ public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, S logger.debug("Kubernetes cluster {} is already in starting state", kubernetesCluster); return true; } + if (KubernetesCluster.State.RotatingCertificates.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is renewing its certificates", kubernetesCluster.getName())); + } final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (zone == null) { logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster %s", kubernetesCluster)); @@ -2113,6 +2135,11 @@ public boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudR } return true; } + // the rotation has the control plane manifests moved aside while it restarts the static pods + if (KubernetesCluster.State.RotatingCertificates.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is renewing its certificates, stop it once that is done", + kubernetesCluster.getName())); + } KubernetesClusterStopWorker stopWorker = new KubernetesClusterStopWorker(kubernetesCluster, this); stopWorker = ComponentContext.inject(stopWorker); return stopWorker.stop(); @@ -2260,7 +2287,7 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC response.setId(kubernetesCluster.getUuid()); response.setName(kubernetesCluster.getName()); String configData = ""; - KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KubernetesClusterActionWorker.KUBE_CONFIG_DATA_DETAIL); if (clusterDetailsVO != null && StringUtils.isNotEmpty(clusterDetailsVO.getValue())) { configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); } else { @@ -2341,6 +2368,43 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws return upgradeWorker.upgradeCluster(); } + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE, + eventDescription = "rotating Kubernetes cluster certificates", async = true) + public boolean rotateKubernetesClusterCertificates(RotateKubernetesClusterCertificatesCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO kubernetesCluster = validateKubernetesClusterCertificateRotationParameters(cmd); + return createCertificateWorker(kubernetesCluster).rotateCertificates(); + } + + protected KubernetesClusterVO validateKubernetesClusterCertificateRotationParameters(RotateKubernetesClusterCertificatesCmd cmd) { + final Long kubernetesClusterId = cmd.getId(); + KubernetesClusterVO kubernetesCluster = kubernetesClusterId == null ? null : kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (!isCommandSupported(kubernetesCluster, cmd.getActualCommandName())) { + throw new InvalidParameterValueException(String.format("Certificate rotation is not supported for an externally managed Kubernetes cluster (%s)", kubernetesCluster.getName())); + } + // Alert lets a failed rotation be retried and a cluster whose certificates already expired be rescued + if (!Arrays.asList(KubernetesCluster.State.Running, KubernetesCluster.State.Alert).contains(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is not in Running or Alert state", kubernetesCluster.getName())); + } + KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (!KubernetesClusterCertificateWorker.isCertificateManagementSupported(clusterVersion)) { + throw new InvalidParameterValueException(String.format("Certificate rotation of Kubernetes cluster %s needs Kubernetes %s or later", + kubernetesCluster.getName(), KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT)); + } + return kubernetesCluster; + } + + protected KubernetesClusterCertificateWorker createCertificateWorker(KubernetesCluster kubernetesCluster) { + return ComponentContext.inject(new KubernetesClusterCertificateWorker(kubernetesCluster, this)); + } + @Override @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE, eventDescription = "updating Kubernetes cluster affinity groups") @@ -2780,6 +2844,7 @@ public List> getCommands() { cmdList.add(GetKubernetesClusterConfigCmd.class); cmdList.add(ScaleKubernetesClusterCmd.class); cmdList.add(UpgradeKubernetesClusterCmd.class); + cmdList.add(RotateKubernetesClusterCertificatesCmd.class); cmdList.add(AddVirtualMachinesToKubernetesClusterCmd.class); cmdList.add(RemoveVirtualMachinesFromKubernetesClusterCmd.class); cmdList.add(AddNodesToKubernetesClusterCmd.class); @@ -2909,8 +2974,10 @@ public void reallyRun() { startWorker = ComponentContext.inject(startWorker); startWorker.reconcileAlertCluster(); } else if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + // unless another operation (e.g. a certificate rotation) took the cluster meanwhile + if (stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } } } catch (Exception e) { logger.warn("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster: {} status scanner", kubernetesCluster, e); @@ -2957,6 +3024,295 @@ public void reallyRun() { } } + /* Checks the control plane certificate expiry of running clusters every few hours, raises alerts and events as + configured thresholds are reached and renews the certificates automatically shortly before they expire. Runs on + its own thread, as a renewal restarts the control plane one node at a time and can take minutes. + */ + public class KubernetesClusterCertificateScanner extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock lock = GlobalLock.getInternLock("KubernetesCluster.Certificate.Scanner.Lock"); + try { + if (lock.lock(3)) { + try { + reallyRun(); + } finally { + lock.unlock(); + } + } + } finally { + lock.releaseRef(); + } + } + + public void reallyRun() { + if (!KubernetesServiceEnabled.value()) { + return; + } + try { + recoverInterruptedCertificateRotations(new Date()); + // Alert too: a cluster can sit in Alert for long (e.g. a stopped worker) and its certificates still expire + List clusters = new ArrayList<>(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Running)); + clusters.addAll(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Alert)); + for (KubernetesClusterVO listed : clusters) { + // a pass can take minutes; skip a cluster stopped or otherwise taken since it was listed + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(listed.getId()); + if (kubernetesCluster == null || !Arrays.asList(KubernetesCluster.State.Running, KubernetesCluster.State.Alert).contains(kubernetesCluster.getState())) { + continue; + } + try { + processKubernetesClusterCertificates(kubernetesCluster, new Date()); + } catch (Exception e) { + logger.warn("Failed to process the certificates of Kubernetes cluster: {}", kubernetesCluster, e); + } + } + } catch (Exception e) { + logger.warn("Caught exception while running the Kubernetes cluster certificate scanner", e); + } + } + } + + private Long getLongDetail(long kubernetesClusterId, String name) { + KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesClusterId, name); + if (detail == null || StringUtils.isBlank(detail.getValue())) { + return null; + } + try { + return Long.parseLong(detail.getValue().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + /** + * A cluster stays RotatingCertificates if the management server running the rotation went away. Move it to Alert, + * so the status scanner can reconcile it and the operator is told: right away when the rotation belonged to this + * management server and is no longer running here (it restarted), otherwise once the rotation is well past its + * timeout. + */ + protected void recoverInterruptedCertificateRotations(Date now) { + long staleAfterMs = 2 * KubernetesClusterCertificateRotationTimeout.value() * 1000L + TimeUnit.MINUTES.toMillis(10); + for (KubernetesClusterVO kubernetesCluster : kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) { + if (KubernetesClusterCertificateWorker.isRotationActive(kubernetesCluster.getId())) { + continue; + } + Long started = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + if (started == null) { + // a rotation writes its start just after entering the state; never write that detail here, it would + // race with it. Note when it was first seen instead, to time a rotation that died in between. + started = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL); + } + if (started == null) { + // the rotation may have only just entered the state, or just finished; time it from now if still running + KubernetesClusterVO current = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (current == null || !KubernetesCluster.State.RotatingCertificates.equals(current.getState()) + || KubernetesClusterCertificateWorker.isRotationActive(kubernetesCluster.getId())) { + continue; + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL, String.valueOf(now.getTime()), false); + continue; + } + Long owner = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + boolean ownedHere = owner != null && owner == ManagementServerNode.getManagementServerId(); + if (!ownedHere && now.getTime() - started < staleAfterMs) { + continue; + } + logger.warn("Certificate rotation of Kubernetes cluster {} did not finish, moving it to Alert", kubernetesCluster); + if (stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed)) { + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(now.getTime()), false); + createCertificateWorker(kubernetesCluster).sendAlert( + String.format("Certificate rotation of Kubernetes cluster %s did not finish", kubernetesCluster.getName()), + String.format("The certificate rotation of Kubernetes cluster %s was interrupted, possibly by a management server restart. " + + "Check the cluster and run rotateKubernetesClusterCertificates again", kubernetesCluster.getName())); + } + } + } + + protected void processKubernetesClusterCertificates(KubernetesCluster kubernetesCluster, Date now) { + Integer checkIntervalHours = KubernetesClusterCertificateCheckInterval.value(); + if (checkIntervalHours == null || checkIntervalHours <= 0) { + return; + } + Long lastCheck = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_DETAIL); + if (lastCheck != null && now.getTime() - lastCheck < TimeUnit.HOURS.toMillis(checkIntervalHours)) { + return; + } + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (!KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)) { + logger.debug("Skipping the certificate expiry check of Kubernetes cluster {}, its Kubernetes version is older than {}", + kubernetesCluster, KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT); + return; + } + // Recorded before checking, so an unreachable cluster is retried at the next interval, not on every pass + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_DETAIL, String.valueOf(now.getTime()), false); + + KubernetesClusterCertificateWorker worker = createCertificateWorker(kubernetesCluster); + CertificateExpiryReport report; + try { + report = worker.checkCertificateExpiry(); + } catch (CloudRuntimeException e) { + // e.g. no address to reach the cluster on: as unwatched as when every node fails + logger.warn("Unable to check the certificate expiry of Kubernetes cluster {}", kubernetesCluster, e); + report = new CertificateExpiryReport(); + report.addUncheckedNode("all control nodes"); + } + boolean allNodesChecked = report.getUncheckedNodes().isEmpty(); + if (!allNodesChecked) { + logger.warn("Unable to check the certificate expiry on control nodes {} of Kubernetes cluster {}", report.getUncheckedNodes(), kubernetesCluster); + } + if (report.isEmpty()) { + alertOnFailedCertificateCheck(kubernetesCluster, worker, report, now); + return; + } + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL); + CertificateExpiry firstLeaf = report.getFirstLeafExpiry(); + CertificateExpiry firstAuthority = report.getFirstAuthorityExpiry(); + logger.info("Kubernetes cluster {}: first control plane certificate to expire is {}, first certificate authority is {}", + kubernetesCluster, firstLeaf, firstAuthority); + + int autoRenewDays = Math.min(Objects.requireNonNullElse(KubernetesClusterCertificateAutoRenewDays.valueIn(kubernetesCluster.getAccountId()), 0), + KubernetesClusterCertificateUtil.MAX_AUTO_RENEW_DAYS); + boolean autoRenew = autoRenewDays > 0 && Boolean.TRUE.equals(KubernetesClusterCertificateAutoRenewEnabled.valueIn(kubernetesCluster.getAccountId())); + List thresholds = KubernetesClusterCertificateUtil.parseAlertThresholds(KubernetesClusterCertificateExpiryAlertDays.value()); + String leafAdvice; + if (firstLeaf != null && firstLeaf.isExternallyManaged()) { + leafAdvice = "It is externally managed, so it has to be renewed outside CloudStack."; + } else if (autoRenew && !allNodesChecked) { + leafAdvice = String.format("Automatic renewal is on hold until control nodes %s can be checked; fix their access or run rotateKubernetesClusterCertificates.", + report.getUncheckedNodes()); + } else if (autoRenew) { + leafAdvice = String.format("They are renewed automatically %d days before they expire while the cluster is Running, or run rotateKubernetesClusterCertificates.", + autoRenewDays); + } else { + leafAdvice = "Run rotateKubernetesClusterCertificates to renew them."; + } + alertOnCertificateExpiry(kubernetesCluster, worker, firstLeaf, thresholds, KubernetesClusterCertificateWorker.CERTIFICATE_ALERT_DETAIL, now, + "control plane certificate", leafAdvice); + alertOnCertificateExpiry(kubernetesCluster, worker, firstAuthority, thresholds, KubernetesClusterCertificateWorker.CA_CERTIFICATE_ALERT_DETAIL, now, + "certificate authority", "CloudStack does not rotate certificate authorities; it has to be done by hand."); + + // Externally managed certificates can't be renewed by kubeadm and would make every check renew again + CertificateExpiry firstRenewable = report.getFirstRenewableLeafExpiry(); + // a cluster in Alert may have nodes down; renewing it is left to the operator + if (firstRenewable == null || !autoRenew || !KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) || + !KubernetesClusterCertificateUtil.isAutoRenewDue(KubernetesClusterCertificateUtil.daysUntil(firstRenewable.getExpires(), now), autoRenewDays)) { + return; + } + Long lastFailure = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL); + if (!allNodesChecked) { + // the rotation would renew the reachable nodes and then fail on the others, leaving the cluster in Alert + logger.warn("Not renewing the certificates of Kubernetes cluster {} automatically while control nodes {} can't be reached", + kubernetesCluster, report.getUncheckedNodes()); + return; + } + if (lastFailure != null && now.getTime() - lastFailure < CERTIFICATE_AUTO_RENEW_RETRY_BACKOFF_MS) { + logger.info("Not renewing the certificates of Kubernetes cluster {} automatically, the last rotation failed less than {} hours ago", + kubernetesCluster, TimeUnit.MILLISECONDS.toHours(CERTIFICATE_AUTO_RENEW_RETRY_BACKOFF_MS)); + return; + } + logger.info("Renewing the certificates of Kubernetes cluster {} automatically, {} expires on {}", kubernetesCluster, firstRenewable.getName(), firstRenewable.getExpires()); + scheduleAutomaticRotation(kubernetesCluster); + } + + /** + * Runs the rotation on its own thread, so a renewal that takes minutes holds up neither the expiry checks of the + * other clusters nor the scanner lock. The state transition keeps a second rotation of the same cluster out. + */ + protected void scheduleAutomaticRotation(KubernetesCluster kubernetesCluster) { + if (_certificateRotationExecutor == null) { + runAutomaticRotation(kubernetesCluster); + return; + } + // a renewal still waiting for a thread must not be queued again by a later scan + if (!queuedCertificateRotations.add(kubernetesCluster.getId())) { + return; + } + final long queuedAt = System.currentTimeMillis(); + try { + _certificateRotationExecutor.execute(new ManagedContextRunnable() { + @Override + protected void runInContext() { + try { + runQueuedAutomaticRotation(kubernetesCluster.getId(), queuedAt); + } finally { + queuedCertificateRotations.remove(kubernetesCluster.getId()); + } + } + }); + } catch (RejectedExecutionException e) { + queuedCertificateRotations.remove(kubernetesCluster.getId()); + logger.warn("Unable to queue the automatic certificate renewal of Kubernetes cluster {}", kubernetesCluster, e); + } + } + + /** The cluster may have changed while the renewal waited for a thread, e.g. been renewed through the API. */ + protected void runQueuedAutomaticRotation(long kubernetesClusterId, long queuedAt) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || !KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + logger.info("Skipping the queued automatic certificate renewal of Kubernetes cluster {}, it is no longer Running", kubernetesClusterId); + return; + } + Long lastSuccess = getLongDetail(kubernetesClusterId, KubernetesClusterCertificateWorker.ROTATION_SUCCEEDED_DETAIL); + Long lastFailure = getLongDetail(kubernetesClusterId, KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL); + if ((lastSuccess != null && lastSuccess >= queuedAt) || (lastFailure != null && lastFailure >= queuedAt)) { + logger.info("Skipping the queued automatic certificate renewal of Kubernetes cluster {}, it was rotated meanwhile", kubernetesCluster); + return; + } + runAutomaticRotation(kubernetesCluster); + } + + protected void runAutomaticRotation(KubernetesCluster kubernetesCluster) { + KubernetesClusterCertificateWorker rotationWorker = createCertificateWorker(kubernetesCluster); + rotationWorker.setAutomatic(true); + try { + rotationWorker.rotateCertificates(); + } catch (Exception e) { + logger.warn("Automatic certificate renewal of Kubernetes cluster {} failed", kubernetesCluster, e); + } + } + + /** Alerts, at most once a day, when the expiry of no control node could be checked, so expiry can't go unnoticed */ + protected void alertOnFailedCertificateCheck(KubernetesCluster kubernetesCluster, KubernetesClusterCertificateWorker worker, + CertificateExpiryReport report, Date now) { + if (report.getUncheckedNodes().isEmpty()) { + return; + } + Long lastAlert = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL); + if (lastAlert != null && now.getTime() - lastAlert < TimeUnit.DAYS.toMillis(1)) { + return; + } + String subject = String.format("Unable to check the certificate expiry of Kubernetes cluster %s", kubernetesCluster.getName()); + worker.sendAlert(subject, String.format("%s: none of its control nodes %s could be checked over SSH. Its certificates are not watched " + + "or renewed automatically until they can.", subject, report.getUncheckedNodes())); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL, + String.valueOf(now.getTime()), false); + } + + protected void alertOnCertificateExpiry(KubernetesCluster kubernetesCluster, KubernetesClusterCertificateWorker worker, CertificateExpiry expiry, + List thresholds, String alertDetail, Date now, String what, String advice) { + if (expiry == null) { + return; + } + long daysLeft = KubernetesClusterCertificateUtil.daysUntil(expiry.getExpires(), now); + KubernetesClusterDetailsVO alertState = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), alertDetail); + Integer lastAlerted = KubernetesClusterCertificateUtil.getLastAlertedThreshold(alertState == null ? null : alertState.getValue(), expiry.getExpires()); + Integer threshold = KubernetesClusterCertificateUtil.getThresholdToAlert(daysLeft, thresholds, lastAlerted); + if (threshold == null) { + return; + } + String subject = daysLeft < 0 ? + String.format("A %s of Kubernetes cluster %s has expired", what, kubernetesCluster.getName()) : + String.format("A %s of Kubernetes cluster %s expires in %d days", what, kubernetesCluster.getName(), daysLeft); + String body = String.format("%s: %s expires on %s. %s", subject, expiry.getName(), expiry.getExpires(), advice); + worker.sendAlert(subject, body); + worker.recordEvent(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY, EventVO.LEVEL_WARN, body); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), alertDetail, + KubernetesClusterCertificateUtil.buildAlertState(expiry.getExpires(), threshold), false); + } + // checks if Kubernetes cluster is in desired state private boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); @@ -3003,10 +3359,29 @@ public boolean start() { _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); + // Separate from the status scanner: SSH checks of many clusters take a while and must not hold up its 30s passes + _certificateScanner.scheduleWithFixedDelay(new KubernetesClusterCertificateScanner(), 600, CERTIFICATE_SCANNER_INTERVAL_SECONDS, TimeUnit.SECONDS); return true; } + @Override + public boolean stop() { + // running rotations are left to finish; one cut short is recovered by the scanner after a restart + if (_certificateScanner != null) { + _certificateScanner.shutdown(); + } + if (_certificateRotationExecutor != null) { + // renewals still waiting for a thread must not start now; they are scheduled again after the restart + if (_certificateRotationExecutor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) _certificateRotationExecutor).getQueue().clear(); + } + queuedCertificateRotations.clear(); + _certificateRotationExecutor.shutdown(); + } + return super.stop(); + } + private void createNetworkOfferingForKubernetes(String offeringName, String offeringDesc, Network.Provider extNetProvider, boolean forVpc) { final Map defaultKubernetesServiceNetworkOfferingProviders = new HashMap(); Network.Provider provider = forVpc ? Network.Provider.VPCVirtualRouter : Network.Provider.VirtualRouter; @@ -3063,6 +3438,8 @@ public boolean configure(String name, Map params) throws Configu _configParams = params; _gcExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Scavenger")); _stateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-State-Scanner")); + _certificateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Certificate-Scanner")); + _certificateRotationExecutor = Executors.newFixedThreadPool(CERTIFICATE_ROTATION_THREADS, new NamedThreadFactory("Kubernetes-Cluster-Certificate-Rotation")); return true; } @@ -3089,7 +3466,12 @@ public ConfigKey[] getConfigKeys() { KubernetesControlNodeInstallReattempts, KubernetesWorkerNodeInstallAttemptWait, KubernetesWorkerNodeInstallReattempts, - KubernetesEtcdNodeStartPort + KubernetesEtcdNodeStartPort, + KubernetesClusterCertificateCheckInterval, + KubernetesClusterCertificateExpiryAlertDays, + KubernetesClusterCertificateAutoRenewEnabled, + KubernetesClusterCertificateAutoRenewDays, + KubernetesClusterCertificateRotationTimeout }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 1d19127dc54a..9953ae194fa0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; @@ -143,6 +144,43 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Start port for Port forwarding rules for etcd nodes", true, KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateCheckInterval = new ConfigKey<>("Advanced", Integer.class, + "cloud.kubernetes.cluster.certificate.check.interval", + "6", + "Interval (in hours) at which the control plane certificate expiry of each running CloudManaged Kubernetes cluster " + + "is checked with 'kubeadm certs check-expiration'. 0 disables the check and the automatic renewal", + true, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateExpiryAlertDays = new ConfigKey<>("Advanced", String.class, + "cloud.kubernetes.cluster.certificate.expiry.alert.days", + "60,30", + "Comma-separated list of days before a Kubernetes cluster control plane certificate (or certificate authority) " + + "expires at which an alert and an event are raised. Each threshold alerts once per expiry date", + true, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateAutoRenewEnabled = new ConfigKey<>("Advanced", Boolean.class, + "cloud.kubernetes.cluster.certificate.auto.renew.enabled", + "true", + "Indicates whether the control plane certificates of a CloudManaged Kubernetes cluster are renewed automatically, " + + "one control node at a time, when they are about to expire. Requires Kubernetes 1.20 or later", + true, + ConfigKey.Scope.Account, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateAutoRenewDays = new ConfigKey<>("Advanced", Integer.class, + "cloud.kubernetes.cluster.certificate.auto.renew.days", + "30", + "Number of days before the first control plane certificate of a Kubernetes cluster expires at which the " + + "certificates are renewed automatically, when cloud.kubernetes.cluster.certificate.auto.renew.enabled is true. " + + "Values above 180 are treated as 180", + true, + ConfigKey.Scope.Account, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateRotationTimeout = new ConfigKey<>("Advanced", Long.class, + "cloud.kubernetes.cluster.certificate.rotation.timeout", + "3600", + "Timeout interval (in seconds) in which the certificate rotation of a Kubernetes cluster should be completed", + true, + KubernetesServiceEnabled.key()); KubernetesCluster findById(final Long id); @@ -172,6 +210,8 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException; + boolean rotateKubernetesClusterCertificates(RotateKubernetesClusterCertificatesCmd cmd) throws CloudRuntimeException; + boolean updateKubernetesClusterAffinityGroups(UpdateKubernetesClusterAffinityGroupCmd cmd) throws CloudRuntimeException; boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..6d521fc05d1e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -158,6 +158,8 @@ public class KubernetesClusterActionWorker { public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks"; + public static final String KUBE_CONFIG_DATA_DETAIL = "kubeConfigData"; + protected static final long KUBE_CONFIG_REFRESH_TIMEOUT_MS = 2 * 60 * 1000L; protected Logger logger = LogManager.getLogger(getClass()); @@ -756,6 +758,83 @@ protected List getKubernetesClusterVMs() { return vmList; } + /** + * @return the control node VMs of the cluster, ordered by VM ID, which is the order they were created and + * received their SSH port forwarding rules in + */ + protected List getControlNodeVMs() { + List controlVms = new ArrayList<>(); + List vmMaps = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), CONTROL); + List controlVmIds; + if (CollectionUtils.isNotEmpty(vmMaps)) { + controlVmIds = vmMaps.stream().map(KubernetesClusterVmMapVO::getVmId).sorted().collect(Collectors.toList()); + } else { + // clusters created before the control node flag was recorded: by position, after any etcd nodes + List allVmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(allVmMaps)) { + return controlVms; + } + long etcdNodes = Objects.requireNonNullElse(kubernetesCluster.getEtcdNodeCount(), 0L); + controlVmIds = allVmMaps.stream().map(KubernetesClusterVmMapVO::getVmId).sorted() + .skip(etcdNodes).limit(Math.max(1, kubernetesCluster.getControlNodeCount())).collect(Collectors.toList()); + } + for (Long vmId : controlVmIds) { + UserVm vm = userVmDao.findById(vmId); + if (vm != null) { + controlVms.add(vm); + } + } + return controlVms; + } + + /** + * Returns the address and port to SSH to one control node. {@link #publicIpAddress} and {@link #sshPort} must be + * set. On directly reachable (shared or routed) networks it is the VM's own address. Otherwise it is the cluster's + * public IP with the port of the SSH port forwarding rule of the VM, falling back to the port assigned by position + * (control nodes are the first nodes behind {@link #CLUSTER_NODES_DEFAULT_START_SSH_PORT}) when there is no rule, + * e.g. behind an external load balancer. + */ + protected Pair getControlNodeSshEndpoint(UserVm vm, int controlNodeIndex) { + if (sshPort == DEFAULT_SSH_PORT) { + return new Pair<>(vm.getPrivateIpAddress(), DEFAULT_SSH_PORT); + } + List rules = portForwardingRulesDao.listByVm(vm.getId()); + if (CollectionUtils.isNotEmpty(rules)) { + for (PortForwardingRuleVO rule : rules) { + if (rule.getDestinationPortStart() != DEFAULT_SSH_PORT || rule.getState() == FirewallRule.State.Revoke) { + continue; + } + // only the cluster's own rule, not one a user added on another public IP + IPAddressVO ruleIp = ipAddressDao.findById(rule.getSourceIpAddressId()); + if (ruleIp != null && publicIpAddress.equals(ruleIp.getAddress().addr())) { + return new Pair<>(publicIpAddress, rule.getSourcePortStart()); + } + } + } + return new Pair<>(publicIpAddress, sshPort + controlNodeIndex); + } + + /** + * Reads the kubeconfig from the control node reachable at {@link #publicIpAddress}:{@link #sshPort}, points it at + * {@link #apiPublicIpAddress} and stores it as the cluster's kubeConfigData, which getKubernetesClusterConfig + * returns. Called after every operation that can issue a new admin certificate (start, upgrade, certificate + * rotation), as the stored copy otherwise keeps the old certificate until it expires. + * @return true if a kubeconfig was read and stored + */ + protected boolean refreshKubeConfigData(final long timeoutTime) { + String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), timeoutTime); + if (StringUtils.isEmpty(kubeConfig)) { + return false; + } + final String controlVMPrivateIpAddress = getControlVmPrivateIp(); + if (StringUtils.isNotEmpty(controlVMPrivateIpAddress) && StringUtils.isNotEmpty(apiPublicIpAddress)) { + kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", controlVMPrivateIpAddress, CLUSTER_API_PORT), + String.format("server: https://%s:%d", apiPublicIpAddress, CLUSTER_API_PORT)); + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL, Base64.encodeBase64String(kubeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())), false); + return true; + } + protected void updateLoginUserDetails(List clusterVMs) { if (clusterVMs == null) { clusterVMs = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java new file mode 100644 index 000000000000..e549922e72e1 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java @@ -0,0 +1,476 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.File; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import javax.inject.Inject; + +import org.apache.cloudstack.alert.AlertService; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; + +import com.cloud.alert.AlertManager; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventVO; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; +import com.cloud.user.User; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; + +/** + * Checks the expiry of, and renews, the kubeadm-managed control plane certificates of a CKS cluster. + *

+ * Renewal runs {@code kubeadm certs renew all} on one control node at a time, restarts that node's control plane + * static pods so they load the new certificates and waits for its API server and the node to be ready before moving + * to the next one. It stops at the first control node that fails. The certificate authorities are not rotated. + *

+ * External etcd nodes created by CKS serve etcd over plain HTTP and hold no certificates, so there is nothing to + * renew on them; only control nodes are touched. + */ +public class KubernetesClusterCertificateWorker extends KubernetesClusterActionWorker { + + public static final String ROTATION_SCRIPT_FILENAME = "rotate-kubernetes-certificates.sh"; + + /** Epoch millis at which the expiry was last checked. */ + public static final String CERTIFICATE_CHECK_DETAIL = "certificateExpiryLastCheck"; + /** Epoch millis of the last alert that no control node could be checked; removed once a check works again. */ + public static final String CERTIFICATE_CHECK_FAILED_ALERT_DETAIL = "certificateExpiryCheckFailedAlert"; + /** Alert state of the first leaf certificate to expire, see {@link KubernetesClusterCertificateUtil#buildAlertState}. */ + public static final String CERTIFICATE_ALERT_DETAIL = "certificateExpiryAlert"; + /** Alert state of the first certificate authority to expire. */ + public static final String CA_CERTIFICATE_ALERT_DETAIL = "caCertificateExpiryAlert"; + /** Epoch millis at which the running rotation started; removed when it ends. */ + public static final String ROTATION_STARTED_DETAIL = "certificateRotationStarted"; + /** ID of the management server running the rotation; removed when it ends. */ + public static final String ROTATION_OWNER_DETAIL = "certificateRotationOwner"; + /** Epoch millis of the last failed rotation, used to back off automatic renewal. */ + public static final String ROTATION_FAILED_DETAIL = "certificateRotationLastFailure"; + /** Epoch millis at which the scanner first saw a rotation that had not recorded its start yet. */ + public static final String ROTATION_SEEN_DETAIL = "certificateRotationFirstSeen"; + /** Epoch millis of the last successful rotation, so a queued automatic renewal doesn't renew again. */ + public static final String ROTATION_SUCCEEDED_DETAIL = "certificateRotationLastSuccess"; + + protected static final int NODE_READY_POLL_INTERVAL_MS = 5000; + private static final Set CLIENT_ONLY_KUBECONFIGS = Set.of("admin.conf", "super-admin.conf", + KubernetesClusterCertificateUtil.ROOT_KUBECONFIG_ENTRY_NAME); + private static final String CHECK_OUTPUT_SEPARATOR = "----- cks root kubeconfig -----"; + private static final String USER_KUBECONFIG_SEPARATOR = "----- cks user kubeconfig -----"; + + /** Clusters whose rotation runs in this management server. */ + private static final Set ACTIVE_ROTATIONS = ConcurrentHashMap.newKeySet(); + + @Inject + protected AlertManager alertManager; + + private File rotationScriptFile; + private long rotationTimeoutTime; + private boolean automatic = false; + private boolean rescueMode = false; + private String rotationStartedAt; + private boolean startedFromAlert = false; + + public KubernetesClusterCertificateWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + /** Marks a rotation started by the expiry check rather than by the API, which records its own event. */ + public void setAutomatic(boolean automatic) { + this.automatic = automatic; + } + + /** @return true if this management server is running a rotation of the cluster right now */ + public static boolean isRotationActive(long kubernetesClusterId) { + return ACTIVE_ROTATIONS.contains(kubernetesClusterId); + } + + /** + * Only the SSH key is needed here; the templates and owner that the other workers load in their init() are not, + * and loading them fails for a cluster whose template has since been removed. + */ + @Override + protected void init() { + this.sshKeyFile = getManagementServerSshPublicKeyFile(); + } + + public static boolean isCertificateManagementSupported(KubernetesSupportedVersion version) { + return version != null && KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), + KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT) >= 0; + } + + protected void resolveClusterEndpoints() { + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + apiPublicIpAddress = resolveApiPublicIpAddress(networkDao.findById(kubernetesCluster.getNetworkId())); + if (StringUtils.isEmpty(publicIpAddress)) { + throw new CloudRuntimeException(String.format("Unable to retrieve the SSH address of Kubernetes cluster %s", kubernetesCluster.getName())); + } + } + + protected Pair runOnNode(String address, int port, String command, int timeoutMs) throws Exception { + return SshHelper.sshExecute(address, port, getControlNodeLoginUser(), sshKeyFile, null, command, 10000, 10000, timeoutMs); + } + + ///////////////////////////////////////////////////// + //////////////// Expiry check /////////////////////// + ///////////////////////////////////////////////////// + + /** + * Runs {@code kubeadm certs check-expiration} on every control node and reads the client certificate of root's + * kubeconfig, which CKS itself uses for kubectl. Nodes that can't be checked are listed in the report. + * Also refreshes the stored kubeConfigData when its admin certificate expires before the one on the cluster, + * e.g. after the certificates were renewed by hand. + */ + public CertificateExpiryReport checkCertificateExpiry() { + init(); + resolveClusterEndpoints(); + CertificateExpiryReport report = new CertificateExpiryReport(); + List controlVms = getControlNodeVMs(); + Date firstNodeAdminExpiry = null; + boolean firstNodeUnreachable = false; + boolean userKubeConfig = false; + Date userKubeConfigExpiry = null; + for (int i = 0; i < controlVms.size(); i++) { + UserVm vm = controlVms.get(i); + Pair endpoint = getControlNodeSshEndpoint(vm, i); + try { + // one SSH session per node: kubeadm's exit status, then the certificate lines of root's kubeconfig and of user.conf + String command = String.format("%s; rc=$?; echo '%s'; %s; echo '%s'; %s; exit $rc", KubernetesClusterCertificateUtil.CHECK_EXPIRATION_COMMAND, + CHECK_OUTPUT_SEPARATOR, KubernetesClusterCertificateUtil.ROOT_KUBECONFIG_CERTIFICATE_COMMAND, + USER_KUBECONFIG_SEPARATOR, KubernetesClusterCertificateUtil.USER_KUBECONFIG_CERTIFICATE_COMMAND); + Pair result = runOnNode(endpoint.first(), endpoint.second(), command, 60000); + String output = StringUtils.defaultString(result.second()); + String kubeadmOutput = StringUtils.substringBefore(output, CHECK_OUTPUT_SEPARATOR); + List entries = KubernetesClusterCertificateUtil.parseCheckExpirationOutput(kubeadmOutput); + if (!result.first() || entries.isEmpty()) { + logger.warn("Unable to check the certificate expiry on control node {} of Kubernetes cluster {}: {}", vm, kubernetesCluster, kubeadmOutput); + report.addUncheckedNode(vm.getHostName()); + if (i == 0) { + firstNodeUnreachable = true; + } + continue; + } + String kubeConfigOutput = StringUtils.substringAfter(output, CHECK_OUTPUT_SEPARATOR); + String userKubeConfigOutput = StringUtils.substringAfter(kubeConfigOutput, USER_KUBECONFIG_SEPARATOR); + if (i == 0 && userKubeConfigOutput.contains(KubernetesClusterCertificateUtil.USER_KUBECONFIG_PRESENT)) { + userKubeConfig = true; + userKubeConfigExpiry = KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(userKubeConfigOutput); + } + Date rootKubeConfigExpiry = KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry( + StringUtils.substringBefore(kubeConfigOutput, USER_KUBECONFIG_SEPARATOR)); + if (rootKubeConfigExpiry != null) { + // a copy of admin.conf: renewable only when admin.conf is (not in external CA mode) + boolean adminConfExternallyManaged = entries.stream().anyMatch(entry -> "admin.conf".equals(entry.getName()) && entry.isExternallyManaged()); + entries.add(new CertificateExpiry(KubernetesClusterCertificateUtil.ROOT_KUBECONFIG_ENTRY_NAME, rootKubeConfigExpiry, false, + adminConfExternallyManaged)); + } + // the kubeconfig is read from the first control node, so that is the one to compare with + if (i == 0) { + for (CertificateExpiry entry : entries) { + if ("admin.conf".equals(entry.getName()) && !entry.isMissing()) { + firstNodeAdminExpiry = entry.getExpires(); + } + } + } + report.add(vm.getHostName(), entries); + } catch (Exception e) { + logger.warn("Unable to check the certificate expiry on control node {} of Kubernetes cluster {}", vm, kubernetesCluster, e); + report.addUncheckedNode(vm.getHostName()); + if (i == 0) { + firstNodeUnreachable = true; + } + } + } + // the kubeconfig is read from the first control node; don't spend the refresh timeout on it when it is down + // and from user.conf rather than admin.conf when the operator put one there + if (!firstNodeUnreachable) { + refreshStaleKubeConfigData(userKubeConfig ? userKubeConfigExpiry : firstNodeAdminExpiry); + } + return report; + } + + protected Date getStoredKubeConfigExpiry() { + KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL); + if (detail == null || StringUtils.isEmpty(detail.getValue())) { + return null; + } + return KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(new String(Base64.decodeBase64(detail.getValue()), + com.cloud.utils.StringUtils.getPreferredCharset())); + } + + protected void refreshStaleKubeConfigData(Date clusterKubeConfigExpiry) { + if (clusterKubeConfigExpiry == null) { + return; + } + Date storedExpiry = getStoredKubeConfigExpiry(); + if (storedExpiry == null || !storedExpiry.before(clusterKubeConfigExpiry)) { + return; + } + logger.info("The stored kubeconfig of Kubernetes cluster {} expires on {}, before the kubeconfig on the cluster ({}), refreshing it", + kubernetesCluster, storedExpiry, clusterKubeConfigExpiry); + if (!refreshKubeConfigData(System.currentTimeMillis() + KUBE_CONFIG_REFRESH_TIMEOUT_MS)) { + logger.warn("Unable to refresh the stored kubeconfig of Kubernetes cluster {}", kubernetesCluster); + } + } + + ///////////////////////////////////////////////////// + //////////////// Rotation /////////////////////////// + ///////////////////////////////////////////////////// + + /** + * Renews the control plane certificates, one control node at a time, then refreshes the stored kubeconfig. + * The cluster must be Running, or Alert to retry a failed rotation; it is RotatingCertificates meanwhile, which keeps every other cluster operation out. + * On failure the cluster moves to Alert, an alert is raised and a CloudRuntimeException is thrown. + */ + public boolean rotateCertificates() throws CloudRuntimeException { + init(); + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (!isCertificateManagementSupported(version)) { + throw new CloudRuntimeException(String.format("Certificate rotation of Kubernetes cluster %s needs Kubernetes %s or later", + kubernetesCluster.getName(), KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT)); + } + // Marked active before the transition, so the scanner never takes it for an interrupted rotation + if (!ACTIVE_ROTATIONS.add(kubernetesCluster.getId())) { + throw new CloudRuntimeException(String.format("The certificates of Kubernetes cluster %s are already being rotated", kubernetesCluster.getName())); + } + try { + KubernetesCluster before = kubernetesClusterDao.findById(kubernetesCluster.getId()); + startedFromAlert = before != null && KubernetesCluster.State.Alert.equals(before.getState()); + // The transition only succeeds from Running or Alert, so it also keeps out any other operation already in progress + if (!stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CertificateRotationRequested)) { + KubernetesCluster current = kubernetesClusterDao.findById(kubernetesCluster.getId()); + throw new CloudRuntimeException(String.format("Unable to rotate the certificates of Kubernetes cluster %s, it is in state %s and must be Running or Alert", + kubernetesCluster.getName(), current == null ? null : current.getState())); + } + return doRotateCertificates(); + } finally { + ACTIVE_ROTATIONS.remove(kubernetesCluster.getId()); + } + } + + private boolean doRotateCertificates() { + rotationStartedAt = String.valueOf(System.currentTimeMillis()); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_STARTED_DETAIL, rotationStartedAt, false); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_OWNER_DETAIL, String.valueOf(ManagementServerNode.getManagementServerId()), false); + logger.info("Rotating the control plane certificates of Kubernetes cluster {}", kubernetesCluster); + rotationTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterCertificateRotationTimeout.value() * 1000; + try { + resolveClusterEndpoints(); + List controlVms = getControlNodeVMs(); + if (CollectionUtils.isEmpty(controlVms)) { + failRotation(String.format("Unable to find the control nodes of Kubernetes cluster %s", kubernetesCluster.getName()), null); + } + rotationScriptFile = retrieveScriptFile(ROTATION_SCRIPT_FILENAME); + // With expired certificates, stacked etcd on a renewed node can't get quorum until the others are renewed + // too, so renew them all before waiting for the cluster. Only then: it restarts every control node at once. + rescueMode = areCertificatesExpired(controlVms); + if (rescueMode) { + logger.warn("Certificates of Kubernetes cluster {} have expired, renewing the certificates on every control node before waiting for it", + kubernetesCluster); + } + for (int i = 0; i < controlVms.size(); i++) { + UserVm vm = controlVms.get(i); + if (System.currentTimeMillis() > rotationTimeoutTime) { + failRotation(String.format("Certificate rotation of Kubernetes cluster %s timed out before control node %s", + kubernetesCluster.getName(), vm.getHostName()), null); + } + if (!renewCertificatesOnControlNode(vm, i)) { + failRotation(String.format("Failed to renew the certificates on control node %s of Kubernetes cluster %s, " + + "the remaining control nodes were left untouched", vm.getHostName(), kubernetesCluster.getName()), null); + } + if (!rescueMode && !waitForControlNodeReady(vm)) { + failRotation(String.format("Control node %s of Kubernetes cluster %s did not become ready after its certificates were renewed, " + + "the remaining control nodes were left untouched", vm.getHostName(), kubernetesCluster.getName()), null); + } + logger.info("Renewed the certificates on control node {} of Kubernetes cluster {}", vm, kubernetesCluster); + } + if (rescueMode) { + for (UserVm vm : controlVms) { + if (!waitForControlNodeReady(vm)) { + failRotation(String.format("Renewed the certificates on every control node of Kubernetes cluster %s, but control node %s did not become ready", + kubernetesCluster.getName(), vm.getHostName()), null); + } + } + } + // a budget of its own: the certificates are renewed by now, even if that used up the rotation timeout + if (!refreshKubeConfigData(System.currentTimeMillis() + KUBE_CONFIG_REFRESH_TIMEOUT_MS)) { + failRotation(String.format("Renewed the certificates of Kubernetes cluster %s but failed to refresh its stored kubeconfig", + kubernetesCluster.getName()), null); + } + } catch (RotationFailedException e) { + throw e; + } catch (Exception e) { + failRotation(String.format("Failed to rotate the certificates of Kubernetes cluster %s: %s", kubernetesCluster.getName(), e.getMessage()), e); + } finally { + if (rotationScriptFile != null) { + rotationScriptFile.delete(); + } + } + if (isStillOwnRotation()) { + // transition first, so the certificate scanner never sees RotatingCertificates without its details. + // A cluster rotated from Alert goes back there: whatever else put it in Alert hasn't been checked, and + // the status scanner moves it to Running once its nodes are ready + stateTransitTo(kubernetesCluster.getId(), startedFromAlert ? KubernetesCluster.Event.OperationFailed : KubernetesCluster.Event.OperationSucceeded); + clearRotationDetails(); + } else { + // e.g. taken for an interrupted rotation meanwhile, maybe retried since: leave the state and details alone + logger.warn("Renewed the certificates of Kubernetes cluster {}, but the cluster was taken out of this rotation meanwhile; leaving its state", + kubernetesCluster); + } + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_FAILED_DETAIL); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_SUCCEEDED_DETAIL, String.valueOf(System.currentTimeMillis()), false); + if (automatic) { + recordEvent(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE, EventVO.LEVEL_INFO, + String.format("Automatically renewed the control plane certificates of Kubernetes cluster %s. Kubeconfigs downloaded " + + "before keep working only until their old certificate expires; download it again", kubernetesCluster.getName())); + } + logger.info("Rotated the control plane certificates of Kubernetes cluster {}", kubernetesCluster); + return true; + } + + protected boolean renewCertificatesOnControlNode(UserVm vm, int controlNodeIndex) { + Pair endpoint = getControlNodeSshEndpoint(vm, controlNodeIndex); + long remainingSeconds = Math.max(60, (rotationTimeoutTime - System.currentTimeMillis()) / 1000); + try { + SshHelper.scpTo(endpoint.first(), endpoint.second(), getControlNodeLoginUser(), sshKeyFile, null, + "~/", rotationScriptFile.getAbsolutePath(), "0755"); + String script = rotationScriptFile.getName(); + String command = String.format("sudo ./%s %d%s; rc=$?; rm -f ./%s; exit $rc", script, remainingSeconds, rescueMode ? " rescue" : "", script); + int timeoutMs = (int) Math.min(Integer.MAX_VALUE, (remainingSeconds + 60) * 1000L); + Pair result = runOnNode(endpoint.first(), endpoint.second(), command, timeoutMs); + if (!result.first()) { + logger.error("Certificate renewal failed on control node {} of Kubernetes cluster {}: {}", vm, kubernetesCluster, result.second()); + } + return result.first(); + } catch (Exception e) { + logger.error("Certificate renewal failed on control node {} of Kubernetes cluster {}", vm, kubernetesCluster, e); + return false; + } + } + + /** + * Whether a renewable leaf certificate on any control node has expired, per kubeadm. Every node counts: after an + * interrupted rescue the first nodes are renewed while the others still hold etcd hostage. Nodes that can't be + * checked are left out. + */ + protected boolean areCertificatesExpired(List controlVms) { + CertificateExpiryReport report = new CertificateExpiryReport(); + for (int i = 0; i < controlVms.size(); i++) { + UserVm vm = controlVms.get(i); + Pair endpoint = getControlNodeSshEndpoint(vm, i); + try { + Pair result = runOnNode(endpoint.first(), endpoint.second(), KubernetesClusterCertificateUtil.CHECK_EXPIRATION_COMMAND, 60000); + report.add(vm.getHostName(), KubernetesClusterCertificateUtil.parseCheckExpirationOutput(StringUtils.defaultString(result.second()))); + } catch (Exception e) { + logger.warn("Unable to check whether the certificates on control node {} of Kubernetes cluster {} have expired", vm, kubernetesCluster, e); + } + } + // Only what the control plane itself runs on: an expired admin.conf or super-admin.conf breaks no node + Date now = new Date(); + return report.getEntries().stream().anyMatch(entry -> !entry.isAuthority() && !entry.isMissing() && !entry.isExternallyManaged() + && !CLIENT_ONLY_KUBECONFIGS.contains(StringUtils.substringBefore(entry.getName(), " on ")) && entry.getExpires().before(now)); + } + + /** Whether the cluster is still in the rotation this worker started, not moved on by recovery or a later rotation */ + protected boolean isStillOwnRotation() { + KubernetesCluster current = kubernetesClusterDao.findById(kubernetesCluster.getId()); + KubernetesClusterDetailsVO started = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ROTATION_STARTED_DETAIL); + return current != null && KubernetesCluster.State.RotatingCertificates.equals(current.getState()) + && started != null && StringUtils.equals(rotationStartedAt, started.getValue()); + } + + /** + * The rotation script already waited for this node's API server, controller manager and scheduler; this checks + * the cluster API endpoint answers and the node reports Ready. + */ + protected boolean waitForControlNodeReady(UserVm vm) { + if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, apiPublicIpAddress, CLUSTER_API_PORT, + rotationTimeoutTime, NODE_READY_POLL_INTERVAL_MS)) { + return false; + } + String hostName = StringUtils.lowerCase(vm.getHostName()); + return KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), + sshKeyFile, hostName, rotationTimeoutTime, NODE_READY_POLL_INTERVAL_MS); + } + + private void failRotation(String message, Exception e) { + logMessage(Level.ERROR, message, e); + if (isStillOwnRotation()) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + clearRotationDetails(); + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_FAILED_DETAIL, String.valueOf(System.currentTimeMillis()), false); + sendAlert(String.format("Certificate rotation failed for Kubernetes cluster %s", kubernetesCluster.getName()), message); + // the API records the failure of its own async job event + if (automatic) { + recordEvent(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE, EventVO.LEVEL_ERROR, message); + } + throw new RotationFailedException(message, e); + } + + private void clearRotationDetails() { + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_STARTED_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_OWNER_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_SEEN_DETAIL); + } + + public void sendAlert(String subject, String body) { + try { + alertManager.sendAlert(AlertService.AlertType.ALERT_TYPE_KUBERNETES_CLUSTER_CERTIFICATE, kubernetesCluster.getZoneId(), null, subject, body); + } catch (Exception e) { + logger.warn("Unable to send the alert \"{}\" for Kubernetes cluster {}", subject, kubernetesCluster, e); + } + } + + public void recordEvent(String eventType, String level, String description) { + try { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, kubernetesCluster.getAccountId(), level, + eventType, true, description, + kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + } catch (Exception e) { + logger.warn("Unable to record the event \"{}\" for Kubernetes cluster {}", description, kubernetesCluster, e); + } + } + + private static class RotationFailedException extends CloudRuntimeException { + RotationFailedException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a15086c60d6c..8f7fe4bd4c87 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -730,22 +730,12 @@ private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromCluster private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { if (StringUtils.isEmpty(publicIpAddress)) { - KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL); if (kubeConfigDetail != null && StringUtils.isNotEmpty(kubeConfigDetail.getValue())) { return true; } } - String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, timeoutTime); - if (StringUtils.isNotEmpty(kubeConfig)) { - final String controlVMPrivateIpAddress = getControlVmPrivateIp(); - if (StringUtils.isNotEmpty(controlVMPrivateIpAddress)) { - kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", controlVMPrivateIpAddress, CLUSTER_API_PORT), - String.format("server: https://%s:%d", apiPublicIpAddress, CLUSTER_API_PORT)); - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())), false); - return true; - } - return false; + return refreshKubeConfigData(timeoutTime); } private boolean isKubernetesClusterDashboardServiceRunning(final boolean onCreate, final Long timeoutTime) { @@ -978,8 +968,10 @@ public boolean reconcileAlertCluster() { if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { return false; } - // mark the cluster to be running - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); + // mark the cluster to be running, unless another operation (e.g. a certificate rotation) took it meanwhile + if (!stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested)) { + return false; + } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 4c2725fc2a2f..aa753303bda3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -79,7 +79,7 @@ private Pair runInstallScriptOnVM(final UserVm vm, final int in 10000, 10000, 10 * 60 * 1000); } - private void upgradeKubernetesClusterNodes() { + protected void upgradeKubernetesClusterNodes() { for (int i = 0; i < clusterVMs.size(); ++i) { UserVm vm = clusterVMs.get(i); String hostName = vm.getHostName(); @@ -184,11 +184,28 @@ public boolean upgradeCluster() throws CloudRuntimeException { if (!updated) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } else { + refreshKubeConfigDataAfterUpgrade(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } return updated; } + /** + * kubeadm upgrade renews the control plane certificates, admin.conf included, so the stored kubeConfigData + * has to be read again or getKubernetesClusterConfig keeps handing out the old admin certificate until it expires. + * A failure here doesn't fail the upgrade: the certificate expiry check refreshes a stale copy later on. + */ + protected void refreshKubeConfigDataAfterUpgrade() { + try { + apiPublicIpAddress = resolveApiPublicIpAddress(networkDao.findById(kubernetesCluster.getNetworkId())); + if (!refreshKubeConfigData(System.currentTimeMillis() + KUBE_CONFIG_REFRESH_TIMEOUT_MS)) { + logger.warn("Unable to refresh the stored kubeconfig of Kubernetes cluster {} after the upgrade", kubernetesCluster); + } + } catch (Exception e) { + logger.warn("Unable to refresh the stored kubeconfig of Kubernetes cluster {} after the upgrade", kubernetesCluster, e); + } + } + protected void filterOutManualUpgradeNodesFromClusterUpgrade() { if (CollectionUtils.isEmpty(clusterVMs)) { return; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java new file mode 100644 index 000000000000..5d466935321c --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.utils; + +import java.io.ByteArrayInputStream; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Helpers for the control plane certificate expiry check and renewal of CKS clusters. + * Everything here is free of side effects so it can be unit tested. + */ +public class KubernetesClusterCertificateUtil { + + protected static Logger LOGGER = LogManager.getLogger(KubernetesClusterCertificateUtil.class); + + /** {@code kubeadm certs} replaced {@code kubeadm alpha certs} in this version. */ + public static final String MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT = "1.20.0"; + + public static final String CHECK_EXPIRATION_COMMAND = "sudo /opt/bin/kubeadm certs check-expiration"; + + /** Prints only the client certificate line of root's kubeconfig, which CKS itself uses for kubectl. Never the key. */ + public static final String ROOT_KUBECONFIG_CERTIFICATE_COMMAND = "sudo grep -m1 'client-certificate-data:' /root/.kube/config"; + + public static final String ROOT_KUBECONFIG_ENTRY_NAME = "/root/.kube/config"; + /** + * getKubernetesClusterConfig hands out an operator-supplied user.conf instead of admin.conf when there is one, so + * the stored kubeconfig has to be compared with that. Prints a marker when it exists, then its certificate line. + */ + public static final String USER_KUBECONFIG_PRESENT = "user.conf present"; + public static final String USER_KUBECONFIG_CERTIFICATE_COMMAND = "if sudo test -f /etc/kubernetes/user.conf; then echo '" + USER_KUBECONFIG_PRESENT + + "'; sudo grep -m1 'client-certificate-data:' /etc/kubernetes/user.conf; fi"; + + /** + * kubeadm issues certificates for one year; renewing more than this many days ahead would leave them inside the + * renewal window right after a renewal and renew on every check. + */ + public static final int MAX_AUTO_RENEW_DAYS = 180; + + /** + * Expiries this close are treated as the same one for alerting: control nodes renewed in one rotation differ by + * minutes, and which node reports the first expiry changes when one can't be checked. A renewal moves it by months. + */ + private static final long SAME_EXPIRY_TOLERANCE_MS = TimeUnit.DAYS.toMillis(7); + + private static final String MISSING_MARKER = "!MISSING!"; + + // kubeadm formats the dates with Go's "Jan 02, 2006 15:04 MST", always in UTC + private static final Pattern CHECK_EXPIRATION_LINE = Pattern.compile( + "^(\\S+)\\s+([A-Z][a-z]{2} \\d{1,2}, \\d{4} \\d{2}:\\d{2} [A-Z]+)\\s+.*$"); + private static final DateTimeFormatter CHECK_EXPIRATION_DATE_FORMAT = + DateTimeFormatter.ofPattern("MMM d, yyyy HH:mm z", Locale.ENGLISH); + private static final Pattern CLIENT_CERTIFICATE_DATA = Pattern.compile("client-certificate-data:\\s*([A-Za-z0-9+/=]+)"); + + /** One line of {@code kubeadm certs check-expiration}. */ + public static class CertificateExpiry { + private final String name; + private final Date expires; + private final boolean authority; + private final boolean externallyManaged; + + public CertificateExpiry(String name, Date expires, boolean authority, boolean externallyManaged) { + this.name = name; + this.expires = expires; + this.authority = authority; + this.externallyManaged = externallyManaged; + } + + public String getName() { + return name; + } + + /** @return the expiry date, or null when kubeadm reported the certificate as missing */ + public Date getExpires() { + return expires; + } + + public boolean isAuthority() { + return authority; + } + + public boolean isMissing() { + return expires == null; + } + + /** kubeadm cannot renew a certificate it reports as externally managed */ + public boolean isExternallyManaged() { + return externallyManaged; + } + + @Override + public String toString() { + return String.format("%s%s%s: %s", name, authority ? " (CA)" : "", externallyManaged ? " (externally managed)" : "", + isMissing() ? "missing" : expires); + } + } + + /** + * The certificates of every control node of one cluster, as far as they could be read. + */ + public static class CertificateExpiryReport { + private final List entries = new ArrayList<>(); + private final List uncheckedNodes = new ArrayList<>(); + + public void add(String node, List nodeEntries) { + for (CertificateExpiry entry : nodeEntries) { + entries.add(new CertificateExpiry(String.format("%s on %s", entry.getName(), node), entry.getExpires(), entry.isAuthority(), + entry.isExternallyManaged())); + } + } + + public void addUncheckedNode(String node) { + uncheckedNodes.add(node); + } + + public List getEntries() { + return entries; + } + + public List getUncheckedNodes() { + return uncheckedNodes; + } + + public boolean isEmpty() { + return entries.isEmpty(); + } + + /** @return the leaf certificate (not a CA) that expires first, or null */ + public CertificateExpiry getFirstLeafExpiry() { + return getFirstExpiry(false, true); + } + + /** @return the leaf certificate that kubeadm can renew which expires first, or null */ + public CertificateExpiry getFirstRenewableLeafExpiry() { + return getFirstExpiry(false, false); + } + + /** @return the certificate authority that expires first, or null */ + public CertificateExpiry getFirstAuthorityExpiry() { + return getFirstExpiry(true, true); + } + + private CertificateExpiry getFirstExpiry(boolean authority, boolean includeExternallyManaged) { + CertificateExpiry first = null; + for (CertificateExpiry entry : entries) { + if (entry.isAuthority() != authority || entry.isMissing() || (!includeExternallyManaged && entry.isExternallyManaged())) { + continue; + } + if (first == null || entry.getExpires().before(first.getExpires())) { + first = entry; + } + } + return first; + } + } + + /** + * Parses the output of {@code kubeadm certs check-expiration}. It has a leaf certificate table followed by a + * certificate authority table; the line starting with {@code CERTIFICATE AUTHORITY} heads the second one. + * Certificates kubeadm cannot find are printed as {@code !MISSING! } and returned with a null expiry. + * The last column of both tables tells whether the certificate is externally managed. + */ + public static List parseCheckExpirationOutput(String output) { + List entries = new ArrayList<>(); + if (StringUtils.isBlank(output)) { + return entries; + } + boolean authoritySection = false; + for (String rawLine : output.split("\\r?\\n")) { + String line = rawLine.trim(); + if (line.isEmpty() || line.startsWith("[")) { + continue; + } + if (line.startsWith("CERTIFICATE AUTHORITY")) { + authoritySection = true; + continue; + } + if (line.startsWith("CERTIFICATE")) { + authoritySection = false; + continue; + } + if (line.startsWith(MISSING_MARKER)) { + String name = StringUtils.trimToNull(line.substring(MISSING_MARKER.length())); + if (name != null) { + entries.add(new CertificateExpiry(name.split("\\s+")[0], null, authoritySection, false)); + } + continue; + } + Matcher matcher = CHECK_EXPIRATION_LINE.matcher(line); + if (!matcher.matches()) { + continue; + } + try { + Date expires = Date.from(ZonedDateTime.parse(matcher.group(2), CHECK_EXPIRATION_DATE_FORMAT).toInstant()); + String[] columns = line.split("\\s+"); + boolean externallyManaged = "yes".equalsIgnoreCase(columns[columns.length - 1]); + entries.add(new CertificateExpiry(matcher.group(1), expires, authoritySection, externallyManaged)); + } catch (DateTimeParseException e) { + LOGGER.debug("Unable to parse the expiry date in kubeadm output line: {}", line, e); + } + } + return entries; + } + + /** + * @return the expiry of the first {@code client-certificate-data} in a kubeconfig (or in a single line of one), + * or null if there is none or it cannot be decoded + */ + public static Date getKubeConfigClientCertificateExpiry(String kubeConfig) { + if (StringUtils.isBlank(kubeConfig)) { + return null; + } + Matcher matcher = CLIENT_CERTIFICATE_DATA.matcher(kubeConfig); + if (!matcher.find()) { + return null; + } + try { + byte[] pem = Base64.decodeBase64(matcher.group(1)); + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + X509Certificate certificate = (X509Certificate) factory.generateCertificate(new ByteArrayInputStream(pem)); + return certificate.getNotAfter(); + } catch (Exception e) { + LOGGER.debug("Unable to decode the client certificate of a kubeconfig", e); + return null; + } + } + + /** + * Parses a comma-separated list of day thresholds, ignoring blanks, non-numbers and non-positive values. + * @return the distinct thresholds, largest first + */ + public static List parseAlertThresholds(String value) { + TreeSet thresholds = new TreeSet<>(Collections.reverseOrder()); + if (StringUtils.isBlank(value)) { + return new ArrayList<>(thresholds); + } + for (String part : value.split(",")) { + try { + int days = Integer.parseInt(part.trim()); + if (days > 0) { + thresholds.add(days); + } + } catch (NumberFormatException e) { + LOGGER.warn("Ignoring invalid Kubernetes certificate expiry alert threshold: {}", part); + } + } + return new ArrayList<>(thresholds); + } + + /** @return whole days from now until expiry, rounded down; negative once expired */ + public static long daysUntil(Date expiry, Date now) { + return Math.floorDiv(expiry.getTime() - now.getTime(), TimeUnit.DAYS.toMillis(1)); + } + + /** + * Decides whether an expiry alert is due. + * @param daysLeft days until the certificate expires + * @param thresholds the alert thresholds, as returned by {@link #parseAlertThresholds(String)} + * @param lastAlertedThreshold the smallest threshold already alerted for this same expiry date, or null + * @return the threshold to alert for now, or null when no new alert is due. Only the smallest threshold reached + * is returned, so a cluster first checked at 20 days alerts once for 30, not for 60 and 30. Expiry itself counts + * as threshold 0, so it is always alerted. + */ + public static Integer getThresholdToAlert(long daysLeft, List thresholds, Integer lastAlertedThreshold) { + if (daysLeft < 0) { + return lastAlertedThreshold != null && lastAlertedThreshold <= 0 ? null : 0; + } + Integer reached = null; + for (Integer threshold : thresholds) { + if (daysLeft <= threshold && (reached == null || threshold < reached)) { + reached = threshold; + } + } + if (reached == null) { + return null; + } + if (lastAlertedThreshold != null && lastAlertedThreshold <= reached) { + return null; + } + return reached; + } + + /** + * Alert state persisted per cluster as {@code :}, so a renewal, + * which changes the expiry, starts the thresholds over. + * @return the threshold already alerted for this expiry, or null + */ + public static Integer getLastAlertedThreshold(String alertState, Date expiry) { + if (StringUtils.isBlank(alertState) || expiry == null) { + return null; + } + String[] parts = alertState.split(":"); + if (parts.length != 2) { + return null; + } + try { + if (Math.abs(Long.parseLong(parts[0]) - expiry.getTime()) > SAME_EXPIRY_TOLERANCE_MS) { + return null; + } + return Integer.parseInt(parts[1]); + } catch (NumberFormatException e) { + return null; + } + } + + public static String buildAlertState(Date expiry, int threshold) { + return String.format("%d:%d", expiry.getTime(), threshold); + } + + /** @param autoRenewDays the configured value, capped at {@link #MAX_AUTO_RENEW_DAYS} */ + public static boolean isAutoRenewDue(long daysLeft, int autoRenewDays) { + return autoRenewDays > 0 && daysLeft <= Math.min(autoRenewDays, MAX_AUTO_RENEW_DAYS); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 023fc9ec7fad..5f34c4c747fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -48,7 +48,7 @@ public class KubernetesClusterUtil { protected static Logger LOGGER = LogManager.getLogger(KubernetesClusterUtil.class); - public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'"; + public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && ($2 == \"Ready\" || $2 ~ /^Ready,/)) print $1}'"; public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $5}'"; public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port, @@ -198,6 +198,8 @@ public static boolean isKubernetesClusterDashboardServiceRunning(final Kubernete return running; } + private static final long KUBE_CONFIG_RETRY_INTERVAL_MS = 5000; + public static String getKubernetesClusterConfig(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String user, final File sshKeyFile, final long timeoutTime) { String kubeConfig = ""; @@ -218,6 +220,13 @@ public static String getKubernetesClusterConfig(final KubernetesCluster kubernet } catch (Exception e) { LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster %s", kubernetesCluster), e); } + // a refused connection fails at once; don't spin on it until the timeout + try { + Thread.sleep(Math.max(0, Math.min(KUBE_CONFIG_RETRY_INTERVAL_MS, timeoutTime - System.currentTimeMillis()))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } return kubeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java new file mode 100644 index 000000000000..c47b8ec16f86 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "rotateKubernetesClusterCertificates", + description = "Renews the kubeadm-managed control plane certificates of a running CloudManaged Kubernetes cluster, " + + "one control node at a time, and refreshes the kubeconfig returned by getKubernetesClusterConfig. " + + "The cluster certificate authorities are not rotated. Requires Kubernetes 1.20 or later", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class RotateKubernetesClusterCertificatesCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, required = true, + description = "The ID of the Kubernetes cluster") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE; + } + + @Override + public String getEventDescription() { + return "Rotating the certificates of Kubernetes cluster with ID: " + getResourceUuid(ApiConstants.ID); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + try { + if (!kubernetesClusterService.rotateKubernetesClusterCertificates(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + String.format("Failed to rotate the certificates of Kubernetes cluster with ID: %s", getResourceUuid(ApiConstants.ID))); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh new file mode 100644 index 000000000000..9372b2a9cb32 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh @@ -0,0 +1,235 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Renews the kubeadm-managed certificates of this control node and restarts its +# control plane static pods so they load them. The certificate authorities are +# left alone. Safe to rerun: an interrupted earlier run is recovered first. +# +# Usage: ./rotate-kubernetes-certificates.sh [WAIT_TIMEOUT_SECONDS] [rescue] +# +# rescue is for a cluster whose API server isn't answering, e.g. because its +# certificates have expired. With stacked etcd and several control nodes, one +# renewed node can't get etcd quorum until the others are renewed as well, so +# this node only waits for its static pods to be back, not for them to be +# healthy: without quorum the API server keeps restarting until it gets one. + +WAIT_TIMEOUT="${1:-300}" +RESCUE="${2:-}" + +# the binaries CKS installs come first, ahead of any the template ships +export PATH=/opt/bin:$PATH:/usr/local/bin:/usr/sbin:/sbin +CRI_ENDPOINT=unix:///run/containerd/containerd.sock + +MANIFESTS_DIR=/etc/kubernetes/manifests +HOLD_DIR=/etc/kubernetes/manifests-cks-certificate-rotation +COMPONENTS="kube-apiserver kube-controller-manager kube-scheduler etcd" +API_SERVER_PORT=6443 +# kubelet looks at the static pod manifests every 20 seconds by default +KUBELET_FILE_CHECK_WAIT=25 + +log() { + echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" +} + +# Puts back any manifest moved out by this script, also when it fails half-way, +# so the control plane is never left without its static pods +restore_manifests() { + if [ ! -d "${HOLD_DIR}" ]; then + return 0 + fi + for manifest in "${HOLD_DIR}"/*.yaml; do + [ -e "${manifest}" ] || continue + if [ -e "${MANIFESTS_DIR}/$(basename "${manifest}")" ]; then + rm -f "${manifest}" + else + mv "${manifest}" "${MANIFESTS_DIR}/" + fi + done + rmdir "${HOLD_DIR}" 2>/dev/null || true +} +trap restore_manifests EXIT + +# Keeps every distinct earlier /root/.kube/config: an existing backup is never overwritten +backup_root_kubeconfig() { + local backup=/root/.kube/config.cks-backup + if [ -e "${backup}" ]; then + cmp -s /root/.kube/config "${backup}" && return 0 + backup="${backup}.$(date +%s)" + fi + cp -p /root/.kube/config "${backup}" +} + +container_running() { + [ -n "$(crictl --runtime-endpoint "${CRI_ENDPOINT}" ps -q --name "^${1}\$" 2>/dev/null)" ] +} + +# A new sandbox of the static pod is up, not one torn down by this run; its +# containers may still be restarting +pod_ready() { + local id + for id in $(crictl --runtime-endpoint "${CRI_ENDPOINT}" pods -q --state ready --name "^${1}-" 2>/dev/null); do + case " ${old_pods} " in + *" ${id} "*) ;; + *) return 0 ;; + esac + done + return 1 +} + +endpoint_healthy() { + [ "$(curl -sk --max-time 5 "$1" 2>/dev/null)" = "ok" ] +} + +served_certificate_current() { + if ! command -v openssl >/dev/null 2>&1; then + return 0 + fi + local expected served + expected=$(openssl x509 -noout -enddate -in /etc/kubernetes/pki/apiserver.crt 2>/dev/null) + served=$(echo | openssl s_client -connect 127.0.0.1:${API_SERVER_PORT} 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null) + [ -n "${expected}" ] && [ "${expected}" = "${served}" ] +} + +# Returns non-zero if the API server isn't ready within the given number of seconds +wait_for_api_server() { + local limit=$(( $(date +%s) + $1 )) + [ "${limit}" -gt "${deadline}" ] && limit=${deadline} + while ! endpoint_healthy "https://127.0.0.1:${API_SERVER_PORT}/readyz"; do + if [ "$(date +%s)" -ge "${limit}" ]; then + return 1 + fi + sleep 3 + done +} + +deadline=$(( $(date +%s) + WAIT_TIMEOUT )) + +restore_manifests + +if [ ! -f /etc/kubernetes/admin.conf ] || [ ! -f "${MANIFESTS_DIR}/kube-apiserver.yaml" ]; then + log "ERROR: this is not a kubeadm control plane node" + exit 1 +fi + +# The static pods must be stopped and confirmed gone before their manifests come back +if ! crictl --runtime-endpoint "${CRI_ENDPOINT}" ps -q >/dev/null 2>&1; then + log "ERROR: unable to list the containers with crictl on ${CRI_ENDPOINT}, so the control plane pods restart can't be confirmed" + exit 1 +fi + +# kubeadm reads its configuration from the cluster and silently falls back to +# defaults when the API server is down, e.g. right after an interrupted run, so +# give it time to come up. Carry on if it doesn't: once the certificates have +# expired it never becomes ready, and renewing is the way out. +if [ "${RESCUE}" = "rescue" ]; then + log "Rescue mode: renewing without waiting for the API server" +elif ! wait_for_api_server 180; then + log "WARNING: the API server is not ready, renewing with the kubeadm configuration it can read" +fi + +# root's kubeconfig is replaced with admin.conf below; note now, before kubeadm +# renews admin.conf, whether it is something else worth a backup +root_kubeconfig_custom=false +if [ -f /root/.kube/config ] && ! cmp -s /etc/kubernetes/admin.conf /root/.kube/config; then + root_kubeconfig_custom=true +fi + +log "Renewing the control plane certificates" +apiserver_before=$(sha256sum /etc/kubernetes/pki/apiserver.crt | cut -d' ' -f1) +admin_before=$(sha256sum /etc/kubernetes/admin.conf | cut -d' ' -f1) +# under -e a failing command substitution would exit before kubeadm's error is printed +if ! renew_output=$(kubeadm certs renew all 2>&1); then + echo "${renew_output}" + log "ERROR: kubeadm certs renew all failed" + exit 1 +fi +echo "${renew_output}" +# kubeadm exits 0 without renewing anything signed by a CA whose key it doesn't have (external CA mode) +if echo "${renew_output}" | grep -q "can't be renewed" || \ + [ "$(sha256sum /etc/kubernetes/pki/apiserver.crt | cut -d' ' -f1)" = "${apiserver_before}" ] || \ + [ "$(sha256sum /etc/kubernetes/admin.conf | cut -d' ' -f1)" = "${admin_before}" ]; then + log "ERROR: kubeadm did not renew the control plane certificates, are they signed by an external CA?" + exit 1 +fi + +# CKS runs kubectl as root with /root/.kube/config, a copy of admin.conf +mkdir -p /root/.kube +if [ "${root_kubeconfig_custom}" = true ]; then + backup_root_kubeconfig +fi +cp /etc/kubernetes/admin.conf /root/.kube/config +chmod 600 /root/.kube/config + +log "Restarting the control plane static pods" +old_pods=$(crictl --runtime-endpoint "${CRI_ENDPOINT}" pods -q 2>/dev/null | tr '\n' ' ') +mkdir -p "${HOLD_DIR}" +moved="" +for component in ${COMPONENTS}; do + if [ -f "${MANIFESTS_DIR}/${component}.yaml" ]; then + mv "${MANIFESTS_DIR}/${component}.yaml" "${HOLD_DIR}/" + moved="${moved} ${component}" + fi +done + +sleep ${KUBELET_FILE_CHECK_WAIT} +for component in ${moved}; do + while container_running "${component}"; do + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "ERROR: timed out waiting for ${component} to stop" + exit 1 + fi + sleep 2 + done +done + +restore_manifests + +if [ "${RESCUE}" = "rescue" ]; then + log "Rescue mode: waiting for the control plane pods to be back" + while true; do + # control nodes of a cluster with separate etcd nodes run no etcd of their own + if { [ ! -f "${MANIFESTS_DIR}/etcd.yaml" ] || container_running etcd; } && \ + pod_ready kube-apiserver && pod_ready kube-controller-manager && pod_ready kube-scheduler; then + break + fi + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "ERROR: timed out waiting for the control plane pods to come back" + exit 1 + fi + sleep 3 + done + log "Control plane certificates renewed, the control plane becomes healthy once every control node is renewed" + exit 0 +fi + +log "Waiting for the control plane to become healthy" +while true; do + if endpoint_healthy "https://127.0.0.1:${API_SERVER_PORT}/readyz" && \ + endpoint_healthy "https://127.0.0.1:10257/healthz" && \ + endpoint_healthy "https://127.0.0.1:10259/healthz" && \ + served_certificate_current; then + break + fi + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "ERROR: timed out waiting for the control plane to become healthy" + exit 1 + fi + sleep 3 +done + +log "Control plane certificates renewed" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 2e73467c9823..53ac258a5e9e 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -123,6 +123,13 @@ if [ -d "$BINARIES_DIR" ]; then tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + # root's kubeconfig is kept in step with admin.conf below; note now, before + # kubeadm renews admin.conf, whether it is something else worth a backup + root_kubeconfig_custom=false + if [ -f /etc/kubernetes/admin.conf ] && [ -f /root/.kube/config ] && ! cmp -s /etc/kubernetes/admin.conf /root/.kube/config; then + root_kubeconfig_custom=true + fi + if [ "${IS_MAIN_CONTROL}" == 'true' ]; then set +e kubeadm --v=5 upgrade apply ${UPGRADE_VERSION} -y @@ -147,6 +154,20 @@ if [ -d "$BINARIES_DIR" ]; then systemctl restart containerd systemctl restart kubelet + # kubeadm upgrade renews admin.conf on control nodes; keep root's copy, used by kubectl here, in step with it + if [ -f /etc/kubernetes/admin.conf ] && [ -d /root/.kube ]; then + # never overwrite an earlier backup + if [ "${root_kubeconfig_custom}" = true ]; then + backup=/root/.kube/config.cks-backup + if [ -e "${backup}" ] && ! cmp -s /root/.kube/config "${backup}"; then + backup="${backup}.$(date +%s)" + fi + cp -p /root/.kube/config "${backup}" + fi + cp /etc/kubernetes/admin.conf /root/.kube/config + chmod 600 /root/.kube/config + fi + if [ "${IS_MAIN_CONTROL}" == 'true' ]; then if [[ ${EXTERNAL_CNI} == true ]]; then /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java new file mode 100644 index 000000000000..eacc44e2b51b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java @@ -0,0 +1,536 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterCertificateWorker; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class KubernetesClusterCertificateManagementTest { + + private static final long CLUSTER_ID = 11L; + private static final long DAY = TimeUnit.DAYS.toMillis(1); + + @Mock + KubernetesClusterDao kubernetesClusterDao; + @Mock + KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Mock + KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock + AccountManager accountManager; + + @Spy + @InjectMocks + KubernetesClusterManagerImpl manager; + + @Mock + KubernetesClusterVO cluster; + @Mock + KubernetesSupportedVersionVO version; + @Mock + KubernetesClusterCertificateWorker worker; + + private final Map details = new HashMap<>(); + private final Map, Object> originalDefaults = new HashMap<>(); + private Date now; + + @Before + public void setUp() throws Exception { + now = new Date(); + Mockito.when(cluster.getId()).thenReturn(CLUSTER_ID); + Mockito.when(cluster.getName()).thenReturn("cks"); + Mockito.when(cluster.getAccountId()).thenReturn(2L); + Mockito.when(cluster.getKubernetesVersionId()).thenReturn(4L); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(kubernetesSupportedVersionDao.findById(4L)).thenReturn(version); + Mockito.when(version.getSemanticVersion()).thenReturn("1.32.5"); + + Mockito.when(kubernetesClusterDetailsDao.findDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.anyString())).thenAnswer(invocation -> { + String name = invocation.getArgument(1); + String value = details.get(name); + return value == null ? null : new KubernetesClusterDetailsVO(CLUSTER_ID, name, value, false); + }); + Mockito.doAnswer(invocation -> details.put(invocation.getArgument(1), invocation.getArgument(2))) + .when(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyBoolean()); + Mockito.doAnswer(invocation -> details.remove(invocation.getArgument(1))) + .when(kubernetesClusterDetailsDao).removeDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.anyString()); + + Mockito.doReturn(worker).when(manager).createCertificateWorker(cluster); + Mockito.doReturn(true).when(manager).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + + setDefault(KubernetesClusterService.KubernetesServiceEnabled, "true"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateCheckInterval, "6"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateExpiryAlertDays, "60,30"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "true"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewDays, "30"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateRotationTimeout, "3600"); + } + + @After + public void tearDown() throws Exception { + for (Map.Entry, Object> entry : originalDefaults.entrySet()) { + Field field = ConfigKey.class.getDeclaredField("_defaultValue"); + field.setAccessible(true); + field.set(entry.getKey(), entry.getValue()); + clearCachedValue(entry.getKey()); + } + } + + // non-dynamic keys such as cloud.kubernetes.service.enabled cache their value, which would leak into other tests + private void clearCachedValue(ConfigKey key) throws Exception { + Field cached = ConfigKey.class.getDeclaredField("_value"); + cached.setAccessible(true); + cached.set(key, null); + } + + private void setDefault(ConfigKey key, String value) throws Exception { + Field field = ConfigKey.class.getDeclaredField("_defaultValue"); + field.setAccessible(true); + originalDefaults.putIfAbsent(key, field.get(key)); + field.set(key, value); + clearCachedValue(key); + } + + private CertificateExpiryReport report(long leafDays, long caDays) { + CertificateExpiryReport report = new CertificateExpiryReport(); + report.add("cks-control-1", Arrays.asList( + new CertificateExpiry("apiserver", new Date(now.getTime() + leafDays * DAY + 3600_000L), false, false), + new CertificateExpiry("admin.conf", new Date(now.getTime() + (leafDays + 10) * DAY), false, false), + new CertificateExpiry("ca", new Date(now.getTime() + caDays * DAY + 3600_000L), true, false))); + return report; + } + + /** Runs a check at now + elapsedDays, the certificates then expiring after the given number of days. */ + private void checkAt(long elapsedDays, long leafDays, long caDays) { + Date when = new Date(now.getTime() + elapsedDays * DAY); + Date oldNow = now; + now = when; + Mockito.doReturn(report(leafDays, caDays)).when(worker).checkCertificateExpiry(); + now = oldNow; + manager.processKubernetesClusterCertificates(cluster, when); + } + + private ArgumentCaptor verifyAlerts(int times) { + ArgumentCaptor subjects = ArgumentCaptor.forClass(String.class); + Mockito.verify(worker, Mockito.times(times)).sendAlert(subjects.capture(), ArgumentMatchers.anyString()); + Mockito.verify(worker, Mockito.times(times)).recordEvent(ArgumentMatchers.eq(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + return subjects; + } + + private ArgumentCaptor verifyAlertBodies(int times) { + ArgumentCaptor bodies = ArgumentCaptor.forClass(String.class); + Mockito.verify(worker, Mockito.times(times)).sendAlert(ArgumentMatchers.anyString(), bodies.capture()); + return bodies; + } + + @Test + public void noAlertWhileFarFromExpiry() { + checkAt(0, 200, 3000); + Mockito.verify(worker).checkCertificateExpiry(); + verifyAlerts(0); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + Assert.assertNotNull(details.get(KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_DETAIL)); + } + + @Test + public void checkRunsAtMostOncePerInterval() { + checkAt(0, 200, 3000); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(5))); + Mockito.verify(worker, Mockito.times(1)).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(6))); + Mockito.verify(worker, Mockito.times(2)).checkCertificateExpiry(); + } + + @Test + public void eachThresholdAlertsOnceThenAutoRenewTriggers() throws Exception { + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "false"); + // the same certificates, checked as their expiry approaches + checkAt(0, 59, 3000); + verifyAlerts(1); + checkAt(1, 58, 3000); + verifyAlerts(1); + checkAt(29, 30, 3000); + ArgumentCaptor subjects = verifyAlerts(2); + Assert.assertTrue(subjects.getAllValues().get(0).contains("expires in 59 days")); + Assert.assertTrue(subjects.getAllValues().get(1).contains("expires in 30 days")); + checkAt(30, 29, 3000); + verifyAlerts(2); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "true"); + checkAt(31, 28, 3000); + verifyAlerts(2); + Mockito.verify(worker).setAutomatic(true); + Mockito.verify(worker).rotateCertificates(); + } + + @Test + public void automaticRenewalRunsOffTheScannerThread() { + ExecutorService executor = Mockito.mock(ExecutorService.class); + manager._certificateRotationExecutor = executor; + try { + checkAt(0, 25, 3000); + verifyAlerts(1); + // handed to the rotation executor; the scanner doesn't wait for it + Mockito.verify(executor).execute(ArgumentMatchers.any(Runnable.class)); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } finally { + manager._certificateRotationExecutor = null; + } + } + + @Test + public void alertAdviceStatesTheEffectiveAutoRenewDays() throws Exception { + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "false"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewDays, "365"); + checkAt(0, 59, 3000); + ArgumentCaptor bodies = ArgumentCaptor.forClass(String.class); + Mockito.verify(worker).sendAlert(ArgumentMatchers.anyString(), bodies.capture()); + Assert.assertTrue(bodies.getValue().contains("Run rotateKubernetesClusterCertificates")); + + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "true"); + checkAt(1, 29, 3000); + Mockito.verify(worker, Mockito.times(2)).sendAlert(ArgumentMatchers.anyString(), bodies.capture()); + Assert.assertTrue(bodies.getValue(), bodies.getValue().contains("renewed automatically 180 days before")); + } + + @Test + public void noAutomaticRenewalWhileAControlNodeIsUnreachable() { + CertificateExpiryReport partial = report(25, 3000); + partial.addUncheckedNode("cks-control-2"); + Mockito.doReturn(partial).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + ArgumentCaptor bodies = verifyAlertBodies(1); + // the alert doesn't promise an automatic renewal that is on hold + Assert.assertTrue(bodies.getValue(), bodies.getValue().contains("on hold until control nodes [cks-control-2] can be checked")); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void clusterInAlertIsCheckedButNotRenewedAutomatically() { + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + checkAt(0, 25, 3000); + verifyAlerts(1); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void queuedAutomaticRotationIsSkippedWhenNoLongerNeeded() { + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + long queuedAt = now.getTime(); + + // renewed through the API while the automatic renewal waited for a thread + details.put(KubernetesClusterCertificateWorker.ROTATION_SUCCEEDED_DETAIL, String.valueOf(queuedAt + 1000)); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + // failed meanwhile, e.g. through the API: the backoff applies + details.put(KubernetesClusterCertificateWorker.ROTATION_SUCCEEDED_DETAIL, String.valueOf(queuedAt - DAY)); + details.put(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(queuedAt + 1000)); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + details.remove(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL); + + // no longer Running + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker).setAutomatic(true); + Mockito.verify(worker).rotateCertificates(); + } + + @Test + public void automaticRotationIsQueuedOnceAndRequeuedAfterARejection() { + ExecutorService executor = Mockito.mock(ExecutorService.class); + manager._certificateRotationExecutor = executor; + try { + Mockito.doThrow(new RejectedExecutionException("shutting down")).doNothing() + .when(executor).execute(ArgumentMatchers.any(Runnable.class)); + manager.scheduleAutomaticRotation(cluster); + // the rejected one isn't left marked as queued + manager.scheduleAutomaticRotation(cluster); + // still waiting for a thread: not queued again + manager.scheduleAutomaticRotation(cluster); + Mockito.verify(executor, Mockito.times(2)).execute(ArgumentMatchers.any(Runnable.class)); + } finally { + manager._certificateRotationExecutor = null; + } + } + + @Test + public void renewedCertificatesStartTheThresholdsOver() { + Mockito.when(worker.rotateCertificates()).thenReturn(true); + checkAt(0, 25, 3000); + verifyAlerts(1); + Mockito.verify(worker).rotateCertificates(); + // renewed: a new expiry date, a year out, then a year later the thresholds alert again + checkAt(1, 364, 3000); + verifyAlerts(1); + checkAt(306, 59, 3000); + verifyAlerts(2); + } + + @Test + public void certificateAuthorityExpiryAlertsButIsNeverRenewed() { + checkAt(0, 300, 45); + ArgumentCaptor subjects = verifyAlerts(1); + Assert.assertTrue(subjects.getValue().contains("certificate authority")); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void autoRenewBacksOffAfterAFailedRotation() { + Mockito.when(worker.rotateCertificates()).thenThrow(new CloudRuntimeException("node 2 failed")); + details.put(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(now.getTime() - TimeUnit.HOURS.toMillis(2))); + checkAt(0, 20, 3000); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + details.put(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(now.getTime() - TimeUnit.HOURS.toMillis(25))); + checkAt(1, 19, 3000); + Mockito.verify(worker).rotateCertificates(); + } + + @Test + public void externallyManagedCertificatesDoNotTriggerAutoRenew() { + CertificateExpiryReport report = new CertificateExpiryReport(); + report.add("cks-control-1", Arrays.asList( + new CertificateExpiry("apiserver-etcd-client", new Date(now.getTime() + 10 * DAY), false, true), + new CertificateExpiry("apiserver", new Date(now.getTime() + 300 * DAY), false, false))); + Mockito.doReturn(report).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + verifyAlerts(1); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void nothingHappensWhenTheCheckIsDisabledOrTheVersionTooOld() throws Exception { + setDefault(KubernetesClusterService.KubernetesClusterCertificateCheckInterval, "0"); + manager.processKubernetesClusterCertificates(cluster, now); + setDefault(KubernetesClusterService.KubernetesClusterCertificateCheckInterval, "6"); + Mockito.when(version.getSemanticVersion()).thenReturn("1.19.4"); + manager.processKubernetesClusterCertificates(cluster, now); + Mockito.verify(worker, Mockito.never()).checkCertificateExpiry(); + Assert.assertTrue(details.isEmpty()); + } + + @Test + public void unreachableClusterIsRetriedAtTheNextInterval() { + CertificateExpiryReport empty = new CertificateExpiryReport(); + empty.addUncheckedNode("cks-control-1"); + Mockito.doReturn(empty).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + 60_000)); + Mockito.verify(worker, Mockito.times(1)).checkCertificateExpiry(); + // no expiry alert, only the one that the check could not run + Mockito.verify(worker, Mockito.never()).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.verify(worker, Mockito.times(1)).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + } + + @Test + public void clusterThatCannotBeReachedAtAllAlerts() { + Mockito.doThrow(new CloudRuntimeException("Unable to retrieve the SSH address")).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + Mockito.verify(worker).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void clusterWhoseNodesCannotBeCheckedAlertsOnceADay() { + CertificateExpiryReport nothing = new CertificateExpiryReport(); + nothing.addUncheckedNode("cks-control-1"); + Mockito.doReturn(nothing).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(6))); + Mockito.verify(worker, Mockito.times(1)).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(24))); + Mockito.verify(worker, Mockito.times(2)).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + + // reachable again: the next failure alerts straight away + Mockito.doReturn(report(200, 3000)).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(30))); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL)); + } + + @Test + public void scannerDoesNothingWhileThePluginIsDisabled() throws Exception { + setDefault(KubernetesClusterService.KubernetesServiceEnabled, "false"); + manager.new KubernetesClusterCertificateScanner().reallyRun(); + Mockito.verify(kubernetesClusterDao, Mockito.never()).findManagedKubernetesClustersInState(ArgumentMatchers.any()); + Mockito.verify(manager, Mockito.never()).createCertificateWorker(ArgumentMatchers.any()); + } + + @Test + public void rotationOwnedByThisRestartedServerIsMovedToAlertRightAway() { + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + details.put(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL, String.valueOf(now.getTime() - TimeUnit.MINUTES.toMillis(2))); + details.put(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL, String.valueOf(ManagementServerNode.getManagementServerId())); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL)); + } + + @Test + public void rotationThatJustFinishedGetsNoStartTime() { + // listed as RotatingCertificates, but finished and cleared its details before it was looked at + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + manager.recoverInterruptedCertificateRotations(now); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL)); + } + + @Test + public void rotationWithoutStartTimeIsTimedFromWhenFirstSeen() { + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + KubernetesClusterVO rotating = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(rotating.getState()).thenReturn(KubernetesCluster.State.RotatingCertificates); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(rotating); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + // the rotation's own start detail is never written here, it would race with the rotation writing it + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL)); + Assert.assertEquals(String.valueOf(now.getTime()), details.get(KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL)); + + // still without a start well past the timeout: it died before recording it + manager.recoverInterruptedCertificateRotations(new Date(now.getTime() + TimeUnit.HOURS.toMillis(3))); + Mockito.verify(manager).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL)); + } + + @Test + public void interruptedRotationIsMovedToAlertOnceStale() { + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + // owned by another management server, which may still be running it + details.put(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL, String.valueOf(ManagementServerNode.getManagementServerId() + 1)); + details.put(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL, String.valueOf(now.getTime() - TimeUnit.MINUTES.toMillis(30))); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + + details.put(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL, String.valueOf(now.getTime() - TimeUnit.HOURS.toMillis(3))); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL)); + Assert.assertNotNull(details.get(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL)); + } + + private RotateKubernetesClusterCertificatesCmd rotateCmd() { + RotateKubernetesClusterCertificatesCmd cmd = Mockito.mock(RotateKubernetesClusterCertificatesCmd.class); + Mockito.when(cmd.getId()).thenReturn(CLUSTER_ID); + Mockito.when(cmd.getActualCommandName()).thenReturn("rotateKubernetesClusterCertificates"); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + return cmd; + } + + @Test + public void rotationApiValidation() { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + RotateKubernetesClusterCertificatesCmd cmd = rotateCmd(); + Assert.assertEquals(cluster, manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Upgrading); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + // a failed rotation leaves the cluster in Alert, and it can be retried from there + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + Assert.assertEquals(cluster, manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(version.getSemanticVersion()).thenReturn("1.18.0"); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(version.getSemanticVersion()).thenReturn("1.32.5"); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(null); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + } finally { + CallContext.unregister(); + } + } + + private KubernetesCluster.State next(KubernetesCluster.State state, KubernetesCluster.Event event) { + try { + return KubernetesCluster.State.getStateMachine().getNextState(state, event); + } catch (NoTransitionException e) { + return null; + } + } + + @Test + public void stateMachineAllowsRotationOnlyFromRunning() { + Assert.assertEquals(KubernetesCluster.State.RotatingCertificates, next(KubernetesCluster.State.Running, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.Upgrading, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertEquals(KubernetesCluster.State.RotatingCertificates, next(KubernetesCluster.State.Alert, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.Stopped, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.UpgradeRequested)); + Assert.assertNull(next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.ScaleUpRequested)); + Assert.assertEquals(KubernetesCluster.State.Running, next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.OperationSucceeded)); + Assert.assertEquals(KubernetesCluster.State.Alert, next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.OperationFailed)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java new file mode 100644 index 000000000000..733697c526b7 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java @@ -0,0 +1,526 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.codec.binary.Base64; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class KubernetesClusterCertificateWorkerTest { + + private static final long CLUSTER_ID = 7L; + + // see KubernetesClusterCertificateUtilTest + private static final String CHECK_EXPIRATION_OUTPUT = + "CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED\n" + + "admin.conf Nov 09, 2027 19:52 UTC 364d ca no\n" + + "apiserver Nov 09, 2027 19:52 UTC 364d ca no\n" + + "\n" + + "CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED\n" + + "ca Nov 06, 2036 19:52 UTC 9y no\n"; + private static final String CLIENT_CERTIFICATE_DATA = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnRENDQVNhZ0F3SUJBZ0lKQUtpUVNqdnZrSkN1TUFvR0NDcUdTTTQ5QkFNRE1EUXhGekFWQmdOVkJBb1QNCkRuTjVjM1JsYlRwdFlYTjBaWEp6TVJrd0Z3WURWUVFERXhCcmRXSmxjbTVsZEdWekxXRmtiV2x1TUI0WERUSTINCk1ERXdNVEF3TURBd01Gb1hEVEkzTURFd01qQXdNREF3TUZvd05ERVhNQlVHQTFVRUNoTU9jM2x6ZEdWdE9tMWgNCmMzUmxjbk14R1RBWEJnTlZCQU1URUd0MVltVnlibVYwWlhNdFlXUnRhVzR3V1RBVEJnY3Foa2pPUFFJQkJnZ3ENCmhrak9QUU1CQndOQ0FBUktSTGpVaDlYUGh4R0JvdUFNaFd5MmdmUWJZaG9HbEFwMVFkRDZKQ2tYTTM2YnBCSmINCjB0NGJRblJJU2pmUkpnWDVETHpBOFZhNC96Q3ZhVDdETW1jTG95RXdIekFkQmdOVkhRNEVGZ1FVVndXTXdmU0oNClRHQjBQLzJweDF4Nzh6eEVSQzR3Q2dZSUtvWkl6ajBFQXdNRFNBQXdSUUlnY1F6SGtsNk9WQm52T0tHNUNkNnMNClNxZmtMYU1oMFk1M0ovcHlBOFAwMm9FQ0lRRHB4ZkxEUFl5QzhLRVZvaW5YSUN2clo1SWVJNG5IRFUwYUtGZ2sNCml4ZGIwZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"; + // what the combined check command prints: kubeadm's tables, root's kubeconfig certificate line, and nothing for + // user.conf when there is none + private static final String CHECK_OUTPUT = CHECK_EXPIRATION_OUTPUT + "----- cks root kubeconfig -----\n" + + " client-certificate-data: " + CLIENT_CERTIFICATE_DATA + "\n" + + "----- cks user kubeconfig -----\n"; + + @Mock + private KubernetesCluster kubernetesCluster; + @Mock + private KubernetesClusterManagerImpl clusterManager; + @Mock + private KubernetesClusterDao kubernetesClusterDao; + @Mock + private KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Mock + private KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock + private KubernetesSupportedVersionVO version; + @Mock + private UserVm control1; + @Mock + private UserVm control2; + @Mock + private UserVm control3; + + private KubernetesClusterCertificateWorker worker; + private File scriptFile; + + @Before + public void setUp() throws IOException { + clusterManager.kubernetesClusterDao = kubernetesClusterDao; + clusterManager.kubernetesClusterDetailsDao = kubernetesClusterDetailsDao; + clusterManager.kubernetesSupportedVersionDao = kubernetesSupportedVersionDao; + Mockito.when(kubernetesCluster.getId()).thenReturn(CLUSTER_ID); + Mockito.when(kubernetesCluster.getName()).thenReturn("cks"); + Mockito.when(kubernetesCluster.getKubernetesVersionId()).thenReturn(3L); + Mockito.when(kubernetesSupportedVersionDao.findById(3L)).thenReturn(version); + Mockito.when(version.getSemanticVersion()).thenReturn("1.31.1"); + Mockito.when(control1.getHostName()).thenReturn("cks-control-1"); + Mockito.when(control2.getHostName()).thenReturn("cks-control-2"); + Mockito.when(control3.getHostName()).thenReturn("cks-control-3"); + + worker = Mockito.spy(new KubernetesClusterCertificateWorker(kubernetesCluster, clusterManager)); + scriptFile = File.createTempFile("rotate-kubernetes-certificates.sh", ".sh"); + Mockito.doNothing().when(worker).init(); + Mockito.doNothing().when(worker).resolveClusterEndpoints(); + Mockito.doReturn(Arrays.asList(control1, control2, control3)).when(worker).getControlNodeVMs(); + Mockito.doReturn(scriptFile).when(worker).retrieveScriptFile(KubernetesClusterCertificateWorker.ROTATION_SCRIPT_FILENAME); + Mockito.doReturn(true).when(worker).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + Mockito.doReturn(true).when(worker).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Mockito.doReturn(true).when(worker).waitForControlNodeReady(ArgumentMatchers.any(UserVm.class)); + Mockito.doReturn(false).when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + Mockito.doReturn(true).when(worker).isStillOwnRotation(); + Mockito.doReturn(true).when(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + Mockito.doNothing().when(worker).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.doNothing().when(worker).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + } + + @After + public void tearDown() { + scriptFile.delete(); + } + + @Test + public void rotationRenewsOneControlNodeAtATimeThenRefreshesTheKubeConfig() { + Assert.assertTrue(worker.rotateCertificates()); + + InOrder inOrder = Mockito.inOrder(worker, kubernetesClusterDetailsDao); + inOrder.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.CertificateRotationRequested); + inOrder.verify(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.eq(false)); + inOrder.verify(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.eq(false)); + inOrder.verify(worker).renewCertificatesOnControlNode(control1, 0); + inOrder.verify(worker).waitForControlNodeReady(control1); + inOrder.verify(worker).renewCertificatesOnControlNode(control2, 1); + inOrder.verify(worker).waitForControlNodeReady(control2); + inOrder.verify(worker).renewCertificatesOnControlNode(control3, 2); + inOrder.verify(worker).waitForControlNodeReady(control3); + inOrder.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + inOrder.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + inOrder.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + inOrder.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + // the API records its own event; only an automatic renewal records one here + Mockito.verify(worker, Mockito.never()).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Assert.assertFalse("the local copy of the script is cleaned up", scriptFile.exists()); + } + + @Test + public void automaticRotationRecordsAnEvent() { + worker.setAutomatic(true); + Assert.assertTrue(worker.rotateCertificates()); + Mockito.verify(worker).recordEvent(ArgumentMatchers.eq("KUBERNETES.CLUSTER.CERTIFICATES.ROTATE"), ArgumentMatchers.eq("INFO"), ArgumentMatchers.anyString()); + } + + private void assertRotationFailed() { + try { + worker.rotateCertificates(); + Assert.fail("rotation should have failed"); + } catch (CloudRuntimeException e) { + // expected + } + Mockito.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(worker).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.verify(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.eq(false)); + Mockito.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + Mockito.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + // the API's own async job event records the failure; only an automatic renewal records one here + Mockito.verify(worker, Mockito.never()).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + } + + @Test + public void automaticRotationRecordsItsFailure() { + worker.setAutomatic(true); + Mockito.doReturn(false).when(worker).renewCertificatesOnControlNode(control1, 0); + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + Mockito.verify(worker).recordEvent(ArgumentMatchers.eq("KUBERNETES.CLUSTER.CERTIFICATES.ROTATE"), ArgumentMatchers.eq("ERROR"), ArgumentMatchers.anyString()); + } + + @Test + public void rotationOfAnUnreachableClusterRenewsEveryNodeBeforeWaiting() { + // expired certificates: with stacked etcd no node gets quorum until all are renewed + Mockito.doReturn(true).when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + + Assert.assertTrue(worker.rotateCertificates()); + + InOrder inOrder = Mockito.inOrder(worker); + inOrder.verify(worker).renewCertificatesOnControlNode(control1, 0); + inOrder.verify(worker).renewCertificatesOnControlNode(control2, 1); + inOrder.verify(worker).renewCertificatesOnControlNode(control3, 2); + inOrder.verify(worker).waitForControlNodeReady(control1); + inOrder.verify(worker).waitForControlNodeReady(control2); + inOrder.verify(worker).waitForControlNodeReady(control3); + inOrder.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + inOrder.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + } + + @Test + public void secondRotationOfTheSameClusterIsRefusedWithoutDisturbingTheFirst() { + Mockito.doAnswer(invocation -> { + // a second rotation starts while the first renews node 0 + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + Assert.assertTrue(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + return true; + }).when(worker).renewCertificatesOnControlNode(control1, 0); + + Assert.assertTrue(worker.rotateCertificates()); + Mockito.verify(worker, Mockito.times(1)).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.CertificateRotationRequested); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + } + + @Test + public void rotationFromAlertReturnsToAlertForTheStatusScanner() { + KubernetesClusterVO alertCluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(alertCluster.getState()).thenReturn(KubernetesCluster.State.Alert); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(alertCluster); + + Assert.assertTrue(worker.rotateCertificates()); + + // whatever else put it in Alert hasn't been checked; the status scanner reconciles it to Running + Mockito.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(worker, Mockito.never()).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).addDetail(ArgumentMatchers.eq(CLUSTER_ID), + ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.anyBoolean()); + } + + @Test + public void rotationLeavesAClusterTakenMeanwhileInItsState() { + // e.g. moved to Alert as an interrupted rotation by another management server, and maybe retried since + Mockito.doReturn(false).when(worker).isStillOwnRotation(); + Assert.assertTrue(worker.rotateCertificates()); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + } + + @Test + public void rotationOfAnUnreachableClusterFailsWhenANodeDoesNotComeBack() { + Mockito.doReturn(true).when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + Mockito.doReturn(false).when(worker).waitForControlNodeReady(control2); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control3, 2); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void rotationStopsAtTheFirstControlNodeThatFailsToRenew() { + Mockito.doReturn(false).when(worker).renewCertificatesOnControlNode(control2, 1); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control1, 0); + Mockito.verify(worker).waitForControlNodeReady(control1); + Mockito.verify(worker).renewCertificatesOnControlNode(control2, 1); + Mockito.verify(worker, Mockito.never()).waitForControlNodeReady(control2); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.eq(control3), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void rotationStopsWhenAControlNodeDoesNotBecomeReady() { + Mockito.doReturn(false).when(worker).waitForControlNodeReady(control1); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control1, 0); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.eq(control2), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.eq(control3), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void rotationFailsWhenTheKubeConfigCannotBeRefreshed() { + Mockito.doReturn(false).when(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control3, 2); + } + + @Test + public void rotationFailsOnAnUnexpectedError() { + Mockito.doThrow(new CloudRuntimeException("no public IP")).when(worker).resolveClusterEndpoints(); + + assertRotationFailed(); + + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + } + + @Test + public void rotationIsRefusedUnlessTheClusterCanTransitionFromRunning() { + Mockito.doReturn(false).when(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.CertificateRotationRequested); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + // a refused request leaves the running rotation's bookkeeping alone + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).addDetail(ArgumentMatchers.anyLong(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyBoolean()); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).removeDetail(ArgumentMatchers.anyLong(), ArgumentMatchers.anyString()); + } + + @Test + public void rotationIsActiveWhileRunning() { + Mockito.doAnswer(invocation -> { + Assert.assertTrue(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + return true; + }).when(worker).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Assert.assertTrue(worker.rotateCertificates()); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + } + + @Test + public void rotationIsRefusedBeforeKubernetes120() { + Mockito.when(version.getSemanticVersion()).thenReturn("1.19.16"); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + + Mockito.verify(worker, Mockito.never()).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + } + + @Test + public void certificateManagementVersionGate() { + Assert.assertFalse(KubernetesClusterCertificateWorker.isCertificateManagementSupported(null)); + Mockito.when(version.getSemanticVersion()).thenReturn("1.19.16"); + Assert.assertFalse(KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)); + Mockito.when(version.getSemanticVersion()).thenReturn("1.20.0"); + Assert.assertTrue(KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)); + Mockito.when(version.getSemanticVersion()).thenReturn("1.33.1"); + Assert.assertTrue(KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)); + } + + private void storeKubeConfig(String certificateData) { + KubernetesClusterDetailsVO detail = Mockito.mock(KubernetesClusterDetailsVO.class); + String kubeConfig = "users:\n- name: kubernetes-admin\n user:\n client-certificate-data: " + certificateData + "\n"; + Mockito.when(detail.getValue()).thenReturn(Base64.encodeBase64String(kubeConfig.getBytes(StandardCharsets.UTF_8))); + Mockito.when(kubernetesClusterDetailsDao.findDetail(CLUSTER_ID, KubernetesClusterActionWorker.KUBE_CONFIG_DATA_DETAIL)).thenReturn(detail); + } + + @Test + public void checkReadsEveryControlNodeAndRefreshesAStaleStoredKubeConfig() throws Exception { + // the stored kubeconfig's certificate expires in Jan 2027, the cluster's admin.conf in Nov 2027 + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, CHECK_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.contains(KubernetesClusterCertificateUtil.CHECK_EXPIRATION_COMMAND), ArgumentMatchers.anyInt()); + Mockito.doReturn(new Pair<>(false, "connection refused")).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2224), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals(List.of("cks-control-3"), report.getUncheckedNodes()); + // 3 certificates from kubeadm and root's kubeconfig, on two nodes + Assert.assertEquals(8, report.getEntries().size()); + // root's kubeconfig expires first: renewing syncs it with admin.conf + Assert.assertEquals("/root/.kube/config on cks-control-1", report.getFirstLeafExpiry().getName()); + Assert.assertEquals("ca on cks-control-1", report.getFirstAuthorityExpiry().getName()); + Mockito.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void checkComparesTheStoredKubeConfigWithUserConfWhenThereIsOne() throws Exception { + // getKubernetesClusterConfig hands out user.conf; its certificate is the one stored, so nothing is stale + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + String withUserConf = CHECK_OUTPUT + "user.conf present\n client-certificate-data: " + CLIENT_CERTIFICATE_DATA + "\n"; + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, withUserConf)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals(12, report.getEntries().size()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + // a user.conf without a client certificate (e.g. a token) can't be compared, so it is left alone too + Mockito.doReturn(new Pair<>(true, CHECK_OUTPUT + "user.conf present\n")).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + worker.checkCertificateExpiry(); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void expiryForRescueLooksAtEveryControlNode() throws Exception { + String expired = CHECK_EXPIRATION_OUTPUT.replace("apiserver Nov 09, 2027 19:52 UTC 364d ", + "apiserver Jan 01, 2020 00:00 UTC "); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, CHECK_EXPIRATION_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2222), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doThrow(new java.io.IOException("timeout")).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2224), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doReturn(new Pair<>(true, CHECK_EXPIRATION_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2223), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doCallRealMethod().when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + List nodes = Arrays.asList(control1, control2, control3); + + Assert.assertFalse(worker.areCertificatesExpired(nodes)); + + // an expired admin client certificate breaks no control plane component: no reason to restart every node at once + String adminExpired = CHECK_EXPIRATION_OUTPUT.replace("admin.conf Nov 09, 2027 19:52 UTC 364d ", + "admin.conf Jan 01, 2020 00:00 UTC "); + Assert.assertNotEquals(CHECK_EXPIRATION_OUTPUT, adminExpired); + Mockito.doReturn(new Pair<>(true, adminExpired)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2223), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Assert.assertFalse(worker.areCertificatesExpired(nodes)); + + // the first node already renewed by an interrupted rescue, the second still expired + Mockito.doReturn(new Pair<>(false, expired)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2223), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Assert.assertTrue(worker.areCertificatesExpired(nodes)); + } + + @Test + public void rootKubeConfigIsNotRenewableInExternalCaMode() throws Exception { + String external = CHECK_OUTPUT.replace("ca no\n", "ca yes\n"); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Mockito.doReturn(new Pair<>(true, external)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals("/root/.kube/config on cks-control-1", report.getFirstLeafExpiry().getName()); + Assert.assertNull(report.getFirstRenewableLeafExpiry()); + } + + @Test + public void checkSkipsTheKubeConfigRefreshWhenTheFirstControlNodeIsDown() throws Exception { + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, CHECK_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doThrow(new java.io.IOException("timeout")).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2222), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals(List.of("cks-control-1"), report.getUncheckedNodes()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void storedKubeConfigIsLeftAloneWhenCurrent() { + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + Date storedExpiry = worker.getStoredKubeConfigExpiry(); + Assert.assertNotNull(storedExpiry); + + worker.refreshStaleKubeConfigData(storedExpiry); + worker.refreshStaleKubeConfigData(new Date(storedExpiry.getTime() - 1000)); + worker.refreshStaleKubeConfigData(null); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + worker.refreshStaleKubeConfigData(new Date(storedExpiry.getTime() + 1000)); + Mockito.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + private PortForwardingRuleVO sshRule(long ipId, int sourcePort, int destinationPort) { + PortForwardingRuleVO rule = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(ipId); + Mockito.when(rule.getSourcePortStart()).thenReturn(sourcePort); + Mockito.when(rule.getDestinationPortStart()).thenReturn(destinationPort); + Mockito.when(rule.getState()).thenReturn(FirewallRule.State.Active); + return rule; + } + + @Test + public void controlNodeSshEndpoint() { + PortForwardingRulesDao portForwardingRulesDao = Mockito.mock(PortForwardingRulesDao.class); + IPAddressDao ipAddressDao = Mockito.mock(IPAddressDao.class); + worker.portForwardingRulesDao = portForwardingRulesDao; + worker.ipAddressDao = ipAddressDao; + IPAddressVO clusterIp = Mockito.mock(IPAddressVO.class); + Mockito.when(clusterIp.getAddress()).thenReturn(new Ip("192.0.2.10")); + IPAddressVO otherIp = Mockito.mock(IPAddressVO.class); + Mockito.when(otherIp.getAddress()).thenReturn(new Ip("192.0.2.99")); + Mockito.when(ipAddressDao.findById(1L)).thenReturn(clusterIp); + Mockito.when(ipAddressDao.findById(2L)).thenReturn(otherIp); + Mockito.when(control1.getId()).thenReturn(101L); + Mockito.when(control2.getId()).thenReturn(102L); + Mockito.when(control2.getPrivateIpAddress()).thenReturn("10.1.1.12"); + + // directly reachable network: the VM's own address + worker.publicIpAddress = "10.1.1.11"; + worker.sshPort = 22; + Assert.assertEquals(new Pair<>("10.1.1.12", 22), worker.getControlNodeSshEndpoint(control2, 1)); + + // behind the cluster public IP: the VM's SSH port forwarding rule on that IP, not a user's on another one + worker.publicIpAddress = "192.0.2.10"; + worker.sshPort = 2222; + List rules = Arrays.asList(sshRule(2L, 2022, 22), sshRule(1L, 8080, 80), sshRule(1L, 2224, 22)); + Mockito.when(portForwardingRulesDao.listByVm(102L)).thenReturn(rules); + Assert.assertEquals(new Pair<>("192.0.2.10", 2224), worker.getControlNodeSshEndpoint(control2, 1)); + + // no rule (external load balancer): by position + Mockito.when(portForwardingRulesDao.listByVm(101L)).thenReturn(Collections.emptyList()); + Assert.assertEquals(new Pair<>("192.0.2.10", 2222), worker.getControlNodeSshEndpoint(control1, 0)); + Assert.assertEquals(new Pair<>("192.0.2.10", 2223), worker.getControlNodeSshEndpoint(control1, 1)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java index ff8f875d06e1..888af3a84121 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java @@ -18,19 +18,27 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -45,6 +53,10 @@ public class KubernetesClusterUpgradeWorkerTest { private KubernetesClusterManagerImpl clusterManager; @Mock private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Mock + private KubernetesClusterDao kubernetesClusterDao; + @Mock + private NetworkDao networkDao; private KubernetesClusterUpgradeWorker worker; @@ -80,4 +92,64 @@ public void testFilterOutManualUpgradeNodesFromClusterUpgrade() { Assert.assertTrue(ids.contains(controlNodeId) && ids.contains(workerNode1Id)); Assert.assertFalse(ids.contains(workerNode2Id)); } + + private KubernetesClusterUpgradeWorker prepareUpgrade(boolean clusterUpdated) { + long clusterId = 5L; + Mockito.when(kubernetesCluster.getId()).thenReturn(clusterId); + Mockito.lenient().when(kubernetesCluster.getNetworkId()).thenReturn(9L); + Mockito.lenient().when(networkDao.findById(9L)).thenReturn(Mockito.mock(NetworkVO.class)); + KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(kubernetesClusterDao.findById(clusterId)).thenReturn(clusterVO); + Mockito.when(kubernetesClusterDao.update(clusterId, clusterVO)).thenReturn(clusterUpdated); + UserVm controlNode = Mockito.mock(UserVm.class); + worker.kubernetesClusterDao = kubernetesClusterDao; + worker.networkDao = networkDao; + KubernetesClusterUpgradeWorker spy = Mockito.spy(worker); + Mockito.doNothing().when(spy).init(); + Mockito.doReturn(new Pair<>("10.1.1.10", 2222)).when(spy).getKubernetesClusterServerIpSshPort(null); + Mockito.doReturn(Collections.singletonList(controlNode)).when(spy).getKubernetesClusterVMs(); + Mockito.doNothing().when(spy).filterOutManualUpgradeNodesFromClusterUpgrade(); + Mockito.doNothing().when(spy).retrieveScriptFiles(); + Mockito.doReturn(true).when(spy).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + Mockito.doNothing().when(spy).attachIsoKubernetesVMs(ArgumentMatchers.anyList(), ArgumentMatchers.any()); + Mockito.doNothing().when(spy).upgradeKubernetesClusterNodes(); + Mockito.doNothing().when(spy).detachIsoKubernetesVMs(ArgumentMatchers.anyList()); + return spy; + } + + @Test + public void testUpgradeRefreshesTheStoredKubeConfigBeforeCompleting() { + KubernetesClusterUpgradeWorker spy = prepareUpgrade(true); + Mockito.doReturn(true).when(spy).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + Assert.assertTrue(spy.upgradeCluster()); + + InOrder inOrder = Mockito.inOrder(spy); + inOrder.verify(spy).upgradeKubernetesClusterNodes(); + inOrder.verify(spy).refreshKubeConfigData(ArgumentMatchers.anyLong()); + inOrder.verify(spy).stateTransitTo(5L, KubernetesCluster.Event.OperationSucceeded); + // the kubeconfig must point at the cluster's public endpoint + Assert.assertEquals("10.1.1.10", spy.apiPublicIpAddress); + } + + @Test + public void testUpgradeSucceedsWhenTheKubeConfigRefreshFails() { + KubernetesClusterUpgradeWorker spy = prepareUpgrade(true); + Mockito.doReturn(false).when(spy).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + Assert.assertTrue(spy.upgradeCluster()); + + Mockito.verify(spy).stateTransitTo(5L, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(spy, Mockito.never()).stateTransitTo(5L, KubernetesCluster.Event.OperationFailed); + } + + @Test + public void testFailedUpgradeDoesNotRefreshTheKubeConfig() { + KubernetesClusterUpgradeWorker spy = prepareUpgrade(false); + + Assert.assertFalse(spy.upgradeCluster()); + + Mockito.verify(spy, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + Mockito.verify(spy).stateTransitTo(5L, KubernetesCluster.Event.OperationFailed); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java new file mode 100644 index 000000000000..f98aecb33d2a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.utils; + +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; + +public class KubernetesClusterCertificateUtilTest { + + // Output of kubeadm v1.31 on a stacked etcd control node + static final String CHECK_EXPIRATION_OUTPUT = + "[check-expiration] Reading configuration from the cluster...\n" + + "[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'\n" + + "\n" + + "CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED\n" + + "admin.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "apiserver Nov 09, 2027 19:52 UTC 364d ca no \n" + + "apiserver-etcd-client Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "apiserver-kubelet-client Nov 09, 2027 19:52 UTC 364d ca no \n" + + "controller-manager.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "etcd-healthcheck-client Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "etcd-peer Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "etcd-server Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "front-proxy-client Nov 01, 2027 08:05 UTC 356d front-proxy-ca no \n" + + "scheduler.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "super-admin.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "\n" + + "CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED\n" + + "ca Nov 06, 2036 19:52 UTC 9y no \n" + + "etcd-ca Nov 06, 2036 19:52 UTC 9y no \n" + + "front-proxy-ca Mar 03, 2035 10:00 UTC 8y no \n"; + + // Certificate of a kubeconfig user, valid until Jan 2 00:00:00 2027 GMT + static final String CLIENT_CERTIFICATE_DATA = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnRENDQVNhZ0F3SUJBZ0lKQUtpUVNqdnZrSkN1TUFvR0NDcUdTTTQ5QkFNRE1EUXhGekFWQmdOVkJBb1QNCkRuTjVjM1JsYlRwdFlYTjBaWEp6TVJrd0Z3WURWUVFERXhCcmRXSmxjbTVsZEdWekxXRmtiV2x1TUI0WERUSTINCk1ERXdNVEF3TURBd01Gb1hEVEkzTURFd01qQXdNREF3TUZvd05ERVhNQlVHQTFVRUNoTU9jM2x6ZEdWdE9tMWgNCmMzUmxjbk14R1RBWEJnTlZCQU1URUd0MVltVnlibVYwWlhNdFlXUnRhVzR3V1RBVEJnY3Foa2pPUFFJQkJnZ3ENCmhrak9QUU1CQndOQ0FBUktSTGpVaDlYUGh4R0JvdUFNaFd5MmdmUWJZaG9HbEFwMVFkRDZKQ2tYTTM2YnBCSmINCjB0NGJRblJJU2pmUkpnWDVETHpBOFZhNC96Q3ZhVDdETW1jTG95RXdIekFkQmdOVkhRNEVGZ1FVVndXTXdmU0oNClRHQjBQLzJweDF4Nzh6eEVSQzR3Q2dZSUtvWkl6ajBFQXdNRFNBQXdSUUlnY1F6SGtsNk9WQm52T0tHNUNkNnMNClNxZmtMYU1oMFk1M0ovcHlBOFAwMm9FQ0lRRHB4ZkxEUFl5QzhLRVZvaW5YSUN2clo1SWVJNG5IRFUwYUtGZ2sNCml4ZGIwZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"; + static final Date CLIENT_CERTIFICATE_EXPIRY = utc(2027, 1, 2, 0, 0); + + static Date utc(int year, int month, int day, int hour, int minute) { + return Date.from(ZonedDateTime.parse(String.format("%04d-%02d-%02dT%02d:%02d:00Z", year, month, day, hour, minute)).toInstant()); + } + + static String kubeConfig(String certificateData) { + return "apiVersion: v1\n" + + "clusters:\n" + + "- cluster:\n" + + " certificate-authority-data: Y2E=\n" + + " server: https://10.1.1.10:6443\n" + + " name: kubernetes\n" + + "kind: Config\n" + + "users:\n" + + "- name: kubernetes-admin\n" + + " user:\n" + + " client-certificate-data: " + certificateData + "\n" + + " client-key-data: a2V5\n"; + } + + private CertificateExpiry find(List entries, String name) { + return entries.stream().filter(e -> e.getName().equals(name)).findFirst().orElse(null); + } + + @Test + public void parseCheckExpirationOutputReadsBothTables() { + List entries = KubernetesClusterCertificateUtil.parseCheckExpirationOutput(CHECK_EXPIRATION_OUTPUT); + Assert.assertEquals(14, entries.size()); + Assert.assertEquals(11, entries.stream().filter(e -> !e.isAuthority()).count()); + Assert.assertEquals(3, entries.stream().filter(CertificateExpiry::isAuthority).count()); + + CertificateExpiry admin = find(entries, "admin.conf"); + Assert.assertFalse(admin.isAuthority()); + Assert.assertFalse(admin.isMissing()); + Assert.assertFalse(admin.isExternallyManaged()); + Assert.assertEquals(utc(2027, 11, 9, 19, 52), admin.getExpires()); + + Assert.assertEquals(utc(2027, 11, 1, 8, 5), find(entries, "front-proxy-client").getExpires()); + CertificateExpiry ca = find(entries, "ca"); + Assert.assertTrue(ca.isAuthority()); + Assert.assertEquals(utc(2036, 11, 6, 19, 52), ca.getExpires()); + Assert.assertEquals(utc(2035, 3, 3, 10, 0), find(entries, "front-proxy-ca").getExpires()); + } + + @Test + public void parseCheckExpirationOutputHandlesMissingExpiredAndExternalCertificates() { + String output = "CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED\n" + + "admin.conf Jan 05, 2026 10:00 UTC ca no\n" + + "apiserver Feb 07, 2027 10:00 UTC 89d ca no\n" + + "!MISSING! apiserver-etcd-client\n" + + "front-proxy-client Dec 24, 2026 10:00 UTC 44d front-proxy-ca yes\n" + + "\n" + + "CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED\n" + + "ca Nov 06, 2036 19:52 UTC 9y no\n" + + "!MISSING! etcd-ca\n" + + "some unexpected line\n"; + List entries = KubernetesClusterCertificateUtil.parseCheckExpirationOutput(output); + Assert.assertEquals(6, entries.size()); + Assert.assertEquals(utc(2026, 1, 5, 10, 0), find(entries, "admin.conf").getExpires()); + CertificateExpiry missing = find(entries, "apiserver-etcd-client"); + Assert.assertTrue(missing.isMissing()); + Assert.assertFalse(missing.isAuthority()); + Assert.assertTrue(find(entries, "front-proxy-client").isExternallyManaged()); + Assert.assertTrue(find(entries, "etcd-ca").isMissing()); + Assert.assertTrue(find(entries, "etcd-ca").isAuthority()); + } + + @Test + public void parseCheckExpirationOutputOfAFailureIsEmpty() { + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput(null).isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput("").isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput("sudo: /opt/bin/kubeadm: command not found").isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput( + "error execution phase check-expiration: unknown command \"certs\" for \"kubeadm\"").isEmpty()); + } + + @Test + public void reportPicksTheFirstExpiryOfEachKind() { + CertificateExpiryReport report = new CertificateExpiryReport(); + report.add("control-1", KubernetesClusterCertificateUtil.parseCheckExpirationOutput(CHECK_EXPIRATION_OUTPUT)); + report.add("control-2", Arrays.asList( + new CertificateExpiry("apiserver", utc(2027, 2, 1, 0, 0), false, false), + new CertificateExpiry("front-proxy-client", utc(2026, 12, 1, 0, 0), false, true), + new CertificateExpiry("apiserver-etcd-client", null, false, false))); + report.addUncheckedNode("control-3"); + + Assert.assertFalse(report.isEmpty()); + Assert.assertEquals(Collections.singletonList("control-3"), report.getUncheckedNodes()); + Assert.assertEquals("front-proxy-client on control-2", report.getFirstLeafExpiry().getName()); + Assert.assertEquals("apiserver on control-2", report.getFirstRenewableLeafExpiry().getName()); + Assert.assertEquals(utc(2027, 2, 1, 0, 0), report.getFirstRenewableLeafExpiry().getExpires()); + Assert.assertEquals("front-proxy-ca on control-1", report.getFirstAuthorityExpiry().getName()); + } + + @Test + public void emptyReportHasNoExpiry() { + CertificateExpiryReport report = new CertificateExpiryReport(); + Assert.assertTrue(report.isEmpty()); + Assert.assertNull(report.getFirstLeafExpiry()); + Assert.assertNull(report.getFirstRenewableLeafExpiry()); + Assert.assertNull(report.getFirstAuthorityExpiry()); + } + + @Test + public void kubeConfigClientCertificateExpiry() { + Assert.assertEquals(CLIENT_CERTIFICATE_EXPIRY, KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(kubeConfig(CLIENT_CERTIFICATE_DATA))); + // what ROOT_KUBECONFIG_CERTIFICATE_COMMAND prints + Assert.assertEquals(CLIENT_CERTIFICATE_EXPIRY, KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry( + " client-certificate-data: " + CLIENT_CERTIFICATE_DATA + "\n")); + } + + @Test + public void kubeConfigWithoutUsableClientCertificate() { + Assert.assertNull(KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(null)); + Assert.assertNull(KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry("users:\n- name: x\n user:\n token: abc\n")); + Assert.assertNull(KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(kubeConfig("bm90IGEgY2VydGlmaWNhdGU="))); + } + + @Test + public void parseAlertThresholds() { + Assert.assertEquals(Arrays.asList(60, 30), KubernetesClusterCertificateUtil.parseAlertThresholds("60,30")); + Assert.assertEquals(Arrays.asList(90, 30, 7), KubernetesClusterCertificateUtil.parseAlertThresholds(" 7, 30 ,90,30")); + Assert.assertEquals(Collections.singletonList(14), KubernetesClusterCertificateUtil.parseAlertThresholds("abc,0,-5,14,")); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseAlertThresholds("").isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseAlertThresholds(null).isEmpty()); + } + + @Test + public void thresholdToAlert() { + List thresholds = Arrays.asList(60, 30); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(61, thresholds, null)); + Assert.assertEquals(Integer.valueOf(60), KubernetesClusterCertificateUtil.getThresholdToAlert(60, thresholds, null)); + // already alerted for 60: nothing until 30 is reached + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(45, thresholds, 60)); + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getThresholdToAlert(30, thresholds, 60)); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(10, thresholds, 30)); + // first seen inside both windows: one alert, for the smaller threshold + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getThresholdToAlert(12, thresholds, null)); + // expiry itself alerts once more, after the thresholds + Assert.assertEquals(Integer.valueOf(0), KubernetesClusterCertificateUtil.getThresholdToAlert(-3, thresholds, null)); + Assert.assertEquals(Integer.valueOf(0), KubernetesClusterCertificateUtil.getThresholdToAlert(-3, thresholds, 30)); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(-10, thresholds, 0)); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(5, Collections.emptyList(), null)); + } + + @Test + public void alertStateIsTiedToTheExpiryDate() { + Date expiry = utc(2027, 1, 2, 0, 0); + String state = KubernetesClusterCertificateUtil.buildAlertState(expiry, 30); + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, expiry)); + // renewed: the expiry moved, so the thresholds start over + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, utc(2028, 1, 2, 0, 0))); + // another control node's copy, renewed minutes apart, is the same expiry + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, utc(2027, 1, 2, 0, 7))); + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, utc(2026, 12, 30, 0, 0))); + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold(null, expiry)); + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold("garbage", expiry)); + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold("1:x", expiry)); + } + + @Test + public void daysUntilAndAutoRenew() { + Date now = utc(2026, 11, 9, 12, 0); + Assert.assertEquals(30, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 12, 9, 12, 0), now)); + Assert.assertEquals(29, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 12, 9, 11, 59), now)); + Assert.assertEquals(0, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 11, 9, 13, 0), now)); + Assert.assertEquals(-1, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 11, 9, 11, 0), now)); + + Assert.assertTrue(KubernetesClusterCertificateUtil.isAutoRenewDue(30, 30)); + Assert.assertTrue(KubernetesClusterCertificateUtil.isAutoRenewDue(-2, 30)); + Assert.assertFalse(KubernetesClusterCertificateUtil.isAutoRenewDue(31, 30)); + Assert.assertFalse(KubernetesClusterCertificateUtil.isAutoRenewDue(5, 0)); + // capped, so a freshly renewed certificate (about 365 days) is never due again right away + Assert.assertFalse(KubernetesClusterCertificateUtil.isAutoRenewDue(364, 400)); + Assert.assertTrue(KubernetesClusterCertificateUtil.isAutoRenewDue(180, 400)); + } +} From f93fff68f39ae03ee7b9c1f27ed333b7b42bafe0 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 11 Nov 2026 14:45:08 +0000 Subject: [PATCH 2/2] Mark rotate-kubernetes-certificates.sh executable It has a shebang, and pre-commit's check-shebang-scripts-are-executable rejects it otherwise. Signed-off-by: Brad House --- .../src/main/resources/script/rotate-kubernetes-certificates.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh old mode 100644 new mode 100755