diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 80f6a6045c72..a316f842b3c5 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -47,6 +47,7 @@ enum Event { AddNodeRequested, RemoveNodeRequested, UpgradeRequested, + CertificateRotationRequested, OperationSucceeded, OperationFailed, CreateFailed, @@ -62,6 +63,7 @@ enum State { Scaling("Transient state in which resources are either getting scaled up/down"), ScalingStoppedCluster("Transient state in which the service offerings of stopped clusters are getting scaled"), Upgrading("Transient state in which cluster is getting upgraded"), + RotatingCertificates("Transient state in which the control plane certificates of the cluster are getting renewed"), Importing("Transient state in which additional nodes are added as worker nodes to a cluster"), RemovingNodes("Transient state in which additional nodes are removed from a cluster"), Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), @@ -106,6 +108,12 @@ enum State { s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.CertificateRotationRequested, State.RotatingCertificates); + // lets a failed or interrupted rotation be retried, and rescues a cluster whose certificates already expired + s_fsm.addTransition(State.Alert, Event.CertificateRotationRequested, State.RotatingCertificates); + s_fsm.addTransition(State.RotatingCertificates, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.RotatingCertificates, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing); s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing); s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running); diff --git a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java index a9c2abc11ce7..b4ec59d38012 100644 --- a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java +++ b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java @@ -86,6 +86,7 @@ private AlertType(short type, String name, boolean isDefault) { public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_FAILED = new AlertType((short)35, "ALERT.BACKUP.VALIDATION.FAILED", true, true); public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_UNABLE_TO_VALIDATE = new AlertType((short)36, "ALERT.BACKUP.VALIDATION.UNABLE.TO.VALIDATE", true, true); public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_CLEANUP_FAILED = new AlertType((short)37, "ALERT.BACKUP.VALIDATION.CLEANUP_FAILED", true, true); + public static final AlertType ALERT_TYPE_KUBERNETES_CLUSTER_CERTIFICATE = new AlertType((short)38, "ALERT.KUBERNETES.CLUSTER.CERTIFICATE", true, true); public short getType() { return type; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index d11a0fc3dea5..54d49d43059b 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -26,4 +26,6 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE"; + public static final String EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE = "KUBERNETES.CLUSTER.CERTIFICATES.ROTATE"; + public static final String EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY = "KUBERNETES.CLUSTER.CERTIFICATES.EXPIRY"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index eb4ebb78ee48..23bc02d70956 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -39,8 +39,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -86,6 +89,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; @@ -124,6 +128,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; +import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; @@ -152,6 +157,7 @@ import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.deploy.DeployDestination; import com.cloud.event.ActionEvent; +import com.cloud.event.EventVO; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; @@ -167,12 +173,16 @@ import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterCertificateWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterDestroyWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; @@ -312,8 +322,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER.name(), CONTROL.name(), ETCD.name()); + protected static final long CERTIFICATE_SCANNER_INTERVAL_SECONDS = 15 * 60; + protected static final int CERTIFICATE_ROTATION_THREADS = 4; + /** Automatic renewal is not retried within this long of a failed rotation; the failure already raised an alert. */ + protected static final long CERTIFICATE_AUTO_RENEW_RETRY_BACKOFF_MS = TimeUnit.HOURS.toMillis(24); + ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; + ScheduledExecutorService _certificateScanner; + ExecutorService _certificateRotationExecutor; + private final Set queuedCertificateRotations = ConcurrentHashMap.newKeySet(); @Inject public KubernetesClusterDao kubernetesClusterDao; @@ -1078,6 +1096,7 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) { BaseCmd.getCommandNameByClass(StartKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(RotateKubernetesClusterCertificatesCmd.class), BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) ).contains(cmdName); @@ -1955,6 +1974,9 @@ public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, S logger.debug("Kubernetes cluster {} is already in starting state", kubernetesCluster); return true; } + if (KubernetesCluster.State.RotatingCertificates.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is renewing its certificates", kubernetesCluster.getName())); + } final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (zone == null) { logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster %s", kubernetesCluster)); @@ -2113,6 +2135,11 @@ public boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudR } return true; } + // the rotation has the control plane manifests moved aside while it restarts the static pods + if (KubernetesCluster.State.RotatingCertificates.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is renewing its certificates, stop it once that is done", + kubernetesCluster.getName())); + } KubernetesClusterStopWorker stopWorker = new KubernetesClusterStopWorker(kubernetesCluster, this); stopWorker = ComponentContext.inject(stopWorker); return stopWorker.stop(); @@ -2260,7 +2287,7 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC response.setId(kubernetesCluster.getUuid()); response.setName(kubernetesCluster.getName()); String configData = ""; - KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KubernetesClusterActionWorker.KUBE_CONFIG_DATA_DETAIL); if (clusterDetailsVO != null && StringUtils.isNotEmpty(clusterDetailsVO.getValue())) { configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); } else { @@ -2341,6 +2368,43 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws return upgradeWorker.upgradeCluster(); } + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE, + eventDescription = "rotating Kubernetes cluster certificates", async = true) + public boolean rotateKubernetesClusterCertificates(RotateKubernetesClusterCertificatesCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO kubernetesCluster = validateKubernetesClusterCertificateRotationParameters(cmd); + return createCertificateWorker(kubernetesCluster).rotateCertificates(); + } + + protected KubernetesClusterVO validateKubernetesClusterCertificateRotationParameters(RotateKubernetesClusterCertificatesCmd cmd) { + final Long kubernetesClusterId = cmd.getId(); + KubernetesClusterVO kubernetesCluster = kubernetesClusterId == null ? null : kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (!isCommandSupported(kubernetesCluster, cmd.getActualCommandName())) { + throw new InvalidParameterValueException(String.format("Certificate rotation is not supported for an externally managed Kubernetes cluster (%s)", kubernetesCluster.getName())); + } + // Alert lets a failed rotation be retried and a cluster whose certificates already expired be rescued + if (!Arrays.asList(KubernetesCluster.State.Running, KubernetesCluster.State.Alert).contains(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is not in Running or Alert state", kubernetesCluster.getName())); + } + KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (!KubernetesClusterCertificateWorker.isCertificateManagementSupported(clusterVersion)) { + throw new InvalidParameterValueException(String.format("Certificate rotation of Kubernetes cluster %s needs Kubernetes %s or later", + kubernetesCluster.getName(), KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT)); + } + return kubernetesCluster; + } + + protected KubernetesClusterCertificateWorker createCertificateWorker(KubernetesCluster kubernetesCluster) { + return ComponentContext.inject(new KubernetesClusterCertificateWorker(kubernetesCluster, this)); + } + @Override @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE, eventDescription = "updating Kubernetes cluster affinity groups") @@ -2780,6 +2844,7 @@ public List> getCommands() { cmdList.add(GetKubernetesClusterConfigCmd.class); cmdList.add(ScaleKubernetesClusterCmd.class); cmdList.add(UpgradeKubernetesClusterCmd.class); + cmdList.add(RotateKubernetesClusterCertificatesCmd.class); cmdList.add(AddVirtualMachinesToKubernetesClusterCmd.class); cmdList.add(RemoveVirtualMachinesFromKubernetesClusterCmd.class); cmdList.add(AddNodesToKubernetesClusterCmd.class); @@ -2909,8 +2974,10 @@ public void reallyRun() { startWorker = ComponentContext.inject(startWorker); startWorker.reconcileAlertCluster(); } else if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + // unless another operation (e.g. a certificate rotation) took the cluster meanwhile + if (stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } } } catch (Exception e) { logger.warn("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster: {} status scanner", kubernetesCluster, e); @@ -2957,6 +3024,295 @@ public void reallyRun() { } } + /* Checks the control plane certificate expiry of running clusters every few hours, raises alerts and events as + configured thresholds are reached and renews the certificates automatically shortly before they expire. Runs on + its own thread, as a renewal restarts the control plane one node at a time and can take minutes. + */ + public class KubernetesClusterCertificateScanner extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock lock = GlobalLock.getInternLock("KubernetesCluster.Certificate.Scanner.Lock"); + try { + if (lock.lock(3)) { + try { + reallyRun(); + } finally { + lock.unlock(); + } + } + } finally { + lock.releaseRef(); + } + } + + public void reallyRun() { + if (!KubernetesServiceEnabled.value()) { + return; + } + try { + recoverInterruptedCertificateRotations(new Date()); + // Alert too: a cluster can sit in Alert for long (e.g. a stopped worker) and its certificates still expire + List clusters = new ArrayList<>(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Running)); + clusters.addAll(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Alert)); + for (KubernetesClusterVO listed : clusters) { + // a pass can take minutes; skip a cluster stopped or otherwise taken since it was listed + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(listed.getId()); + if (kubernetesCluster == null || !Arrays.asList(KubernetesCluster.State.Running, KubernetesCluster.State.Alert).contains(kubernetesCluster.getState())) { + continue; + } + try { + processKubernetesClusterCertificates(kubernetesCluster, new Date()); + } catch (Exception e) { + logger.warn("Failed to process the certificates of Kubernetes cluster: {}", kubernetesCluster, e); + } + } + } catch (Exception e) { + logger.warn("Caught exception while running the Kubernetes cluster certificate scanner", e); + } + } + } + + private Long getLongDetail(long kubernetesClusterId, String name) { + KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesClusterId, name); + if (detail == null || StringUtils.isBlank(detail.getValue())) { + return null; + } + try { + return Long.parseLong(detail.getValue().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + /** + * A cluster stays RotatingCertificates if the management server running the rotation went away. Move it to Alert, + * so the status scanner can reconcile it and the operator is told: right away when the rotation belonged to this + * management server and is no longer running here (it restarted), otherwise once the rotation is well past its + * timeout. + */ + protected void recoverInterruptedCertificateRotations(Date now) { + long staleAfterMs = 2 * KubernetesClusterCertificateRotationTimeout.value() * 1000L + TimeUnit.MINUTES.toMillis(10); + for (KubernetesClusterVO kubernetesCluster : kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) { + if (KubernetesClusterCertificateWorker.isRotationActive(kubernetesCluster.getId())) { + continue; + } + Long started = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + if (started == null) { + // a rotation writes its start just after entering the state; never write that detail here, it would + // race with it. Note when it was first seen instead, to time a rotation that died in between. + started = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL); + } + if (started == null) { + // the rotation may have only just entered the state, or just finished; time it from now if still running + KubernetesClusterVO current = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (current == null || !KubernetesCluster.State.RotatingCertificates.equals(current.getState()) + || KubernetesClusterCertificateWorker.isRotationActive(kubernetesCluster.getId())) { + continue; + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL, String.valueOf(now.getTime()), false); + continue; + } + Long owner = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + boolean ownedHere = owner != null && owner == ManagementServerNode.getManagementServerId(); + if (!ownedHere && now.getTime() - started < staleAfterMs) { + continue; + } + logger.warn("Certificate rotation of Kubernetes cluster {} did not finish, moving it to Alert", kubernetesCluster); + if (stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed)) { + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(now.getTime()), false); + createCertificateWorker(kubernetesCluster).sendAlert( + String.format("Certificate rotation of Kubernetes cluster %s did not finish", kubernetesCluster.getName()), + String.format("The certificate rotation of Kubernetes cluster %s was interrupted, possibly by a management server restart. " + + "Check the cluster and run rotateKubernetesClusterCertificates again", kubernetesCluster.getName())); + } + } + } + + protected void processKubernetesClusterCertificates(KubernetesCluster kubernetesCluster, Date now) { + Integer checkIntervalHours = KubernetesClusterCertificateCheckInterval.value(); + if (checkIntervalHours == null || checkIntervalHours <= 0) { + return; + } + Long lastCheck = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_DETAIL); + if (lastCheck != null && now.getTime() - lastCheck < TimeUnit.HOURS.toMillis(checkIntervalHours)) { + return; + } + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (!KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)) { + logger.debug("Skipping the certificate expiry check of Kubernetes cluster {}, its Kubernetes version is older than {}", + kubernetesCluster, KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT); + return; + } + // Recorded before checking, so an unreachable cluster is retried at the next interval, not on every pass + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_DETAIL, String.valueOf(now.getTime()), false); + + KubernetesClusterCertificateWorker worker = createCertificateWorker(kubernetesCluster); + CertificateExpiryReport report; + try { + report = worker.checkCertificateExpiry(); + } catch (CloudRuntimeException e) { + // e.g. no address to reach the cluster on: as unwatched as when every node fails + logger.warn("Unable to check the certificate expiry of Kubernetes cluster {}", kubernetesCluster, e); + report = new CertificateExpiryReport(); + report.addUncheckedNode("all control nodes"); + } + boolean allNodesChecked = report.getUncheckedNodes().isEmpty(); + if (!allNodesChecked) { + logger.warn("Unable to check the certificate expiry on control nodes {} of Kubernetes cluster {}", report.getUncheckedNodes(), kubernetesCluster); + } + if (report.isEmpty()) { + alertOnFailedCertificateCheck(kubernetesCluster, worker, report, now); + return; + } + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL); + CertificateExpiry firstLeaf = report.getFirstLeafExpiry(); + CertificateExpiry firstAuthority = report.getFirstAuthorityExpiry(); + logger.info("Kubernetes cluster {}: first control plane certificate to expire is {}, first certificate authority is {}", + kubernetesCluster, firstLeaf, firstAuthority); + + int autoRenewDays = Math.min(Objects.requireNonNullElse(KubernetesClusterCertificateAutoRenewDays.valueIn(kubernetesCluster.getAccountId()), 0), + KubernetesClusterCertificateUtil.MAX_AUTO_RENEW_DAYS); + boolean autoRenew = autoRenewDays > 0 && Boolean.TRUE.equals(KubernetesClusterCertificateAutoRenewEnabled.valueIn(kubernetesCluster.getAccountId())); + List thresholds = KubernetesClusterCertificateUtil.parseAlertThresholds(KubernetesClusterCertificateExpiryAlertDays.value()); + String leafAdvice; + if (firstLeaf != null && firstLeaf.isExternallyManaged()) { + leafAdvice = "It is externally managed, so it has to be renewed outside CloudStack."; + } else if (autoRenew && !allNodesChecked) { + leafAdvice = String.format("Automatic renewal is on hold until control nodes %s can be checked; fix their access or run rotateKubernetesClusterCertificates.", + report.getUncheckedNodes()); + } else if (autoRenew) { + leafAdvice = String.format("They are renewed automatically %d days before they expire while the cluster is Running, or run rotateKubernetesClusterCertificates.", + autoRenewDays); + } else { + leafAdvice = "Run rotateKubernetesClusterCertificates to renew them."; + } + alertOnCertificateExpiry(kubernetesCluster, worker, firstLeaf, thresholds, KubernetesClusterCertificateWorker.CERTIFICATE_ALERT_DETAIL, now, + "control plane certificate", leafAdvice); + alertOnCertificateExpiry(kubernetesCluster, worker, firstAuthority, thresholds, KubernetesClusterCertificateWorker.CA_CERTIFICATE_ALERT_DETAIL, now, + "certificate authority", "CloudStack does not rotate certificate authorities; it has to be done by hand."); + + // Externally managed certificates can't be renewed by kubeadm and would make every check renew again + CertificateExpiry firstRenewable = report.getFirstRenewableLeafExpiry(); + // a cluster in Alert may have nodes down; renewing it is left to the operator + if (firstRenewable == null || !autoRenew || !KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) || + !KubernetesClusterCertificateUtil.isAutoRenewDue(KubernetesClusterCertificateUtil.daysUntil(firstRenewable.getExpires(), now), autoRenewDays)) { + return; + } + Long lastFailure = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL); + if (!allNodesChecked) { + // the rotation would renew the reachable nodes and then fail on the others, leaving the cluster in Alert + logger.warn("Not renewing the certificates of Kubernetes cluster {} automatically while control nodes {} can't be reached", + kubernetesCluster, report.getUncheckedNodes()); + return; + } + if (lastFailure != null && now.getTime() - lastFailure < CERTIFICATE_AUTO_RENEW_RETRY_BACKOFF_MS) { + logger.info("Not renewing the certificates of Kubernetes cluster {} automatically, the last rotation failed less than {} hours ago", + kubernetesCluster, TimeUnit.MILLISECONDS.toHours(CERTIFICATE_AUTO_RENEW_RETRY_BACKOFF_MS)); + return; + } + logger.info("Renewing the certificates of Kubernetes cluster {} automatically, {} expires on {}", kubernetesCluster, firstRenewable.getName(), firstRenewable.getExpires()); + scheduleAutomaticRotation(kubernetesCluster); + } + + /** + * Runs the rotation on its own thread, so a renewal that takes minutes holds up neither the expiry checks of the + * other clusters nor the scanner lock. The state transition keeps a second rotation of the same cluster out. + */ + protected void scheduleAutomaticRotation(KubernetesCluster kubernetesCluster) { + if (_certificateRotationExecutor == null) { + runAutomaticRotation(kubernetesCluster); + return; + } + // a renewal still waiting for a thread must not be queued again by a later scan + if (!queuedCertificateRotations.add(kubernetesCluster.getId())) { + return; + } + final long queuedAt = System.currentTimeMillis(); + try { + _certificateRotationExecutor.execute(new ManagedContextRunnable() { + @Override + protected void runInContext() { + try { + runQueuedAutomaticRotation(kubernetesCluster.getId(), queuedAt); + } finally { + queuedCertificateRotations.remove(kubernetesCluster.getId()); + } + } + }); + } catch (RejectedExecutionException e) { + queuedCertificateRotations.remove(kubernetesCluster.getId()); + logger.warn("Unable to queue the automatic certificate renewal of Kubernetes cluster {}", kubernetesCluster, e); + } + } + + /** The cluster may have changed while the renewal waited for a thread, e.g. been renewed through the API. */ + protected void runQueuedAutomaticRotation(long kubernetesClusterId, long queuedAt) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || !KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + logger.info("Skipping the queued automatic certificate renewal of Kubernetes cluster {}, it is no longer Running", kubernetesClusterId); + return; + } + Long lastSuccess = getLongDetail(kubernetesClusterId, KubernetesClusterCertificateWorker.ROTATION_SUCCEEDED_DETAIL); + Long lastFailure = getLongDetail(kubernetesClusterId, KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL); + if ((lastSuccess != null && lastSuccess >= queuedAt) || (lastFailure != null && lastFailure >= queuedAt)) { + logger.info("Skipping the queued automatic certificate renewal of Kubernetes cluster {}, it was rotated meanwhile", kubernetesCluster); + return; + } + runAutomaticRotation(kubernetesCluster); + } + + protected void runAutomaticRotation(KubernetesCluster kubernetesCluster) { + KubernetesClusterCertificateWorker rotationWorker = createCertificateWorker(kubernetesCluster); + rotationWorker.setAutomatic(true); + try { + rotationWorker.rotateCertificates(); + } catch (Exception e) { + logger.warn("Automatic certificate renewal of Kubernetes cluster {} failed", kubernetesCluster, e); + } + } + + /** Alerts, at most once a day, when the expiry of no control node could be checked, so expiry can't go unnoticed */ + protected void alertOnFailedCertificateCheck(KubernetesCluster kubernetesCluster, KubernetesClusterCertificateWorker worker, + CertificateExpiryReport report, Date now) { + if (report.getUncheckedNodes().isEmpty()) { + return; + } + Long lastAlert = getLongDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL); + if (lastAlert != null && now.getTime() - lastAlert < TimeUnit.DAYS.toMillis(1)) { + return; + } + String subject = String.format("Unable to check the certificate expiry of Kubernetes cluster %s", kubernetesCluster.getName()); + worker.sendAlert(subject, String.format("%s: none of its control nodes %s could be checked over SSH. Its certificates are not watched " + + "or renewed automatically until they can.", subject, report.getUncheckedNodes())); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL, + String.valueOf(now.getTime()), false); + } + + protected void alertOnCertificateExpiry(KubernetesCluster kubernetesCluster, KubernetesClusterCertificateWorker worker, CertificateExpiry expiry, + List thresholds, String alertDetail, Date now, String what, String advice) { + if (expiry == null) { + return; + } + long daysLeft = KubernetesClusterCertificateUtil.daysUntil(expiry.getExpires(), now); + KubernetesClusterDetailsVO alertState = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), alertDetail); + Integer lastAlerted = KubernetesClusterCertificateUtil.getLastAlertedThreshold(alertState == null ? null : alertState.getValue(), expiry.getExpires()); + Integer threshold = KubernetesClusterCertificateUtil.getThresholdToAlert(daysLeft, thresholds, lastAlerted); + if (threshold == null) { + return; + } + String subject = daysLeft < 0 ? + String.format("A %s of Kubernetes cluster %s has expired", what, kubernetesCluster.getName()) : + String.format("A %s of Kubernetes cluster %s expires in %d days", what, kubernetesCluster.getName(), daysLeft); + String body = String.format("%s: %s expires on %s. %s", subject, expiry.getName(), expiry.getExpires(), advice); + worker.sendAlert(subject, body); + worker.recordEvent(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY, EventVO.LEVEL_WARN, body); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), alertDetail, + KubernetesClusterCertificateUtil.buildAlertState(expiry.getExpires(), threshold), false); + } + // checks if Kubernetes cluster is in desired state private boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); @@ -3003,10 +3359,29 @@ public boolean start() { _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); + // Separate from the status scanner: SSH checks of many clusters take a while and must not hold up its 30s passes + _certificateScanner.scheduleWithFixedDelay(new KubernetesClusterCertificateScanner(), 600, CERTIFICATE_SCANNER_INTERVAL_SECONDS, TimeUnit.SECONDS); return true; } + @Override + public boolean stop() { + // running rotations are left to finish; one cut short is recovered by the scanner after a restart + if (_certificateScanner != null) { + _certificateScanner.shutdown(); + } + if (_certificateRotationExecutor != null) { + // renewals still waiting for a thread must not start now; they are scheduled again after the restart + if (_certificateRotationExecutor instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) _certificateRotationExecutor).getQueue().clear(); + } + queuedCertificateRotations.clear(); + _certificateRotationExecutor.shutdown(); + } + return super.stop(); + } + private void createNetworkOfferingForKubernetes(String offeringName, String offeringDesc, Network.Provider extNetProvider, boolean forVpc) { final Map defaultKubernetesServiceNetworkOfferingProviders = new HashMap(); Network.Provider provider = forVpc ? Network.Provider.VPCVirtualRouter : Network.Provider.VirtualRouter; @@ -3063,6 +3438,8 @@ public boolean configure(String name, Map params) throws Configu _configParams = params; _gcExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Scavenger")); _stateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-State-Scanner")); + _certificateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Certificate-Scanner")); + _certificateRotationExecutor = Executors.newFixedThreadPool(CERTIFICATE_ROTATION_THREADS, new NamedThreadFactory("Kubernetes-Cluster-Certificate-Rotation")); return true; } @@ -3089,7 +3466,12 @@ public ConfigKey[] getConfigKeys() { KubernetesControlNodeInstallReattempts, KubernetesWorkerNodeInstallAttemptWait, KubernetesWorkerNodeInstallReattempts, - KubernetesEtcdNodeStartPort + KubernetesEtcdNodeStartPort, + KubernetesClusterCertificateCheckInterval, + KubernetesClusterCertificateExpiryAlertDays, + KubernetesClusterCertificateAutoRenewEnabled, + KubernetesClusterCertificateAutoRenewDays, + KubernetesClusterCertificateRotationTimeout }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 1d19127dc54a..9953ae194fa0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; @@ -143,6 +144,43 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Start port for Port forwarding rules for etcd nodes", true, KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateCheckInterval = new ConfigKey<>("Advanced", Integer.class, + "cloud.kubernetes.cluster.certificate.check.interval", + "6", + "Interval (in hours) at which the control plane certificate expiry of each running CloudManaged Kubernetes cluster " + + "is checked with 'kubeadm certs check-expiration'. 0 disables the check and the automatic renewal", + true, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateExpiryAlertDays = new ConfigKey<>("Advanced", String.class, + "cloud.kubernetes.cluster.certificate.expiry.alert.days", + "60,30", + "Comma-separated list of days before a Kubernetes cluster control plane certificate (or certificate authority) " + + "expires at which an alert and an event are raised. Each threshold alerts once per expiry date", + true, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateAutoRenewEnabled = new ConfigKey<>("Advanced", Boolean.class, + "cloud.kubernetes.cluster.certificate.auto.renew.enabled", + "true", + "Indicates whether the control plane certificates of a CloudManaged Kubernetes cluster are renewed automatically, " + + "one control node at a time, when they are about to expire. Requires Kubernetes 1.20 or later", + true, + ConfigKey.Scope.Account, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateAutoRenewDays = new ConfigKey<>("Advanced", Integer.class, + "cloud.kubernetes.cluster.certificate.auto.renew.days", + "30", + "Number of days before the first control plane certificate of a Kubernetes cluster expires at which the " + + "certificates are renewed automatically, when cloud.kubernetes.cluster.certificate.auto.renew.enabled is true. " + + "Values above 180 are treated as 180", + true, + ConfigKey.Scope.Account, + KubernetesServiceEnabled.key()); + ConfigKey KubernetesClusterCertificateRotationTimeout = new ConfigKey<>("Advanced", Long.class, + "cloud.kubernetes.cluster.certificate.rotation.timeout", + "3600", + "Timeout interval (in seconds) in which the certificate rotation of a Kubernetes cluster should be completed", + true, + KubernetesServiceEnabled.key()); KubernetesCluster findById(final Long id); @@ -172,6 +210,8 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException; + boolean rotateKubernetesClusterCertificates(RotateKubernetesClusterCertificatesCmd cmd) throws CloudRuntimeException; + boolean updateKubernetesClusterAffinityGroups(UpdateKubernetesClusterAffinityGroupCmd cmd) throws CloudRuntimeException; boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..6d521fc05d1e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -158,6 +158,8 @@ public class KubernetesClusterActionWorker { public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks"; + public static final String KUBE_CONFIG_DATA_DETAIL = "kubeConfigData"; + protected static final long KUBE_CONFIG_REFRESH_TIMEOUT_MS = 2 * 60 * 1000L; protected Logger logger = LogManager.getLogger(getClass()); @@ -756,6 +758,83 @@ protected List getKubernetesClusterVMs() { return vmList; } + /** + * @return the control node VMs of the cluster, ordered by VM ID, which is the order they were created and + * received their SSH port forwarding rules in + */ + protected List getControlNodeVMs() { + List controlVms = new ArrayList<>(); + List vmMaps = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), CONTROL); + List controlVmIds; + if (CollectionUtils.isNotEmpty(vmMaps)) { + controlVmIds = vmMaps.stream().map(KubernetesClusterVmMapVO::getVmId).sorted().collect(Collectors.toList()); + } else { + // clusters created before the control node flag was recorded: by position, after any etcd nodes + List allVmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(allVmMaps)) { + return controlVms; + } + long etcdNodes = Objects.requireNonNullElse(kubernetesCluster.getEtcdNodeCount(), 0L); + controlVmIds = allVmMaps.stream().map(KubernetesClusterVmMapVO::getVmId).sorted() + .skip(etcdNodes).limit(Math.max(1, kubernetesCluster.getControlNodeCount())).collect(Collectors.toList()); + } + for (Long vmId : controlVmIds) { + UserVm vm = userVmDao.findById(vmId); + if (vm != null) { + controlVms.add(vm); + } + } + return controlVms; + } + + /** + * Returns the address and port to SSH to one control node. {@link #publicIpAddress} and {@link #sshPort} must be + * set. On directly reachable (shared or routed) networks it is the VM's own address. Otherwise it is the cluster's + * public IP with the port of the SSH port forwarding rule of the VM, falling back to the port assigned by position + * (control nodes are the first nodes behind {@link #CLUSTER_NODES_DEFAULT_START_SSH_PORT}) when there is no rule, + * e.g. behind an external load balancer. + */ + protected Pair getControlNodeSshEndpoint(UserVm vm, int controlNodeIndex) { + if (sshPort == DEFAULT_SSH_PORT) { + return new Pair<>(vm.getPrivateIpAddress(), DEFAULT_SSH_PORT); + } + List rules = portForwardingRulesDao.listByVm(vm.getId()); + if (CollectionUtils.isNotEmpty(rules)) { + for (PortForwardingRuleVO rule : rules) { + if (rule.getDestinationPortStart() != DEFAULT_SSH_PORT || rule.getState() == FirewallRule.State.Revoke) { + continue; + } + // only the cluster's own rule, not one a user added on another public IP + IPAddressVO ruleIp = ipAddressDao.findById(rule.getSourceIpAddressId()); + if (ruleIp != null && publicIpAddress.equals(ruleIp.getAddress().addr())) { + return new Pair<>(publicIpAddress, rule.getSourcePortStart()); + } + } + } + return new Pair<>(publicIpAddress, sshPort + controlNodeIndex); + } + + /** + * Reads the kubeconfig from the control node reachable at {@link #publicIpAddress}:{@link #sshPort}, points it at + * {@link #apiPublicIpAddress} and stores it as the cluster's kubeConfigData, which getKubernetesClusterConfig + * returns. Called after every operation that can issue a new admin certificate (start, upgrade, certificate + * rotation), as the stored copy otherwise keeps the old certificate until it expires. + * @return true if a kubeconfig was read and stored + */ + protected boolean refreshKubeConfigData(final long timeoutTime) { + String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), timeoutTime); + if (StringUtils.isEmpty(kubeConfig)) { + return false; + } + final String controlVMPrivateIpAddress = getControlVmPrivateIp(); + if (StringUtils.isNotEmpty(controlVMPrivateIpAddress) && StringUtils.isNotEmpty(apiPublicIpAddress)) { + kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", controlVMPrivateIpAddress, CLUSTER_API_PORT), + String.format("server: https://%s:%d", apiPublicIpAddress, CLUSTER_API_PORT)); + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL, Base64.encodeBase64String(kubeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())), false); + return true; + } + protected void updateLoginUserDetails(List clusterVMs) { if (clusterVMs == null) { clusterVMs = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java new file mode 100644 index 000000000000..e549922e72e1 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorker.java @@ -0,0 +1,476 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.File; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import javax.inject.Inject; + +import org.apache.cloudstack.alert.AlertService; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; + +import com.cloud.alert.AlertManager; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventVO; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; +import com.cloud.user.User; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; + +/** + * Checks the expiry of, and renews, the kubeadm-managed control plane certificates of a CKS cluster. + *

+ * Renewal runs {@code kubeadm certs renew all} on one control node at a time, restarts that node's control plane + * static pods so they load the new certificates and waits for its API server and the node to be ready before moving + * to the next one. It stops at the first control node that fails. The certificate authorities are not rotated. + *

+ * External etcd nodes created by CKS serve etcd over plain HTTP and hold no certificates, so there is nothing to + * renew on them; only control nodes are touched. + */ +public class KubernetesClusterCertificateWorker extends KubernetesClusterActionWorker { + + public static final String ROTATION_SCRIPT_FILENAME = "rotate-kubernetes-certificates.sh"; + + /** Epoch millis at which the expiry was last checked. */ + public static final String CERTIFICATE_CHECK_DETAIL = "certificateExpiryLastCheck"; + /** Epoch millis of the last alert that no control node could be checked; removed once a check works again. */ + public static final String CERTIFICATE_CHECK_FAILED_ALERT_DETAIL = "certificateExpiryCheckFailedAlert"; + /** Alert state of the first leaf certificate to expire, see {@link KubernetesClusterCertificateUtil#buildAlertState}. */ + public static final String CERTIFICATE_ALERT_DETAIL = "certificateExpiryAlert"; + /** Alert state of the first certificate authority to expire. */ + public static final String CA_CERTIFICATE_ALERT_DETAIL = "caCertificateExpiryAlert"; + /** Epoch millis at which the running rotation started; removed when it ends. */ + public static final String ROTATION_STARTED_DETAIL = "certificateRotationStarted"; + /** ID of the management server running the rotation; removed when it ends. */ + public static final String ROTATION_OWNER_DETAIL = "certificateRotationOwner"; + /** Epoch millis of the last failed rotation, used to back off automatic renewal. */ + public static final String ROTATION_FAILED_DETAIL = "certificateRotationLastFailure"; + /** Epoch millis at which the scanner first saw a rotation that had not recorded its start yet. */ + public static final String ROTATION_SEEN_DETAIL = "certificateRotationFirstSeen"; + /** Epoch millis of the last successful rotation, so a queued automatic renewal doesn't renew again. */ + public static final String ROTATION_SUCCEEDED_DETAIL = "certificateRotationLastSuccess"; + + protected static final int NODE_READY_POLL_INTERVAL_MS = 5000; + private static final Set CLIENT_ONLY_KUBECONFIGS = Set.of("admin.conf", "super-admin.conf", + KubernetesClusterCertificateUtil.ROOT_KUBECONFIG_ENTRY_NAME); + private static final String CHECK_OUTPUT_SEPARATOR = "----- cks root kubeconfig -----"; + private static final String USER_KUBECONFIG_SEPARATOR = "----- cks user kubeconfig -----"; + + /** Clusters whose rotation runs in this management server. */ + private static final Set ACTIVE_ROTATIONS = ConcurrentHashMap.newKeySet(); + + @Inject + protected AlertManager alertManager; + + private File rotationScriptFile; + private long rotationTimeoutTime; + private boolean automatic = false; + private boolean rescueMode = false; + private String rotationStartedAt; + private boolean startedFromAlert = false; + + public KubernetesClusterCertificateWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + /** Marks a rotation started by the expiry check rather than by the API, which records its own event. */ + public void setAutomatic(boolean automatic) { + this.automatic = automatic; + } + + /** @return true if this management server is running a rotation of the cluster right now */ + public static boolean isRotationActive(long kubernetesClusterId) { + return ACTIVE_ROTATIONS.contains(kubernetesClusterId); + } + + /** + * Only the SSH key is needed here; the templates and owner that the other workers load in their init() are not, + * and loading them fails for a cluster whose template has since been removed. + */ + @Override + protected void init() { + this.sshKeyFile = getManagementServerSshPublicKeyFile(); + } + + public static boolean isCertificateManagementSupported(KubernetesSupportedVersion version) { + return version != null && KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), + KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT) >= 0; + } + + protected void resolveClusterEndpoints() { + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + apiPublicIpAddress = resolveApiPublicIpAddress(networkDao.findById(kubernetesCluster.getNetworkId())); + if (StringUtils.isEmpty(publicIpAddress)) { + throw new CloudRuntimeException(String.format("Unable to retrieve the SSH address of Kubernetes cluster %s", kubernetesCluster.getName())); + } + } + + protected Pair runOnNode(String address, int port, String command, int timeoutMs) throws Exception { + return SshHelper.sshExecute(address, port, getControlNodeLoginUser(), sshKeyFile, null, command, 10000, 10000, timeoutMs); + } + + ///////////////////////////////////////////////////// + //////////////// Expiry check /////////////////////// + ///////////////////////////////////////////////////// + + /** + * Runs {@code kubeadm certs check-expiration} on every control node and reads the client certificate of root's + * kubeconfig, which CKS itself uses for kubectl. Nodes that can't be checked are listed in the report. + * Also refreshes the stored kubeConfigData when its admin certificate expires before the one on the cluster, + * e.g. after the certificates were renewed by hand. + */ + public CertificateExpiryReport checkCertificateExpiry() { + init(); + resolveClusterEndpoints(); + CertificateExpiryReport report = new CertificateExpiryReport(); + List controlVms = getControlNodeVMs(); + Date firstNodeAdminExpiry = null; + boolean firstNodeUnreachable = false; + boolean userKubeConfig = false; + Date userKubeConfigExpiry = null; + for (int i = 0; i < controlVms.size(); i++) { + UserVm vm = controlVms.get(i); + Pair endpoint = getControlNodeSshEndpoint(vm, i); + try { + // one SSH session per node: kubeadm's exit status, then the certificate lines of root's kubeconfig and of user.conf + String command = String.format("%s; rc=$?; echo '%s'; %s; echo '%s'; %s; exit $rc", KubernetesClusterCertificateUtil.CHECK_EXPIRATION_COMMAND, + CHECK_OUTPUT_SEPARATOR, KubernetesClusterCertificateUtil.ROOT_KUBECONFIG_CERTIFICATE_COMMAND, + USER_KUBECONFIG_SEPARATOR, KubernetesClusterCertificateUtil.USER_KUBECONFIG_CERTIFICATE_COMMAND); + Pair result = runOnNode(endpoint.first(), endpoint.second(), command, 60000); + String output = StringUtils.defaultString(result.second()); + String kubeadmOutput = StringUtils.substringBefore(output, CHECK_OUTPUT_SEPARATOR); + List entries = KubernetesClusterCertificateUtil.parseCheckExpirationOutput(kubeadmOutput); + if (!result.first() || entries.isEmpty()) { + logger.warn("Unable to check the certificate expiry on control node {} of Kubernetes cluster {}: {}", vm, kubernetesCluster, kubeadmOutput); + report.addUncheckedNode(vm.getHostName()); + if (i == 0) { + firstNodeUnreachable = true; + } + continue; + } + String kubeConfigOutput = StringUtils.substringAfter(output, CHECK_OUTPUT_SEPARATOR); + String userKubeConfigOutput = StringUtils.substringAfter(kubeConfigOutput, USER_KUBECONFIG_SEPARATOR); + if (i == 0 && userKubeConfigOutput.contains(KubernetesClusterCertificateUtil.USER_KUBECONFIG_PRESENT)) { + userKubeConfig = true; + userKubeConfigExpiry = KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(userKubeConfigOutput); + } + Date rootKubeConfigExpiry = KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry( + StringUtils.substringBefore(kubeConfigOutput, USER_KUBECONFIG_SEPARATOR)); + if (rootKubeConfigExpiry != null) { + // a copy of admin.conf: renewable only when admin.conf is (not in external CA mode) + boolean adminConfExternallyManaged = entries.stream().anyMatch(entry -> "admin.conf".equals(entry.getName()) && entry.isExternallyManaged()); + entries.add(new CertificateExpiry(KubernetesClusterCertificateUtil.ROOT_KUBECONFIG_ENTRY_NAME, rootKubeConfigExpiry, false, + adminConfExternallyManaged)); + } + // the kubeconfig is read from the first control node, so that is the one to compare with + if (i == 0) { + for (CertificateExpiry entry : entries) { + if ("admin.conf".equals(entry.getName()) && !entry.isMissing()) { + firstNodeAdminExpiry = entry.getExpires(); + } + } + } + report.add(vm.getHostName(), entries); + } catch (Exception e) { + logger.warn("Unable to check the certificate expiry on control node {} of Kubernetes cluster {}", vm, kubernetesCluster, e); + report.addUncheckedNode(vm.getHostName()); + if (i == 0) { + firstNodeUnreachable = true; + } + } + } + // the kubeconfig is read from the first control node; don't spend the refresh timeout on it when it is down + // and from user.conf rather than admin.conf when the operator put one there + if (!firstNodeUnreachable) { + refreshStaleKubeConfigData(userKubeConfig ? userKubeConfigExpiry : firstNodeAdminExpiry); + } + return report; + } + + protected Date getStoredKubeConfigExpiry() { + KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL); + if (detail == null || StringUtils.isEmpty(detail.getValue())) { + return null; + } + return KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(new String(Base64.decodeBase64(detail.getValue()), + com.cloud.utils.StringUtils.getPreferredCharset())); + } + + protected void refreshStaleKubeConfigData(Date clusterKubeConfigExpiry) { + if (clusterKubeConfigExpiry == null) { + return; + } + Date storedExpiry = getStoredKubeConfigExpiry(); + if (storedExpiry == null || !storedExpiry.before(clusterKubeConfigExpiry)) { + return; + } + logger.info("The stored kubeconfig of Kubernetes cluster {} expires on {}, before the kubeconfig on the cluster ({}), refreshing it", + kubernetesCluster, storedExpiry, clusterKubeConfigExpiry); + if (!refreshKubeConfigData(System.currentTimeMillis() + KUBE_CONFIG_REFRESH_TIMEOUT_MS)) { + logger.warn("Unable to refresh the stored kubeconfig of Kubernetes cluster {}", kubernetesCluster); + } + } + + ///////////////////////////////////////////////////// + //////////////// Rotation /////////////////////////// + ///////////////////////////////////////////////////// + + /** + * Renews the control plane certificates, one control node at a time, then refreshes the stored kubeconfig. + * The cluster must be Running, or Alert to retry a failed rotation; it is RotatingCertificates meanwhile, which keeps every other cluster operation out. + * On failure the cluster moves to Alert, an alert is raised and a CloudRuntimeException is thrown. + */ + public boolean rotateCertificates() throws CloudRuntimeException { + init(); + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (!isCertificateManagementSupported(version)) { + throw new CloudRuntimeException(String.format("Certificate rotation of Kubernetes cluster %s needs Kubernetes %s or later", + kubernetesCluster.getName(), KubernetesClusterCertificateUtil.MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT)); + } + // Marked active before the transition, so the scanner never takes it for an interrupted rotation + if (!ACTIVE_ROTATIONS.add(kubernetesCluster.getId())) { + throw new CloudRuntimeException(String.format("The certificates of Kubernetes cluster %s are already being rotated", kubernetesCluster.getName())); + } + try { + KubernetesCluster before = kubernetesClusterDao.findById(kubernetesCluster.getId()); + startedFromAlert = before != null && KubernetesCluster.State.Alert.equals(before.getState()); + // The transition only succeeds from Running or Alert, so it also keeps out any other operation already in progress + if (!stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CertificateRotationRequested)) { + KubernetesCluster current = kubernetesClusterDao.findById(kubernetesCluster.getId()); + throw new CloudRuntimeException(String.format("Unable to rotate the certificates of Kubernetes cluster %s, it is in state %s and must be Running or Alert", + kubernetesCluster.getName(), current == null ? null : current.getState())); + } + return doRotateCertificates(); + } finally { + ACTIVE_ROTATIONS.remove(kubernetesCluster.getId()); + } + } + + private boolean doRotateCertificates() { + rotationStartedAt = String.valueOf(System.currentTimeMillis()); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_STARTED_DETAIL, rotationStartedAt, false); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_OWNER_DETAIL, String.valueOf(ManagementServerNode.getManagementServerId()), false); + logger.info("Rotating the control plane certificates of Kubernetes cluster {}", kubernetesCluster); + rotationTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterCertificateRotationTimeout.value() * 1000; + try { + resolveClusterEndpoints(); + List controlVms = getControlNodeVMs(); + if (CollectionUtils.isEmpty(controlVms)) { + failRotation(String.format("Unable to find the control nodes of Kubernetes cluster %s", kubernetesCluster.getName()), null); + } + rotationScriptFile = retrieveScriptFile(ROTATION_SCRIPT_FILENAME); + // With expired certificates, stacked etcd on a renewed node can't get quorum until the others are renewed + // too, so renew them all before waiting for the cluster. Only then: it restarts every control node at once. + rescueMode = areCertificatesExpired(controlVms); + if (rescueMode) { + logger.warn("Certificates of Kubernetes cluster {} have expired, renewing the certificates on every control node before waiting for it", + kubernetesCluster); + } + for (int i = 0; i < controlVms.size(); i++) { + UserVm vm = controlVms.get(i); + if (System.currentTimeMillis() > rotationTimeoutTime) { + failRotation(String.format("Certificate rotation of Kubernetes cluster %s timed out before control node %s", + kubernetesCluster.getName(), vm.getHostName()), null); + } + if (!renewCertificatesOnControlNode(vm, i)) { + failRotation(String.format("Failed to renew the certificates on control node %s of Kubernetes cluster %s, " + + "the remaining control nodes were left untouched", vm.getHostName(), kubernetesCluster.getName()), null); + } + if (!rescueMode && !waitForControlNodeReady(vm)) { + failRotation(String.format("Control node %s of Kubernetes cluster %s did not become ready after its certificates were renewed, " + + "the remaining control nodes were left untouched", vm.getHostName(), kubernetesCluster.getName()), null); + } + logger.info("Renewed the certificates on control node {} of Kubernetes cluster {}", vm, kubernetesCluster); + } + if (rescueMode) { + for (UserVm vm : controlVms) { + if (!waitForControlNodeReady(vm)) { + failRotation(String.format("Renewed the certificates on every control node of Kubernetes cluster %s, but control node %s did not become ready", + kubernetesCluster.getName(), vm.getHostName()), null); + } + } + } + // a budget of its own: the certificates are renewed by now, even if that used up the rotation timeout + if (!refreshKubeConfigData(System.currentTimeMillis() + KUBE_CONFIG_REFRESH_TIMEOUT_MS)) { + failRotation(String.format("Renewed the certificates of Kubernetes cluster %s but failed to refresh its stored kubeconfig", + kubernetesCluster.getName()), null); + } + } catch (RotationFailedException e) { + throw e; + } catch (Exception e) { + failRotation(String.format("Failed to rotate the certificates of Kubernetes cluster %s: %s", kubernetesCluster.getName(), e.getMessage()), e); + } finally { + if (rotationScriptFile != null) { + rotationScriptFile.delete(); + } + } + if (isStillOwnRotation()) { + // transition first, so the certificate scanner never sees RotatingCertificates without its details. + // A cluster rotated from Alert goes back there: whatever else put it in Alert hasn't been checked, and + // the status scanner moves it to Running once its nodes are ready + stateTransitTo(kubernetesCluster.getId(), startedFromAlert ? KubernetesCluster.Event.OperationFailed : KubernetesCluster.Event.OperationSucceeded); + clearRotationDetails(); + } else { + // e.g. taken for an interrupted rotation meanwhile, maybe retried since: leave the state and details alone + logger.warn("Renewed the certificates of Kubernetes cluster {}, but the cluster was taken out of this rotation meanwhile; leaving its state", + kubernetesCluster); + } + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_FAILED_DETAIL); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_SUCCEEDED_DETAIL, String.valueOf(System.currentTimeMillis()), false); + if (automatic) { + recordEvent(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE, EventVO.LEVEL_INFO, + String.format("Automatically renewed the control plane certificates of Kubernetes cluster %s. Kubeconfigs downloaded " + + "before keep working only until their old certificate expires; download it again", kubernetesCluster.getName())); + } + logger.info("Rotated the control plane certificates of Kubernetes cluster {}", kubernetesCluster); + return true; + } + + protected boolean renewCertificatesOnControlNode(UserVm vm, int controlNodeIndex) { + Pair endpoint = getControlNodeSshEndpoint(vm, controlNodeIndex); + long remainingSeconds = Math.max(60, (rotationTimeoutTime - System.currentTimeMillis()) / 1000); + try { + SshHelper.scpTo(endpoint.first(), endpoint.second(), getControlNodeLoginUser(), sshKeyFile, null, + "~/", rotationScriptFile.getAbsolutePath(), "0755"); + String script = rotationScriptFile.getName(); + String command = String.format("sudo ./%s %d%s; rc=$?; rm -f ./%s; exit $rc", script, remainingSeconds, rescueMode ? " rescue" : "", script); + int timeoutMs = (int) Math.min(Integer.MAX_VALUE, (remainingSeconds + 60) * 1000L); + Pair result = runOnNode(endpoint.first(), endpoint.second(), command, timeoutMs); + if (!result.first()) { + logger.error("Certificate renewal failed on control node {} of Kubernetes cluster {}: {}", vm, kubernetesCluster, result.second()); + } + return result.first(); + } catch (Exception e) { + logger.error("Certificate renewal failed on control node {} of Kubernetes cluster {}", vm, kubernetesCluster, e); + return false; + } + } + + /** + * Whether a renewable leaf certificate on any control node has expired, per kubeadm. Every node counts: after an + * interrupted rescue the first nodes are renewed while the others still hold etcd hostage. Nodes that can't be + * checked are left out. + */ + protected boolean areCertificatesExpired(List controlVms) { + CertificateExpiryReport report = new CertificateExpiryReport(); + for (int i = 0; i < controlVms.size(); i++) { + UserVm vm = controlVms.get(i); + Pair endpoint = getControlNodeSshEndpoint(vm, i); + try { + Pair result = runOnNode(endpoint.first(), endpoint.second(), KubernetesClusterCertificateUtil.CHECK_EXPIRATION_COMMAND, 60000); + report.add(vm.getHostName(), KubernetesClusterCertificateUtil.parseCheckExpirationOutput(StringUtils.defaultString(result.second()))); + } catch (Exception e) { + logger.warn("Unable to check whether the certificates on control node {} of Kubernetes cluster {} have expired", vm, kubernetesCluster, e); + } + } + // Only what the control plane itself runs on: an expired admin.conf or super-admin.conf breaks no node + Date now = new Date(); + return report.getEntries().stream().anyMatch(entry -> !entry.isAuthority() && !entry.isMissing() && !entry.isExternallyManaged() + && !CLIENT_ONLY_KUBECONFIGS.contains(StringUtils.substringBefore(entry.getName(), " on ")) && entry.getExpires().before(now)); + } + + /** Whether the cluster is still in the rotation this worker started, not moved on by recovery or a later rotation */ + protected boolean isStillOwnRotation() { + KubernetesCluster current = kubernetesClusterDao.findById(kubernetesCluster.getId()); + KubernetesClusterDetailsVO started = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ROTATION_STARTED_DETAIL); + return current != null && KubernetesCluster.State.RotatingCertificates.equals(current.getState()) + && started != null && StringUtils.equals(rotationStartedAt, started.getValue()); + } + + /** + * The rotation script already waited for this node's API server, controller manager and scheduler; this checks + * the cluster API endpoint answers and the node reports Ready. + */ + protected boolean waitForControlNodeReady(UserVm vm) { + if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, apiPublicIpAddress, CLUSTER_API_PORT, + rotationTimeoutTime, NODE_READY_POLL_INTERVAL_MS)) { + return false; + } + String hostName = StringUtils.lowerCase(vm.getHostName()); + return KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), + sshKeyFile, hostName, rotationTimeoutTime, NODE_READY_POLL_INTERVAL_MS); + } + + private void failRotation(String message, Exception e) { + logMessage(Level.ERROR, message, e); + if (isStillOwnRotation()) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + clearRotationDetails(); + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ROTATION_FAILED_DETAIL, String.valueOf(System.currentTimeMillis()), false); + sendAlert(String.format("Certificate rotation failed for Kubernetes cluster %s", kubernetesCluster.getName()), message); + // the API records the failure of its own async job event + if (automatic) { + recordEvent(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE, EventVO.LEVEL_ERROR, message); + } + throw new RotationFailedException(message, e); + } + + private void clearRotationDetails() { + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_STARTED_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_OWNER_DETAIL); + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ROTATION_SEEN_DETAIL); + } + + public void sendAlert(String subject, String body) { + try { + alertManager.sendAlert(AlertService.AlertType.ALERT_TYPE_KUBERNETES_CLUSTER_CERTIFICATE, kubernetesCluster.getZoneId(), null, subject, body); + } catch (Exception e) { + logger.warn("Unable to send the alert \"{}\" for Kubernetes cluster {}", subject, kubernetesCluster, e); + } + } + + public void recordEvent(String eventType, String level, String description) { + try { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, kubernetesCluster.getAccountId(), level, + eventType, true, description, + kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + } catch (Exception e) { + logger.warn("Unable to record the event \"{}\" for Kubernetes cluster {}", description, kubernetesCluster, e); + } + } + + private static class RotationFailedException extends CloudRuntimeException { + RotationFailedException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a15086c60d6c..8f7fe4bd4c87 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -730,22 +730,12 @@ private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromCluster private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { if (StringUtils.isEmpty(publicIpAddress)) { - KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL); if (kubeConfigDetail != null && StringUtils.isNotEmpty(kubeConfigDetail.getValue())) { return true; } } - String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, timeoutTime); - if (StringUtils.isNotEmpty(kubeConfig)) { - final String controlVMPrivateIpAddress = getControlVmPrivateIp(); - if (StringUtils.isNotEmpty(controlVMPrivateIpAddress)) { - kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", controlVMPrivateIpAddress, CLUSTER_API_PORT), - String.format("server: https://%s:%d", apiPublicIpAddress, CLUSTER_API_PORT)); - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())), false); - return true; - } - return false; + return refreshKubeConfigData(timeoutTime); } private boolean isKubernetesClusterDashboardServiceRunning(final boolean onCreate, final Long timeoutTime) { @@ -978,8 +968,10 @@ public boolean reconcileAlertCluster() { if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { return false; } - // mark the cluster to be running - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); + // mark the cluster to be running, unless another operation (e.g. a certificate rotation) took it meanwhile + if (!stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested)) { + return false; + } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 4c2725fc2a2f..aa753303bda3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -79,7 +79,7 @@ private Pair runInstallScriptOnVM(final UserVm vm, final int in 10000, 10000, 10 * 60 * 1000); } - private void upgradeKubernetesClusterNodes() { + protected void upgradeKubernetesClusterNodes() { for (int i = 0; i < clusterVMs.size(); ++i) { UserVm vm = clusterVMs.get(i); String hostName = vm.getHostName(); @@ -184,11 +184,28 @@ public boolean upgradeCluster() throws CloudRuntimeException { if (!updated) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } else { + refreshKubeConfigDataAfterUpgrade(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } return updated; } + /** + * kubeadm upgrade renews the control plane certificates, admin.conf included, so the stored kubeConfigData + * has to be read again or getKubernetesClusterConfig keeps handing out the old admin certificate until it expires. + * A failure here doesn't fail the upgrade: the certificate expiry check refreshes a stale copy later on. + */ + protected void refreshKubeConfigDataAfterUpgrade() { + try { + apiPublicIpAddress = resolveApiPublicIpAddress(networkDao.findById(kubernetesCluster.getNetworkId())); + if (!refreshKubeConfigData(System.currentTimeMillis() + KUBE_CONFIG_REFRESH_TIMEOUT_MS)) { + logger.warn("Unable to refresh the stored kubeconfig of Kubernetes cluster {} after the upgrade", kubernetesCluster); + } + } catch (Exception e) { + logger.warn("Unable to refresh the stored kubeconfig of Kubernetes cluster {} after the upgrade", kubernetesCluster, e); + } + } + protected void filterOutManualUpgradeNodesFromClusterUpgrade() { if (CollectionUtils.isEmpty(clusterVMs)) { return; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java new file mode 100644 index 000000000000..5d466935321c --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtil.java @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.utils; + +import java.io.ByteArrayInputStream; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Helpers for the control plane certificate expiry check and renewal of CKS clusters. + * Everything here is free of side effects so it can be unit tested. + */ +public class KubernetesClusterCertificateUtil { + + protected static Logger LOGGER = LogManager.getLogger(KubernetesClusterCertificateUtil.class); + + /** {@code kubeadm certs} replaced {@code kubeadm alpha certs} in this version. */ + public static final String MIN_KUBERNETES_VERSION_CERTIFICATES_SUPPORT = "1.20.0"; + + public static final String CHECK_EXPIRATION_COMMAND = "sudo /opt/bin/kubeadm certs check-expiration"; + + /** Prints only the client certificate line of root's kubeconfig, which CKS itself uses for kubectl. Never the key. */ + public static final String ROOT_KUBECONFIG_CERTIFICATE_COMMAND = "sudo grep -m1 'client-certificate-data:' /root/.kube/config"; + + public static final String ROOT_KUBECONFIG_ENTRY_NAME = "/root/.kube/config"; + /** + * getKubernetesClusterConfig hands out an operator-supplied user.conf instead of admin.conf when there is one, so + * the stored kubeconfig has to be compared with that. Prints a marker when it exists, then its certificate line. + */ + public static final String USER_KUBECONFIG_PRESENT = "user.conf present"; + public static final String USER_KUBECONFIG_CERTIFICATE_COMMAND = "if sudo test -f /etc/kubernetes/user.conf; then echo '" + USER_KUBECONFIG_PRESENT + + "'; sudo grep -m1 'client-certificate-data:' /etc/kubernetes/user.conf; fi"; + + /** + * kubeadm issues certificates for one year; renewing more than this many days ahead would leave them inside the + * renewal window right after a renewal and renew on every check. + */ + public static final int MAX_AUTO_RENEW_DAYS = 180; + + /** + * Expiries this close are treated as the same one for alerting: control nodes renewed in one rotation differ by + * minutes, and which node reports the first expiry changes when one can't be checked. A renewal moves it by months. + */ + private static final long SAME_EXPIRY_TOLERANCE_MS = TimeUnit.DAYS.toMillis(7); + + private static final String MISSING_MARKER = "!MISSING!"; + + // kubeadm formats the dates with Go's "Jan 02, 2006 15:04 MST", always in UTC + private static final Pattern CHECK_EXPIRATION_LINE = Pattern.compile( + "^(\\S+)\\s+([A-Z][a-z]{2} \\d{1,2}, \\d{4} \\d{2}:\\d{2} [A-Z]+)\\s+.*$"); + private static final DateTimeFormatter CHECK_EXPIRATION_DATE_FORMAT = + DateTimeFormatter.ofPattern("MMM d, yyyy HH:mm z", Locale.ENGLISH); + private static final Pattern CLIENT_CERTIFICATE_DATA = Pattern.compile("client-certificate-data:\\s*([A-Za-z0-9+/=]+)"); + + /** One line of {@code kubeadm certs check-expiration}. */ + public static class CertificateExpiry { + private final String name; + private final Date expires; + private final boolean authority; + private final boolean externallyManaged; + + public CertificateExpiry(String name, Date expires, boolean authority, boolean externallyManaged) { + this.name = name; + this.expires = expires; + this.authority = authority; + this.externallyManaged = externallyManaged; + } + + public String getName() { + return name; + } + + /** @return the expiry date, or null when kubeadm reported the certificate as missing */ + public Date getExpires() { + return expires; + } + + public boolean isAuthority() { + return authority; + } + + public boolean isMissing() { + return expires == null; + } + + /** kubeadm cannot renew a certificate it reports as externally managed */ + public boolean isExternallyManaged() { + return externallyManaged; + } + + @Override + public String toString() { + return String.format("%s%s%s: %s", name, authority ? " (CA)" : "", externallyManaged ? " (externally managed)" : "", + isMissing() ? "missing" : expires); + } + } + + /** + * The certificates of every control node of one cluster, as far as they could be read. + */ + public static class CertificateExpiryReport { + private final List entries = new ArrayList<>(); + private final List uncheckedNodes = new ArrayList<>(); + + public void add(String node, List nodeEntries) { + for (CertificateExpiry entry : nodeEntries) { + entries.add(new CertificateExpiry(String.format("%s on %s", entry.getName(), node), entry.getExpires(), entry.isAuthority(), + entry.isExternallyManaged())); + } + } + + public void addUncheckedNode(String node) { + uncheckedNodes.add(node); + } + + public List getEntries() { + return entries; + } + + public List getUncheckedNodes() { + return uncheckedNodes; + } + + public boolean isEmpty() { + return entries.isEmpty(); + } + + /** @return the leaf certificate (not a CA) that expires first, or null */ + public CertificateExpiry getFirstLeafExpiry() { + return getFirstExpiry(false, true); + } + + /** @return the leaf certificate that kubeadm can renew which expires first, or null */ + public CertificateExpiry getFirstRenewableLeafExpiry() { + return getFirstExpiry(false, false); + } + + /** @return the certificate authority that expires first, or null */ + public CertificateExpiry getFirstAuthorityExpiry() { + return getFirstExpiry(true, true); + } + + private CertificateExpiry getFirstExpiry(boolean authority, boolean includeExternallyManaged) { + CertificateExpiry first = null; + for (CertificateExpiry entry : entries) { + if (entry.isAuthority() != authority || entry.isMissing() || (!includeExternallyManaged && entry.isExternallyManaged())) { + continue; + } + if (first == null || entry.getExpires().before(first.getExpires())) { + first = entry; + } + } + return first; + } + } + + /** + * Parses the output of {@code kubeadm certs check-expiration}. It has a leaf certificate table followed by a + * certificate authority table; the line starting with {@code CERTIFICATE AUTHORITY} heads the second one. + * Certificates kubeadm cannot find are printed as {@code !MISSING! } and returned with a null expiry. + * The last column of both tables tells whether the certificate is externally managed. + */ + public static List parseCheckExpirationOutput(String output) { + List entries = new ArrayList<>(); + if (StringUtils.isBlank(output)) { + return entries; + } + boolean authoritySection = false; + for (String rawLine : output.split("\\r?\\n")) { + String line = rawLine.trim(); + if (line.isEmpty() || line.startsWith("[")) { + continue; + } + if (line.startsWith("CERTIFICATE AUTHORITY")) { + authoritySection = true; + continue; + } + if (line.startsWith("CERTIFICATE")) { + authoritySection = false; + continue; + } + if (line.startsWith(MISSING_MARKER)) { + String name = StringUtils.trimToNull(line.substring(MISSING_MARKER.length())); + if (name != null) { + entries.add(new CertificateExpiry(name.split("\\s+")[0], null, authoritySection, false)); + } + continue; + } + Matcher matcher = CHECK_EXPIRATION_LINE.matcher(line); + if (!matcher.matches()) { + continue; + } + try { + Date expires = Date.from(ZonedDateTime.parse(matcher.group(2), CHECK_EXPIRATION_DATE_FORMAT).toInstant()); + String[] columns = line.split("\\s+"); + boolean externallyManaged = "yes".equalsIgnoreCase(columns[columns.length - 1]); + entries.add(new CertificateExpiry(matcher.group(1), expires, authoritySection, externallyManaged)); + } catch (DateTimeParseException e) { + LOGGER.debug("Unable to parse the expiry date in kubeadm output line: {}", line, e); + } + } + return entries; + } + + /** + * @return the expiry of the first {@code client-certificate-data} in a kubeconfig (or in a single line of one), + * or null if there is none or it cannot be decoded + */ + public static Date getKubeConfigClientCertificateExpiry(String kubeConfig) { + if (StringUtils.isBlank(kubeConfig)) { + return null; + } + Matcher matcher = CLIENT_CERTIFICATE_DATA.matcher(kubeConfig); + if (!matcher.find()) { + return null; + } + try { + byte[] pem = Base64.decodeBase64(matcher.group(1)); + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + X509Certificate certificate = (X509Certificate) factory.generateCertificate(new ByteArrayInputStream(pem)); + return certificate.getNotAfter(); + } catch (Exception e) { + LOGGER.debug("Unable to decode the client certificate of a kubeconfig", e); + return null; + } + } + + /** + * Parses a comma-separated list of day thresholds, ignoring blanks, non-numbers and non-positive values. + * @return the distinct thresholds, largest first + */ + public static List parseAlertThresholds(String value) { + TreeSet thresholds = new TreeSet<>(Collections.reverseOrder()); + if (StringUtils.isBlank(value)) { + return new ArrayList<>(thresholds); + } + for (String part : value.split(",")) { + try { + int days = Integer.parseInt(part.trim()); + if (days > 0) { + thresholds.add(days); + } + } catch (NumberFormatException e) { + LOGGER.warn("Ignoring invalid Kubernetes certificate expiry alert threshold: {}", part); + } + } + return new ArrayList<>(thresholds); + } + + /** @return whole days from now until expiry, rounded down; negative once expired */ + public static long daysUntil(Date expiry, Date now) { + return Math.floorDiv(expiry.getTime() - now.getTime(), TimeUnit.DAYS.toMillis(1)); + } + + /** + * Decides whether an expiry alert is due. + * @param daysLeft days until the certificate expires + * @param thresholds the alert thresholds, as returned by {@link #parseAlertThresholds(String)} + * @param lastAlertedThreshold the smallest threshold already alerted for this same expiry date, or null + * @return the threshold to alert for now, or null when no new alert is due. Only the smallest threshold reached + * is returned, so a cluster first checked at 20 days alerts once for 30, not for 60 and 30. Expiry itself counts + * as threshold 0, so it is always alerted. + */ + public static Integer getThresholdToAlert(long daysLeft, List thresholds, Integer lastAlertedThreshold) { + if (daysLeft < 0) { + return lastAlertedThreshold != null && lastAlertedThreshold <= 0 ? null : 0; + } + Integer reached = null; + for (Integer threshold : thresholds) { + if (daysLeft <= threshold && (reached == null || threshold < reached)) { + reached = threshold; + } + } + if (reached == null) { + return null; + } + if (lastAlertedThreshold != null && lastAlertedThreshold <= reached) { + return null; + } + return reached; + } + + /** + * Alert state persisted per cluster as {@code :}, so a renewal, + * which changes the expiry, starts the thresholds over. + * @return the threshold already alerted for this expiry, or null + */ + public static Integer getLastAlertedThreshold(String alertState, Date expiry) { + if (StringUtils.isBlank(alertState) || expiry == null) { + return null; + } + String[] parts = alertState.split(":"); + if (parts.length != 2) { + return null; + } + try { + if (Math.abs(Long.parseLong(parts[0]) - expiry.getTime()) > SAME_EXPIRY_TOLERANCE_MS) { + return null; + } + return Integer.parseInt(parts[1]); + } catch (NumberFormatException e) { + return null; + } + } + + public static String buildAlertState(Date expiry, int threshold) { + return String.format("%d:%d", expiry.getTime(), threshold); + } + + /** @param autoRenewDays the configured value, capped at {@link #MAX_AUTO_RENEW_DAYS} */ + public static boolean isAutoRenewDue(long daysLeft, int autoRenewDays) { + return autoRenewDays > 0 && daysLeft <= Math.min(autoRenewDays, MAX_AUTO_RENEW_DAYS); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 023fc9ec7fad..5f34c4c747fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -48,7 +48,7 @@ public class KubernetesClusterUtil { protected static Logger LOGGER = LogManager.getLogger(KubernetesClusterUtil.class); - public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'"; + public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && ($2 == \"Ready\" || $2 ~ /^Ready,/)) print $1}'"; public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $5}'"; public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port, @@ -198,6 +198,8 @@ public static boolean isKubernetesClusterDashboardServiceRunning(final Kubernete return running; } + private static final long KUBE_CONFIG_RETRY_INTERVAL_MS = 5000; + public static String getKubernetesClusterConfig(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String user, final File sshKeyFile, final long timeoutTime) { String kubeConfig = ""; @@ -218,6 +220,13 @@ public static String getKubernetesClusterConfig(final KubernetesCluster kubernet } catch (Exception e) { LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster %s", kubernetesCluster), e); } + // a refused connection fails at once; don't spin on it until the timeout + try { + Thread.sleep(Math.max(0, Math.min(KUBE_CONFIG_RETRY_INTERVAL_MS, timeoutTime - System.currentTimeMillis()))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } return kubeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java new file mode 100644 index 000000000000..c47b8ec16f86 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RotateKubernetesClusterCertificatesCmd.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "rotateKubernetesClusterCertificates", + description = "Renews the kubeadm-managed control plane certificates of a running CloudManaged Kubernetes cluster, " + + "one control node at a time, and refreshes the kubeconfig returned by getKubernetesClusterConfig. " + + "The cluster certificate authorities are not rotated. Requires Kubernetes 1.20 or later", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class RotateKubernetesClusterCertificatesCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, required = true, + description = "The ID of the Kubernetes cluster") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE; + } + + @Override + public String getEventDescription() { + return "Rotating the certificates of Kubernetes cluster with ID: " + getResourceUuid(ApiConstants.ID); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + try { + if (!kubernetesClusterService.rotateKubernetesClusterCertificates(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + String.format("Failed to rotate the certificates of Kubernetes cluster with ID: %s", getResourceUuid(ApiConstants.ID))); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh new file mode 100755 index 000000000000..9372b2a9cb32 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/rotate-kubernetes-certificates.sh @@ -0,0 +1,235 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Renews the kubeadm-managed certificates of this control node and restarts its +# control plane static pods so they load them. The certificate authorities are +# left alone. Safe to rerun: an interrupted earlier run is recovered first. +# +# Usage: ./rotate-kubernetes-certificates.sh [WAIT_TIMEOUT_SECONDS] [rescue] +# +# rescue is for a cluster whose API server isn't answering, e.g. because its +# certificates have expired. With stacked etcd and several control nodes, one +# renewed node can't get etcd quorum until the others are renewed as well, so +# this node only waits for its static pods to be back, not for them to be +# healthy: without quorum the API server keeps restarting until it gets one. + +WAIT_TIMEOUT="${1:-300}" +RESCUE="${2:-}" + +# the binaries CKS installs come first, ahead of any the template ships +export PATH=/opt/bin:$PATH:/usr/local/bin:/usr/sbin:/sbin +CRI_ENDPOINT=unix:///run/containerd/containerd.sock + +MANIFESTS_DIR=/etc/kubernetes/manifests +HOLD_DIR=/etc/kubernetes/manifests-cks-certificate-rotation +COMPONENTS="kube-apiserver kube-controller-manager kube-scheduler etcd" +API_SERVER_PORT=6443 +# kubelet looks at the static pod manifests every 20 seconds by default +KUBELET_FILE_CHECK_WAIT=25 + +log() { + echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" +} + +# Puts back any manifest moved out by this script, also when it fails half-way, +# so the control plane is never left without its static pods +restore_manifests() { + if [ ! -d "${HOLD_DIR}" ]; then + return 0 + fi + for manifest in "${HOLD_DIR}"/*.yaml; do + [ -e "${manifest}" ] || continue + if [ -e "${MANIFESTS_DIR}/$(basename "${manifest}")" ]; then + rm -f "${manifest}" + else + mv "${manifest}" "${MANIFESTS_DIR}/" + fi + done + rmdir "${HOLD_DIR}" 2>/dev/null || true +} +trap restore_manifests EXIT + +# Keeps every distinct earlier /root/.kube/config: an existing backup is never overwritten +backup_root_kubeconfig() { + local backup=/root/.kube/config.cks-backup + if [ -e "${backup}" ]; then + cmp -s /root/.kube/config "${backup}" && return 0 + backup="${backup}.$(date +%s)" + fi + cp -p /root/.kube/config "${backup}" +} + +container_running() { + [ -n "$(crictl --runtime-endpoint "${CRI_ENDPOINT}" ps -q --name "^${1}\$" 2>/dev/null)" ] +} + +# A new sandbox of the static pod is up, not one torn down by this run; its +# containers may still be restarting +pod_ready() { + local id + for id in $(crictl --runtime-endpoint "${CRI_ENDPOINT}" pods -q --state ready --name "^${1}-" 2>/dev/null); do + case " ${old_pods} " in + *" ${id} "*) ;; + *) return 0 ;; + esac + done + return 1 +} + +endpoint_healthy() { + [ "$(curl -sk --max-time 5 "$1" 2>/dev/null)" = "ok" ] +} + +served_certificate_current() { + if ! command -v openssl >/dev/null 2>&1; then + return 0 + fi + local expected served + expected=$(openssl x509 -noout -enddate -in /etc/kubernetes/pki/apiserver.crt 2>/dev/null) + served=$(echo | openssl s_client -connect 127.0.0.1:${API_SERVER_PORT} 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null) + [ -n "${expected}" ] && [ "${expected}" = "${served}" ] +} + +# Returns non-zero if the API server isn't ready within the given number of seconds +wait_for_api_server() { + local limit=$(( $(date +%s) + $1 )) + [ "${limit}" -gt "${deadline}" ] && limit=${deadline} + while ! endpoint_healthy "https://127.0.0.1:${API_SERVER_PORT}/readyz"; do + if [ "$(date +%s)" -ge "${limit}" ]; then + return 1 + fi + sleep 3 + done +} + +deadline=$(( $(date +%s) + WAIT_TIMEOUT )) + +restore_manifests + +if [ ! -f /etc/kubernetes/admin.conf ] || [ ! -f "${MANIFESTS_DIR}/kube-apiserver.yaml" ]; then + log "ERROR: this is not a kubeadm control plane node" + exit 1 +fi + +# The static pods must be stopped and confirmed gone before their manifests come back +if ! crictl --runtime-endpoint "${CRI_ENDPOINT}" ps -q >/dev/null 2>&1; then + log "ERROR: unable to list the containers with crictl on ${CRI_ENDPOINT}, so the control plane pods restart can't be confirmed" + exit 1 +fi + +# kubeadm reads its configuration from the cluster and silently falls back to +# defaults when the API server is down, e.g. right after an interrupted run, so +# give it time to come up. Carry on if it doesn't: once the certificates have +# expired it never becomes ready, and renewing is the way out. +if [ "${RESCUE}" = "rescue" ]; then + log "Rescue mode: renewing without waiting for the API server" +elif ! wait_for_api_server 180; then + log "WARNING: the API server is not ready, renewing with the kubeadm configuration it can read" +fi + +# root's kubeconfig is replaced with admin.conf below; note now, before kubeadm +# renews admin.conf, whether it is something else worth a backup +root_kubeconfig_custom=false +if [ -f /root/.kube/config ] && ! cmp -s /etc/kubernetes/admin.conf /root/.kube/config; then + root_kubeconfig_custom=true +fi + +log "Renewing the control plane certificates" +apiserver_before=$(sha256sum /etc/kubernetes/pki/apiserver.crt | cut -d' ' -f1) +admin_before=$(sha256sum /etc/kubernetes/admin.conf | cut -d' ' -f1) +# under -e a failing command substitution would exit before kubeadm's error is printed +if ! renew_output=$(kubeadm certs renew all 2>&1); then + echo "${renew_output}" + log "ERROR: kubeadm certs renew all failed" + exit 1 +fi +echo "${renew_output}" +# kubeadm exits 0 without renewing anything signed by a CA whose key it doesn't have (external CA mode) +if echo "${renew_output}" | grep -q "can't be renewed" || \ + [ "$(sha256sum /etc/kubernetes/pki/apiserver.crt | cut -d' ' -f1)" = "${apiserver_before}" ] || \ + [ "$(sha256sum /etc/kubernetes/admin.conf | cut -d' ' -f1)" = "${admin_before}" ]; then + log "ERROR: kubeadm did not renew the control plane certificates, are they signed by an external CA?" + exit 1 +fi + +# CKS runs kubectl as root with /root/.kube/config, a copy of admin.conf +mkdir -p /root/.kube +if [ "${root_kubeconfig_custom}" = true ]; then + backup_root_kubeconfig +fi +cp /etc/kubernetes/admin.conf /root/.kube/config +chmod 600 /root/.kube/config + +log "Restarting the control plane static pods" +old_pods=$(crictl --runtime-endpoint "${CRI_ENDPOINT}" pods -q 2>/dev/null | tr '\n' ' ') +mkdir -p "${HOLD_DIR}" +moved="" +for component in ${COMPONENTS}; do + if [ -f "${MANIFESTS_DIR}/${component}.yaml" ]; then + mv "${MANIFESTS_DIR}/${component}.yaml" "${HOLD_DIR}/" + moved="${moved} ${component}" + fi +done + +sleep ${KUBELET_FILE_CHECK_WAIT} +for component in ${moved}; do + while container_running "${component}"; do + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "ERROR: timed out waiting for ${component} to stop" + exit 1 + fi + sleep 2 + done +done + +restore_manifests + +if [ "${RESCUE}" = "rescue" ]; then + log "Rescue mode: waiting for the control plane pods to be back" + while true; do + # control nodes of a cluster with separate etcd nodes run no etcd of their own + if { [ ! -f "${MANIFESTS_DIR}/etcd.yaml" ] || container_running etcd; } && \ + pod_ready kube-apiserver && pod_ready kube-controller-manager && pod_ready kube-scheduler; then + break + fi + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "ERROR: timed out waiting for the control plane pods to come back" + exit 1 + fi + sleep 3 + done + log "Control plane certificates renewed, the control plane becomes healthy once every control node is renewed" + exit 0 +fi + +log "Waiting for the control plane to become healthy" +while true; do + if endpoint_healthy "https://127.0.0.1:${API_SERVER_PORT}/readyz" && \ + endpoint_healthy "https://127.0.0.1:10257/healthz" && \ + endpoint_healthy "https://127.0.0.1:10259/healthz" && \ + served_certificate_current; then + break + fi + if [ "$(date +%s)" -ge "${deadline}" ]; then + log "ERROR: timed out waiting for the control plane to become healthy" + exit 1 + fi + sleep 3 +done + +log "Control plane certificates renewed" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 2e73467c9823..53ac258a5e9e 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -123,6 +123,13 @@ if [ -d "$BINARIES_DIR" ]; then tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + # root's kubeconfig is kept in step with admin.conf below; note now, before + # kubeadm renews admin.conf, whether it is something else worth a backup + root_kubeconfig_custom=false + if [ -f /etc/kubernetes/admin.conf ] && [ -f /root/.kube/config ] && ! cmp -s /etc/kubernetes/admin.conf /root/.kube/config; then + root_kubeconfig_custom=true + fi + if [ "${IS_MAIN_CONTROL}" == 'true' ]; then set +e kubeadm --v=5 upgrade apply ${UPGRADE_VERSION} -y @@ -147,6 +154,20 @@ if [ -d "$BINARIES_DIR" ]; then systemctl restart containerd systemctl restart kubelet + # kubeadm upgrade renews admin.conf on control nodes; keep root's copy, used by kubectl here, in step with it + if [ -f /etc/kubernetes/admin.conf ] && [ -d /root/.kube ]; then + # never overwrite an earlier backup + if [ "${root_kubeconfig_custom}" = true ]; then + backup=/root/.kube/config.cks-backup + if [ -e "${backup}" ] && ! cmp -s /root/.kube/config "${backup}"; then + backup="${backup}.$(date +%s)" + fi + cp -p /root/.kube/config "${backup}" + fi + cp /etc/kubernetes/admin.conf /root/.kube/config + chmod 600 /root/.kube/config + fi + if [ "${IS_MAIN_CONTROL}" == 'true' ]; then if [[ ${EXTERNAL_CNI} == true ]]; then /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java new file mode 100644 index 000000000000..eacc44e2b51b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterCertificateManagementTest.java @@ -0,0 +1,536 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterCertificateWorker; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class KubernetesClusterCertificateManagementTest { + + private static final long CLUSTER_ID = 11L; + private static final long DAY = TimeUnit.DAYS.toMillis(1); + + @Mock + KubernetesClusterDao kubernetesClusterDao; + @Mock + KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Mock + KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock + AccountManager accountManager; + + @Spy + @InjectMocks + KubernetesClusterManagerImpl manager; + + @Mock + KubernetesClusterVO cluster; + @Mock + KubernetesSupportedVersionVO version; + @Mock + KubernetesClusterCertificateWorker worker; + + private final Map details = new HashMap<>(); + private final Map, Object> originalDefaults = new HashMap<>(); + private Date now; + + @Before + public void setUp() throws Exception { + now = new Date(); + Mockito.when(cluster.getId()).thenReturn(CLUSTER_ID); + Mockito.when(cluster.getName()).thenReturn("cks"); + Mockito.when(cluster.getAccountId()).thenReturn(2L); + Mockito.when(cluster.getKubernetesVersionId()).thenReturn(4L); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.CloudManaged); + Mockito.when(kubernetesSupportedVersionDao.findById(4L)).thenReturn(version); + Mockito.when(version.getSemanticVersion()).thenReturn("1.32.5"); + + Mockito.when(kubernetesClusterDetailsDao.findDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.anyString())).thenAnswer(invocation -> { + String name = invocation.getArgument(1); + String value = details.get(name); + return value == null ? null : new KubernetesClusterDetailsVO(CLUSTER_ID, name, value, false); + }); + Mockito.doAnswer(invocation -> details.put(invocation.getArgument(1), invocation.getArgument(2))) + .when(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyBoolean()); + Mockito.doAnswer(invocation -> details.remove(invocation.getArgument(1))) + .when(kubernetesClusterDetailsDao).removeDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.anyString()); + + Mockito.doReturn(worker).when(manager).createCertificateWorker(cluster); + Mockito.doReturn(true).when(manager).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + + setDefault(KubernetesClusterService.KubernetesServiceEnabled, "true"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateCheckInterval, "6"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateExpiryAlertDays, "60,30"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "true"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewDays, "30"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateRotationTimeout, "3600"); + } + + @After + public void tearDown() throws Exception { + for (Map.Entry, Object> entry : originalDefaults.entrySet()) { + Field field = ConfigKey.class.getDeclaredField("_defaultValue"); + field.setAccessible(true); + field.set(entry.getKey(), entry.getValue()); + clearCachedValue(entry.getKey()); + } + } + + // non-dynamic keys such as cloud.kubernetes.service.enabled cache their value, which would leak into other tests + private void clearCachedValue(ConfigKey key) throws Exception { + Field cached = ConfigKey.class.getDeclaredField("_value"); + cached.setAccessible(true); + cached.set(key, null); + } + + private void setDefault(ConfigKey key, String value) throws Exception { + Field field = ConfigKey.class.getDeclaredField("_defaultValue"); + field.setAccessible(true); + originalDefaults.putIfAbsent(key, field.get(key)); + field.set(key, value); + clearCachedValue(key); + } + + private CertificateExpiryReport report(long leafDays, long caDays) { + CertificateExpiryReport report = new CertificateExpiryReport(); + report.add("cks-control-1", Arrays.asList( + new CertificateExpiry("apiserver", new Date(now.getTime() + leafDays * DAY + 3600_000L), false, false), + new CertificateExpiry("admin.conf", new Date(now.getTime() + (leafDays + 10) * DAY), false, false), + new CertificateExpiry("ca", new Date(now.getTime() + caDays * DAY + 3600_000L), true, false))); + return report; + } + + /** Runs a check at now + elapsedDays, the certificates then expiring after the given number of days. */ + private void checkAt(long elapsedDays, long leafDays, long caDays) { + Date when = new Date(now.getTime() + elapsedDays * DAY); + Date oldNow = now; + now = when; + Mockito.doReturn(report(leafDays, caDays)).when(worker).checkCertificateExpiry(); + now = oldNow; + manager.processKubernetesClusterCertificates(cluster, when); + } + + private ArgumentCaptor verifyAlerts(int times) { + ArgumentCaptor subjects = ArgumentCaptor.forClass(String.class); + Mockito.verify(worker, Mockito.times(times)).sendAlert(subjects.capture(), ArgumentMatchers.anyString()); + Mockito.verify(worker, Mockito.times(times)).recordEvent(ArgumentMatchers.eq(KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + return subjects; + } + + private ArgumentCaptor verifyAlertBodies(int times) { + ArgumentCaptor bodies = ArgumentCaptor.forClass(String.class); + Mockito.verify(worker, Mockito.times(times)).sendAlert(ArgumentMatchers.anyString(), bodies.capture()); + return bodies; + } + + @Test + public void noAlertWhileFarFromExpiry() { + checkAt(0, 200, 3000); + Mockito.verify(worker).checkCertificateExpiry(); + verifyAlerts(0); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + Assert.assertNotNull(details.get(KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_DETAIL)); + } + + @Test + public void checkRunsAtMostOncePerInterval() { + checkAt(0, 200, 3000); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(5))); + Mockito.verify(worker, Mockito.times(1)).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(6))); + Mockito.verify(worker, Mockito.times(2)).checkCertificateExpiry(); + } + + @Test + public void eachThresholdAlertsOnceThenAutoRenewTriggers() throws Exception { + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "false"); + // the same certificates, checked as their expiry approaches + checkAt(0, 59, 3000); + verifyAlerts(1); + checkAt(1, 58, 3000); + verifyAlerts(1); + checkAt(29, 30, 3000); + ArgumentCaptor subjects = verifyAlerts(2); + Assert.assertTrue(subjects.getAllValues().get(0).contains("expires in 59 days")); + Assert.assertTrue(subjects.getAllValues().get(1).contains("expires in 30 days")); + checkAt(30, 29, 3000); + verifyAlerts(2); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "true"); + checkAt(31, 28, 3000); + verifyAlerts(2); + Mockito.verify(worker).setAutomatic(true); + Mockito.verify(worker).rotateCertificates(); + } + + @Test + public void automaticRenewalRunsOffTheScannerThread() { + ExecutorService executor = Mockito.mock(ExecutorService.class); + manager._certificateRotationExecutor = executor; + try { + checkAt(0, 25, 3000); + verifyAlerts(1); + // handed to the rotation executor; the scanner doesn't wait for it + Mockito.verify(executor).execute(ArgumentMatchers.any(Runnable.class)); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } finally { + manager._certificateRotationExecutor = null; + } + } + + @Test + public void alertAdviceStatesTheEffectiveAutoRenewDays() throws Exception { + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "false"); + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewDays, "365"); + checkAt(0, 59, 3000); + ArgumentCaptor bodies = ArgumentCaptor.forClass(String.class); + Mockito.verify(worker).sendAlert(ArgumentMatchers.anyString(), bodies.capture()); + Assert.assertTrue(bodies.getValue().contains("Run rotateKubernetesClusterCertificates")); + + setDefault(KubernetesClusterService.KubernetesClusterCertificateAutoRenewEnabled, "true"); + checkAt(1, 29, 3000); + Mockito.verify(worker, Mockito.times(2)).sendAlert(ArgumentMatchers.anyString(), bodies.capture()); + Assert.assertTrue(bodies.getValue(), bodies.getValue().contains("renewed automatically 180 days before")); + } + + @Test + public void noAutomaticRenewalWhileAControlNodeIsUnreachable() { + CertificateExpiryReport partial = report(25, 3000); + partial.addUncheckedNode("cks-control-2"); + Mockito.doReturn(partial).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + ArgumentCaptor bodies = verifyAlertBodies(1); + // the alert doesn't promise an automatic renewal that is on hold + Assert.assertTrue(bodies.getValue(), bodies.getValue().contains("on hold until control nodes [cks-control-2] can be checked")); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void clusterInAlertIsCheckedButNotRenewedAutomatically() { + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + checkAt(0, 25, 3000); + verifyAlerts(1); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void queuedAutomaticRotationIsSkippedWhenNoLongerNeeded() { + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + long queuedAt = now.getTime(); + + // renewed through the API while the automatic renewal waited for a thread + details.put(KubernetesClusterCertificateWorker.ROTATION_SUCCEEDED_DETAIL, String.valueOf(queuedAt + 1000)); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + // failed meanwhile, e.g. through the API: the backoff applies + details.put(KubernetesClusterCertificateWorker.ROTATION_SUCCEEDED_DETAIL, String.valueOf(queuedAt - DAY)); + details.put(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(queuedAt + 1000)); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + details.remove(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL); + + // no longer Running + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + manager.runQueuedAutomaticRotation(CLUSTER_ID, queuedAt); + Mockito.verify(worker).setAutomatic(true); + Mockito.verify(worker).rotateCertificates(); + } + + @Test + public void automaticRotationIsQueuedOnceAndRequeuedAfterARejection() { + ExecutorService executor = Mockito.mock(ExecutorService.class); + manager._certificateRotationExecutor = executor; + try { + Mockito.doThrow(new RejectedExecutionException("shutting down")).doNothing() + .when(executor).execute(ArgumentMatchers.any(Runnable.class)); + manager.scheduleAutomaticRotation(cluster); + // the rejected one isn't left marked as queued + manager.scheduleAutomaticRotation(cluster); + // still waiting for a thread: not queued again + manager.scheduleAutomaticRotation(cluster); + Mockito.verify(executor, Mockito.times(2)).execute(ArgumentMatchers.any(Runnable.class)); + } finally { + manager._certificateRotationExecutor = null; + } + } + + @Test + public void renewedCertificatesStartTheThresholdsOver() { + Mockito.when(worker.rotateCertificates()).thenReturn(true); + checkAt(0, 25, 3000); + verifyAlerts(1); + Mockito.verify(worker).rotateCertificates(); + // renewed: a new expiry date, a year out, then a year later the thresholds alert again + checkAt(1, 364, 3000); + verifyAlerts(1); + checkAt(306, 59, 3000); + verifyAlerts(2); + } + + @Test + public void certificateAuthorityExpiryAlertsButIsNeverRenewed() { + checkAt(0, 300, 45); + ArgumentCaptor subjects = verifyAlerts(1); + Assert.assertTrue(subjects.getValue().contains("certificate authority")); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void autoRenewBacksOffAfterAFailedRotation() { + Mockito.when(worker.rotateCertificates()).thenThrow(new CloudRuntimeException("node 2 failed")); + details.put(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(now.getTime() - TimeUnit.HOURS.toMillis(2))); + checkAt(0, 20, 3000); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + + details.put(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL, String.valueOf(now.getTime() - TimeUnit.HOURS.toMillis(25))); + checkAt(1, 19, 3000); + Mockito.verify(worker).rotateCertificates(); + } + + @Test + public void externallyManagedCertificatesDoNotTriggerAutoRenew() { + CertificateExpiryReport report = new CertificateExpiryReport(); + report.add("cks-control-1", Arrays.asList( + new CertificateExpiry("apiserver-etcd-client", new Date(now.getTime() + 10 * DAY), false, true), + new CertificateExpiry("apiserver", new Date(now.getTime() + 300 * DAY), false, false))); + Mockito.doReturn(report).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + verifyAlerts(1); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void nothingHappensWhenTheCheckIsDisabledOrTheVersionTooOld() throws Exception { + setDefault(KubernetesClusterService.KubernetesClusterCertificateCheckInterval, "0"); + manager.processKubernetesClusterCertificates(cluster, now); + setDefault(KubernetesClusterService.KubernetesClusterCertificateCheckInterval, "6"); + Mockito.when(version.getSemanticVersion()).thenReturn("1.19.4"); + manager.processKubernetesClusterCertificates(cluster, now); + Mockito.verify(worker, Mockito.never()).checkCertificateExpiry(); + Assert.assertTrue(details.isEmpty()); + } + + @Test + public void unreachableClusterIsRetriedAtTheNextInterval() { + CertificateExpiryReport empty = new CertificateExpiryReport(); + empty.addUncheckedNode("cks-control-1"); + Mockito.doReturn(empty).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + 60_000)); + Mockito.verify(worker, Mockito.times(1)).checkCertificateExpiry(); + // no expiry alert, only the one that the check could not run + Mockito.verify(worker, Mockito.never()).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.verify(worker, Mockito.times(1)).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + } + + @Test + public void clusterThatCannotBeReachedAtAllAlerts() { + Mockito.doThrow(new CloudRuntimeException("Unable to retrieve the SSH address")).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + Mockito.verify(worker).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + Mockito.verify(worker, Mockito.never()).rotateCertificates(); + } + + @Test + public void clusterWhoseNodesCannotBeCheckedAlertsOnceADay() { + CertificateExpiryReport nothing = new CertificateExpiryReport(); + nothing.addUncheckedNode("cks-control-1"); + Mockito.doReturn(nothing).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, now); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(6))); + Mockito.verify(worker, Mockito.times(1)).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(24))); + Mockito.verify(worker, Mockito.times(2)).sendAlert(ArgumentMatchers.contains("Unable to check"), ArgumentMatchers.anyString()); + + // reachable again: the next failure alerts straight away + Mockito.doReturn(report(200, 3000)).when(worker).checkCertificateExpiry(); + manager.processKubernetesClusterCertificates(cluster, new Date(now.getTime() + TimeUnit.HOURS.toMillis(30))); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.CERTIFICATE_CHECK_FAILED_ALERT_DETAIL)); + } + + @Test + public void scannerDoesNothingWhileThePluginIsDisabled() throws Exception { + setDefault(KubernetesClusterService.KubernetesServiceEnabled, "false"); + manager.new KubernetesClusterCertificateScanner().reallyRun(); + Mockito.verify(kubernetesClusterDao, Mockito.never()).findManagedKubernetesClustersInState(ArgumentMatchers.any()); + Mockito.verify(manager, Mockito.never()).createCertificateWorker(ArgumentMatchers.any()); + } + + @Test + public void rotationOwnedByThisRestartedServerIsMovedToAlertRightAway() { + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + details.put(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL, String.valueOf(now.getTime() - TimeUnit.MINUTES.toMillis(2))); + details.put(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL, String.valueOf(ManagementServerNode.getManagementServerId())); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL)); + } + + @Test + public void rotationThatJustFinishedGetsNoStartTime() { + // listed as RotatingCertificates, but finished and cleared its details before it was looked at + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + manager.recoverInterruptedCertificateRotations(now); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL)); + } + + @Test + public void rotationWithoutStartTimeIsTimedFromWhenFirstSeen() { + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + KubernetesClusterVO rotating = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(rotating.getState()).thenReturn(KubernetesCluster.State.RotatingCertificates); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(rotating); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + // the rotation's own start detail is never written here, it would race with the rotation writing it + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL)); + Assert.assertEquals(String.valueOf(now.getTime()), details.get(KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL)); + + // still without a start well past the timeout: it died before recording it + manager.recoverInterruptedCertificateRotations(new Date(now.getTime() + TimeUnit.HOURS.toMillis(3))); + Mockito.verify(manager).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_SEEN_DETAIL)); + } + + @Test + public void interruptedRotationIsMovedToAlertOnceStale() { + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.RotatingCertificates)) + .thenReturn(Collections.singletonList(cluster)); + // owned by another management server, which may still be running it + details.put(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL, String.valueOf(ManagementServerNode.getManagementServerId() + 1)); + details.put(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL, String.valueOf(now.getTime() - TimeUnit.MINUTES.toMillis(30))); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + + details.put(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL, String.valueOf(now.getTime() - TimeUnit.HOURS.toMillis(3))); + manager.recoverInterruptedCertificateRotations(now); + Mockito.verify(manager).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Assert.assertNull(details.get(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL)); + Assert.assertNotNull(details.get(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL)); + } + + private RotateKubernetesClusterCertificatesCmd rotateCmd() { + RotateKubernetesClusterCertificatesCmd cmd = Mockito.mock(RotateKubernetesClusterCertificatesCmd.class); + Mockito.when(cmd.getId()).thenReturn(CLUSTER_ID); + Mockito.when(cmd.getActualCommandName()).thenReturn("rotateKubernetesClusterCertificates"); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + return cmd; + } + + @Test + public void rotationApiValidation() { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + try { + RotateKubernetesClusterCertificatesCmd cmd = rotateCmd(); + Assert.assertEquals(cluster, manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Upgrading); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + // a failed rotation leaves the cluster in Alert, and it can be retried from there + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + Assert.assertEquals(cluster, manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(version.getSemanticVersion()).thenReturn("1.18.0"); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(version.getSemanticVersion()).thenReturn("1.32.5"); + Mockito.when(cluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(null); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterCertificateRotationParameters(cmd)); + } finally { + CallContext.unregister(); + } + } + + private KubernetesCluster.State next(KubernetesCluster.State state, KubernetesCluster.Event event) { + try { + return KubernetesCluster.State.getStateMachine().getNextState(state, event); + } catch (NoTransitionException e) { + return null; + } + } + + @Test + public void stateMachineAllowsRotationOnlyFromRunning() { + Assert.assertEquals(KubernetesCluster.State.RotatingCertificates, next(KubernetesCluster.State.Running, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.Upgrading, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertEquals(KubernetesCluster.State.RotatingCertificates, next(KubernetesCluster.State.Alert, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.Stopped, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.CertificateRotationRequested)); + Assert.assertNull(next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.UpgradeRequested)); + Assert.assertNull(next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.ScaleUpRequested)); + Assert.assertEquals(KubernetesCluster.State.Running, next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.OperationSucceeded)); + Assert.assertEquals(KubernetesCluster.State.Alert, next(KubernetesCluster.State.RotatingCertificates, KubernetesCluster.Event.OperationFailed)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java new file mode 100644 index 000000000000..733697c526b7 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterCertificateWorkerTest.java @@ -0,0 +1,526 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.codec.binary.Base64; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class KubernetesClusterCertificateWorkerTest { + + private static final long CLUSTER_ID = 7L; + + // see KubernetesClusterCertificateUtilTest + private static final String CHECK_EXPIRATION_OUTPUT = + "CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED\n" + + "admin.conf Nov 09, 2027 19:52 UTC 364d ca no\n" + + "apiserver Nov 09, 2027 19:52 UTC 364d ca no\n" + + "\n" + + "CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED\n" + + "ca Nov 06, 2036 19:52 UTC 9y no\n"; + private static final String CLIENT_CERTIFICATE_DATA = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnRENDQVNhZ0F3SUJBZ0lKQUtpUVNqdnZrSkN1TUFvR0NDcUdTTTQ5QkFNRE1EUXhGekFWQmdOVkJBb1QNCkRuTjVjM1JsYlRwdFlYTjBaWEp6TVJrd0Z3WURWUVFERXhCcmRXSmxjbTVsZEdWekxXRmtiV2x1TUI0WERUSTINCk1ERXdNVEF3TURBd01Gb1hEVEkzTURFd01qQXdNREF3TUZvd05ERVhNQlVHQTFVRUNoTU9jM2x6ZEdWdE9tMWgNCmMzUmxjbk14R1RBWEJnTlZCQU1URUd0MVltVnlibVYwWlhNdFlXUnRhVzR3V1RBVEJnY3Foa2pPUFFJQkJnZ3ENCmhrak9QUU1CQndOQ0FBUktSTGpVaDlYUGh4R0JvdUFNaFd5MmdmUWJZaG9HbEFwMVFkRDZKQ2tYTTM2YnBCSmINCjB0NGJRblJJU2pmUkpnWDVETHpBOFZhNC96Q3ZhVDdETW1jTG95RXdIekFkQmdOVkhRNEVGZ1FVVndXTXdmU0oNClRHQjBQLzJweDF4Nzh6eEVSQzR3Q2dZSUtvWkl6ajBFQXdNRFNBQXdSUUlnY1F6SGtsNk9WQm52T0tHNUNkNnMNClNxZmtMYU1oMFk1M0ovcHlBOFAwMm9FQ0lRRHB4ZkxEUFl5QzhLRVZvaW5YSUN2clo1SWVJNG5IRFUwYUtGZ2sNCml4ZGIwZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"; + // what the combined check command prints: kubeadm's tables, root's kubeconfig certificate line, and nothing for + // user.conf when there is none + private static final String CHECK_OUTPUT = CHECK_EXPIRATION_OUTPUT + "----- cks root kubeconfig -----\n" + + " client-certificate-data: " + CLIENT_CERTIFICATE_DATA + "\n" + + "----- cks user kubeconfig -----\n"; + + @Mock + private KubernetesCluster kubernetesCluster; + @Mock + private KubernetesClusterManagerImpl clusterManager; + @Mock + private KubernetesClusterDao kubernetesClusterDao; + @Mock + private KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Mock + private KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock + private KubernetesSupportedVersionVO version; + @Mock + private UserVm control1; + @Mock + private UserVm control2; + @Mock + private UserVm control3; + + private KubernetesClusterCertificateWorker worker; + private File scriptFile; + + @Before + public void setUp() throws IOException { + clusterManager.kubernetesClusterDao = kubernetesClusterDao; + clusterManager.kubernetesClusterDetailsDao = kubernetesClusterDetailsDao; + clusterManager.kubernetesSupportedVersionDao = kubernetesSupportedVersionDao; + Mockito.when(kubernetesCluster.getId()).thenReturn(CLUSTER_ID); + Mockito.when(kubernetesCluster.getName()).thenReturn("cks"); + Mockito.when(kubernetesCluster.getKubernetesVersionId()).thenReturn(3L); + Mockito.when(kubernetesSupportedVersionDao.findById(3L)).thenReturn(version); + Mockito.when(version.getSemanticVersion()).thenReturn("1.31.1"); + Mockito.when(control1.getHostName()).thenReturn("cks-control-1"); + Mockito.when(control2.getHostName()).thenReturn("cks-control-2"); + Mockito.when(control3.getHostName()).thenReturn("cks-control-3"); + + worker = Mockito.spy(new KubernetesClusterCertificateWorker(kubernetesCluster, clusterManager)); + scriptFile = File.createTempFile("rotate-kubernetes-certificates.sh", ".sh"); + Mockito.doNothing().when(worker).init(); + Mockito.doNothing().when(worker).resolveClusterEndpoints(); + Mockito.doReturn(Arrays.asList(control1, control2, control3)).when(worker).getControlNodeVMs(); + Mockito.doReturn(scriptFile).when(worker).retrieveScriptFile(KubernetesClusterCertificateWorker.ROTATION_SCRIPT_FILENAME); + Mockito.doReturn(true).when(worker).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + Mockito.doReturn(true).when(worker).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Mockito.doReturn(true).when(worker).waitForControlNodeReady(ArgumentMatchers.any(UserVm.class)); + Mockito.doReturn(false).when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + Mockito.doReturn(true).when(worker).isStillOwnRotation(); + Mockito.doReturn(true).when(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + Mockito.doNothing().when(worker).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.doNothing().when(worker).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + } + + @After + public void tearDown() { + scriptFile.delete(); + } + + @Test + public void rotationRenewsOneControlNodeAtATimeThenRefreshesTheKubeConfig() { + Assert.assertTrue(worker.rotateCertificates()); + + InOrder inOrder = Mockito.inOrder(worker, kubernetesClusterDetailsDao); + inOrder.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.CertificateRotationRequested); + inOrder.verify(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.eq(false)); + inOrder.verify(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.eq(false)); + inOrder.verify(worker).renewCertificatesOnControlNode(control1, 0); + inOrder.verify(worker).waitForControlNodeReady(control1); + inOrder.verify(worker).renewCertificatesOnControlNode(control2, 1); + inOrder.verify(worker).waitForControlNodeReady(control2); + inOrder.verify(worker).renewCertificatesOnControlNode(control3, 2); + inOrder.verify(worker).waitForControlNodeReady(control3); + inOrder.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + inOrder.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + inOrder.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + inOrder.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + // the API records its own event; only an automatic renewal records one here + Mockito.verify(worker, Mockito.never()).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Assert.assertFalse("the local copy of the script is cleaned up", scriptFile.exists()); + } + + @Test + public void automaticRotationRecordsAnEvent() { + worker.setAutomatic(true); + Assert.assertTrue(worker.rotateCertificates()); + Mockito.verify(worker).recordEvent(ArgumentMatchers.eq("KUBERNETES.CLUSTER.CERTIFICATES.ROTATE"), ArgumentMatchers.eq("INFO"), ArgumentMatchers.anyString()); + } + + private void assertRotationFailed() { + try { + worker.rotateCertificates(); + Assert.fail("rotation should have failed"); + } catch (CloudRuntimeException e) { + // expected + } + Mockito.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(worker).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.verify(kubernetesClusterDetailsDao).addDetail(ArgumentMatchers.eq(CLUSTER_ID), ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.eq(false)); + Mockito.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + Mockito.verify(kubernetesClusterDetailsDao).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_OWNER_DETAIL); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + // the API's own async job event records the failure; only an automatic renewal records one here + Mockito.verify(worker, Mockito.never()).recordEvent(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + } + + @Test + public void automaticRotationRecordsItsFailure() { + worker.setAutomatic(true); + Mockito.doReturn(false).when(worker).renewCertificatesOnControlNode(control1, 0); + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + Mockito.verify(worker).recordEvent(ArgumentMatchers.eq("KUBERNETES.CLUSTER.CERTIFICATES.ROTATE"), ArgumentMatchers.eq("ERROR"), ArgumentMatchers.anyString()); + } + + @Test + public void rotationOfAnUnreachableClusterRenewsEveryNodeBeforeWaiting() { + // expired certificates: with stacked etcd no node gets quorum until all are renewed + Mockito.doReturn(true).when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + + Assert.assertTrue(worker.rotateCertificates()); + + InOrder inOrder = Mockito.inOrder(worker); + inOrder.verify(worker).renewCertificatesOnControlNode(control1, 0); + inOrder.verify(worker).renewCertificatesOnControlNode(control2, 1); + inOrder.verify(worker).renewCertificatesOnControlNode(control3, 2); + inOrder.verify(worker).waitForControlNodeReady(control1); + inOrder.verify(worker).waitForControlNodeReady(control2); + inOrder.verify(worker).waitForControlNodeReady(control3); + inOrder.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + inOrder.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + } + + @Test + public void secondRotationOfTheSameClusterIsRefusedWithoutDisturbingTheFirst() { + Mockito.doAnswer(invocation -> { + // a second rotation starts while the first renews node 0 + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + Assert.assertTrue(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + return true; + }).when(worker).renewCertificatesOnControlNode(control1, 0); + + Assert.assertTrue(worker.rotateCertificates()); + Mockito.verify(worker, Mockito.times(1)).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.CertificateRotationRequested); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + } + + @Test + public void rotationFromAlertReturnsToAlertForTheStatusScanner() { + KubernetesClusterVO alertCluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(alertCluster.getState()).thenReturn(KubernetesCluster.State.Alert); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(alertCluster); + + Assert.assertTrue(worker.rotateCertificates()); + + // whatever else put it in Alert hasn't been checked; the status scanner reconciles it to Running + Mockito.verify(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(worker, Mockito.never()).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).addDetail(ArgumentMatchers.eq(CLUSTER_ID), + ArgumentMatchers.eq(KubernetesClusterCertificateWorker.ROTATION_FAILED_DETAIL), ArgumentMatchers.anyString(), ArgumentMatchers.anyBoolean()); + } + + @Test + public void rotationLeavesAClusterTakenMeanwhileInItsState() { + // e.g. moved to Alert as an interrupted rotation by another management server, and maybe retried since + Mockito.doReturn(false).when(worker).isStillOwnRotation(); + Assert.assertTrue(worker.rotateCertificates()); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).removeDetail(CLUSTER_ID, KubernetesClusterCertificateWorker.ROTATION_STARTED_DETAIL); + } + + @Test + public void rotationOfAnUnreachableClusterFailsWhenANodeDoesNotComeBack() { + Mockito.doReturn(true).when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + Mockito.doReturn(false).when(worker).waitForControlNodeReady(control2); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control3, 2); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void rotationStopsAtTheFirstControlNodeThatFailsToRenew() { + Mockito.doReturn(false).when(worker).renewCertificatesOnControlNode(control2, 1); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control1, 0); + Mockito.verify(worker).waitForControlNodeReady(control1); + Mockito.verify(worker).renewCertificatesOnControlNode(control2, 1); + Mockito.verify(worker, Mockito.never()).waitForControlNodeReady(control2); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.eq(control3), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void rotationStopsWhenAControlNodeDoesNotBecomeReady() { + Mockito.doReturn(false).when(worker).waitForControlNodeReady(control1); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control1, 0); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.eq(control2), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.eq(control3), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void rotationFailsWhenTheKubeConfigCannotBeRefreshed() { + Mockito.doReturn(false).when(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + assertRotationFailed(); + + Mockito.verify(worker).renewCertificatesOnControlNode(control3, 2); + } + + @Test + public void rotationFailsOnAnUnexpectedError() { + Mockito.doThrow(new CloudRuntimeException("no public IP")).when(worker).resolveClusterEndpoints(); + + assertRotationFailed(); + + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + } + + @Test + public void rotationIsRefusedUnlessTheClusterCanTransitionFromRunning() { + Mockito.doReturn(false).when(worker).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.CertificateRotationRequested); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Mockito.verify(worker, Mockito.never()).stateTransitTo(CLUSTER_ID, KubernetesCluster.Event.OperationFailed); + Mockito.verify(worker, Mockito.never()).sendAlert(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); + // a refused request leaves the running rotation's bookkeeping alone + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).addDetail(ArgumentMatchers.anyLong(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyBoolean()); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).removeDetail(ArgumentMatchers.anyLong(), ArgumentMatchers.anyString()); + } + + @Test + public void rotationIsActiveWhileRunning() { + Mockito.doAnswer(invocation -> { + Assert.assertTrue(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + return true; + }).when(worker).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Assert.assertTrue(worker.rotateCertificates()); + Assert.assertFalse(KubernetesClusterCertificateWorker.isRotationActive(CLUSTER_ID)); + } + + @Test + public void rotationIsRefusedBeforeKubernetes120() { + Mockito.when(version.getSemanticVersion()).thenReturn("1.19.16"); + + Assert.assertThrows(CloudRuntimeException.class, () -> worker.rotateCertificates()); + + Mockito.verify(worker, Mockito.never()).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + Mockito.verify(worker, Mockito.never()).renewCertificatesOnControlNode(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + } + + @Test + public void certificateManagementVersionGate() { + Assert.assertFalse(KubernetesClusterCertificateWorker.isCertificateManagementSupported(null)); + Mockito.when(version.getSemanticVersion()).thenReturn("1.19.16"); + Assert.assertFalse(KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)); + Mockito.when(version.getSemanticVersion()).thenReturn("1.20.0"); + Assert.assertTrue(KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)); + Mockito.when(version.getSemanticVersion()).thenReturn("1.33.1"); + Assert.assertTrue(KubernetesClusterCertificateWorker.isCertificateManagementSupported(version)); + } + + private void storeKubeConfig(String certificateData) { + KubernetesClusterDetailsVO detail = Mockito.mock(KubernetesClusterDetailsVO.class); + String kubeConfig = "users:\n- name: kubernetes-admin\n user:\n client-certificate-data: " + certificateData + "\n"; + Mockito.when(detail.getValue()).thenReturn(Base64.encodeBase64String(kubeConfig.getBytes(StandardCharsets.UTF_8))); + Mockito.when(kubernetesClusterDetailsDao.findDetail(CLUSTER_ID, KubernetesClusterActionWorker.KUBE_CONFIG_DATA_DETAIL)).thenReturn(detail); + } + + @Test + public void checkReadsEveryControlNodeAndRefreshesAStaleStoredKubeConfig() throws Exception { + // the stored kubeconfig's certificate expires in Jan 2027, the cluster's admin.conf in Nov 2027 + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, CHECK_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.contains(KubernetesClusterCertificateUtil.CHECK_EXPIRATION_COMMAND), ArgumentMatchers.anyInt()); + Mockito.doReturn(new Pair<>(false, "connection refused")).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2224), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals(List.of("cks-control-3"), report.getUncheckedNodes()); + // 3 certificates from kubeadm and root's kubeconfig, on two nodes + Assert.assertEquals(8, report.getEntries().size()); + // root's kubeconfig expires first: renewing syncs it with admin.conf + Assert.assertEquals("/root/.kube/config on cks-control-1", report.getFirstLeafExpiry().getName()); + Assert.assertEquals("ca on cks-control-1", report.getFirstAuthorityExpiry().getName()); + Mockito.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void checkComparesTheStoredKubeConfigWithUserConfWhenThereIsOne() throws Exception { + // getKubernetesClusterConfig hands out user.conf; its certificate is the one stored, so nothing is stale + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + String withUserConf = CHECK_OUTPUT + "user.conf present\n client-certificate-data: " + CLIENT_CERTIFICATE_DATA + "\n"; + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, withUserConf)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals(12, report.getEntries().size()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + // a user.conf without a client certificate (e.g. a token) can't be compared, so it is left alone too + Mockito.doReturn(new Pair<>(true, CHECK_OUTPUT + "user.conf present\n")).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + worker.checkCertificateExpiry(); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void expiryForRescueLooksAtEveryControlNode() throws Exception { + String expired = CHECK_EXPIRATION_OUTPUT.replace("apiserver Nov 09, 2027 19:52 UTC 364d ", + "apiserver Jan 01, 2020 00:00 UTC "); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, CHECK_EXPIRATION_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2222), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doThrow(new java.io.IOException("timeout")).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2224), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doReturn(new Pair<>(true, CHECK_EXPIRATION_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2223), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doCallRealMethod().when(worker).areCertificatesExpired(ArgumentMatchers.anyList()); + List nodes = Arrays.asList(control1, control2, control3); + + Assert.assertFalse(worker.areCertificatesExpired(nodes)); + + // an expired admin client certificate breaks no control plane component: no reason to restart every node at once + String adminExpired = CHECK_EXPIRATION_OUTPUT.replace("admin.conf Nov 09, 2027 19:52 UTC 364d ", + "admin.conf Jan 01, 2020 00:00 UTC "); + Assert.assertNotEquals(CHECK_EXPIRATION_OUTPUT, adminExpired); + Mockito.doReturn(new Pair<>(true, adminExpired)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2223), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Assert.assertFalse(worker.areCertificatesExpired(nodes)); + + // the first node already renewed by an interrupted rescue, the second still expired + Mockito.doReturn(new Pair<>(false, expired)).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2223), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Assert.assertTrue(worker.areCertificatesExpired(nodes)); + } + + @Test + public void rootKubeConfigIsNotRenewableInExternalCaMode() throws Exception { + String external = CHECK_OUTPUT.replace("ca no\n", "ca yes\n"); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(ArgumentMatchers.any(UserVm.class), ArgumentMatchers.anyInt()); + Mockito.doReturn(new Pair<>(true, external)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals("/root/.kube/config on cks-control-1", report.getFirstLeafExpiry().getName()); + Assert.assertNull(report.getFirstRenewableLeafExpiry()); + } + + @Test + public void checkSkipsTheKubeConfigRefreshWhenTheFirstControlNodeIsDown() throws Exception { + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + Mockito.doReturn(new Pair<>("10.1.1.1", 2222)).when(worker).getControlNodeSshEndpoint(control1, 0); + Mockito.doReturn(new Pair<>("10.1.1.1", 2223)).when(worker).getControlNodeSshEndpoint(control2, 1); + Mockito.doReturn(new Pair<>("10.1.1.1", 2224)).when(worker).getControlNodeSshEndpoint(control3, 2); + Mockito.doReturn(new Pair<>(true, CHECK_OUTPUT)).when(worker).runOnNode(ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + Mockito.doThrow(new java.io.IOException("timeout")).when(worker).runOnNode(ArgumentMatchers.eq("10.1.1.1"), ArgumentMatchers.eq(2222), + ArgumentMatchers.anyString(), ArgumentMatchers.anyInt()); + + CertificateExpiryReport report = worker.checkCertificateExpiry(); + + Assert.assertEquals(List.of("cks-control-1"), report.getUncheckedNodes()); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + @Test + public void storedKubeConfigIsLeftAloneWhenCurrent() { + storeKubeConfig(CLIENT_CERTIFICATE_DATA); + Date storedExpiry = worker.getStoredKubeConfigExpiry(); + Assert.assertNotNull(storedExpiry); + + worker.refreshStaleKubeConfigData(storedExpiry); + worker.refreshStaleKubeConfigData(new Date(storedExpiry.getTime() - 1000)); + worker.refreshStaleKubeConfigData(null); + Mockito.verify(worker, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + worker.refreshStaleKubeConfigData(new Date(storedExpiry.getTime() + 1000)); + Mockito.verify(worker).refreshKubeConfigData(ArgumentMatchers.anyLong()); + } + + private PortForwardingRuleVO sshRule(long ipId, int sourcePort, int destinationPort) { + PortForwardingRuleVO rule = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(ipId); + Mockito.when(rule.getSourcePortStart()).thenReturn(sourcePort); + Mockito.when(rule.getDestinationPortStart()).thenReturn(destinationPort); + Mockito.when(rule.getState()).thenReturn(FirewallRule.State.Active); + return rule; + } + + @Test + public void controlNodeSshEndpoint() { + PortForwardingRulesDao portForwardingRulesDao = Mockito.mock(PortForwardingRulesDao.class); + IPAddressDao ipAddressDao = Mockito.mock(IPAddressDao.class); + worker.portForwardingRulesDao = portForwardingRulesDao; + worker.ipAddressDao = ipAddressDao; + IPAddressVO clusterIp = Mockito.mock(IPAddressVO.class); + Mockito.when(clusterIp.getAddress()).thenReturn(new Ip("192.0.2.10")); + IPAddressVO otherIp = Mockito.mock(IPAddressVO.class); + Mockito.when(otherIp.getAddress()).thenReturn(new Ip("192.0.2.99")); + Mockito.when(ipAddressDao.findById(1L)).thenReturn(clusterIp); + Mockito.when(ipAddressDao.findById(2L)).thenReturn(otherIp); + Mockito.when(control1.getId()).thenReturn(101L); + Mockito.when(control2.getId()).thenReturn(102L); + Mockito.when(control2.getPrivateIpAddress()).thenReturn("10.1.1.12"); + + // directly reachable network: the VM's own address + worker.publicIpAddress = "10.1.1.11"; + worker.sshPort = 22; + Assert.assertEquals(new Pair<>("10.1.1.12", 22), worker.getControlNodeSshEndpoint(control2, 1)); + + // behind the cluster public IP: the VM's SSH port forwarding rule on that IP, not a user's on another one + worker.publicIpAddress = "192.0.2.10"; + worker.sshPort = 2222; + List rules = Arrays.asList(sshRule(2L, 2022, 22), sshRule(1L, 8080, 80), sshRule(1L, 2224, 22)); + Mockito.when(portForwardingRulesDao.listByVm(102L)).thenReturn(rules); + Assert.assertEquals(new Pair<>("192.0.2.10", 2224), worker.getControlNodeSshEndpoint(control2, 1)); + + // no rule (external load balancer): by position + Mockito.when(portForwardingRulesDao.listByVm(101L)).thenReturn(Collections.emptyList()); + Assert.assertEquals(new Pair<>("192.0.2.10", 2222), worker.getControlNodeSshEndpoint(control1, 0)); + Assert.assertEquals(new Pair<>("192.0.2.10", 2223), worker.getControlNodeSshEndpoint(control1, 1)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java index ff8f875d06e1..888af3a84121 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java @@ -18,19 +18,27 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -45,6 +53,10 @@ public class KubernetesClusterUpgradeWorkerTest { private KubernetesClusterManagerImpl clusterManager; @Mock private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Mock + private KubernetesClusterDao kubernetesClusterDao; + @Mock + private NetworkDao networkDao; private KubernetesClusterUpgradeWorker worker; @@ -80,4 +92,64 @@ public void testFilterOutManualUpgradeNodesFromClusterUpgrade() { Assert.assertTrue(ids.contains(controlNodeId) && ids.contains(workerNode1Id)); Assert.assertFalse(ids.contains(workerNode2Id)); } + + private KubernetesClusterUpgradeWorker prepareUpgrade(boolean clusterUpdated) { + long clusterId = 5L; + Mockito.when(kubernetesCluster.getId()).thenReturn(clusterId); + Mockito.lenient().when(kubernetesCluster.getNetworkId()).thenReturn(9L); + Mockito.lenient().when(networkDao.findById(9L)).thenReturn(Mockito.mock(NetworkVO.class)); + KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(kubernetesClusterDao.findById(clusterId)).thenReturn(clusterVO); + Mockito.when(kubernetesClusterDao.update(clusterId, clusterVO)).thenReturn(clusterUpdated); + UserVm controlNode = Mockito.mock(UserVm.class); + worker.kubernetesClusterDao = kubernetesClusterDao; + worker.networkDao = networkDao; + KubernetesClusterUpgradeWorker spy = Mockito.spy(worker); + Mockito.doNothing().when(spy).init(); + Mockito.doReturn(new Pair<>("10.1.1.10", 2222)).when(spy).getKubernetesClusterServerIpSshPort(null); + Mockito.doReturn(Collections.singletonList(controlNode)).when(spy).getKubernetesClusterVMs(); + Mockito.doNothing().when(spy).filterOutManualUpgradeNodesFromClusterUpgrade(); + Mockito.doNothing().when(spy).retrieveScriptFiles(); + Mockito.doReturn(true).when(spy).stateTransitTo(ArgumentMatchers.anyLong(), ArgumentMatchers.any(KubernetesCluster.Event.class)); + Mockito.doNothing().when(spy).attachIsoKubernetesVMs(ArgumentMatchers.anyList(), ArgumentMatchers.any()); + Mockito.doNothing().when(spy).upgradeKubernetesClusterNodes(); + Mockito.doNothing().when(spy).detachIsoKubernetesVMs(ArgumentMatchers.anyList()); + return spy; + } + + @Test + public void testUpgradeRefreshesTheStoredKubeConfigBeforeCompleting() { + KubernetesClusterUpgradeWorker spy = prepareUpgrade(true); + Mockito.doReturn(true).when(spy).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + Assert.assertTrue(spy.upgradeCluster()); + + InOrder inOrder = Mockito.inOrder(spy); + inOrder.verify(spy).upgradeKubernetesClusterNodes(); + inOrder.verify(spy).refreshKubeConfigData(ArgumentMatchers.anyLong()); + inOrder.verify(spy).stateTransitTo(5L, KubernetesCluster.Event.OperationSucceeded); + // the kubeconfig must point at the cluster's public endpoint + Assert.assertEquals("10.1.1.10", spy.apiPublicIpAddress); + } + + @Test + public void testUpgradeSucceedsWhenTheKubeConfigRefreshFails() { + KubernetesClusterUpgradeWorker spy = prepareUpgrade(true); + Mockito.doReturn(false).when(spy).refreshKubeConfigData(ArgumentMatchers.anyLong()); + + Assert.assertTrue(spy.upgradeCluster()); + + Mockito.verify(spy).stateTransitTo(5L, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(spy, Mockito.never()).stateTransitTo(5L, KubernetesCluster.Event.OperationFailed); + } + + @Test + public void testFailedUpgradeDoesNotRefreshTheKubeConfig() { + KubernetesClusterUpgradeWorker spy = prepareUpgrade(false); + + Assert.assertFalse(spy.upgradeCluster()); + + Mockito.verify(spy, Mockito.never()).refreshKubeConfigData(ArgumentMatchers.anyLong()); + Mockito.verify(spy).stateTransitTo(5L, KubernetesCluster.Event.OperationFailed); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java new file mode 100644 index 000000000000..f98aecb33d2a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterCertificateUtilTest.java @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.utils; + +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiry; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterCertificateUtil.CertificateExpiryReport; + +public class KubernetesClusterCertificateUtilTest { + + // Output of kubeadm v1.31 on a stacked etcd control node + static final String CHECK_EXPIRATION_OUTPUT = + "[check-expiration] Reading configuration from the cluster...\n" + + "[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'\n" + + "\n" + + "CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED\n" + + "admin.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "apiserver Nov 09, 2027 19:52 UTC 364d ca no \n" + + "apiserver-etcd-client Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "apiserver-kubelet-client Nov 09, 2027 19:52 UTC 364d ca no \n" + + "controller-manager.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "etcd-healthcheck-client Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "etcd-peer Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "etcd-server Nov 09, 2027 19:52 UTC 364d etcd-ca no \n" + + "front-proxy-client Nov 01, 2027 08:05 UTC 356d front-proxy-ca no \n" + + "scheduler.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "super-admin.conf Nov 09, 2027 19:52 UTC 364d ca no \n" + + "\n" + + "CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED\n" + + "ca Nov 06, 2036 19:52 UTC 9y no \n" + + "etcd-ca Nov 06, 2036 19:52 UTC 9y no \n" + + "front-proxy-ca Mar 03, 2035 10:00 UTC 8y no \n"; + + // Certificate of a kubeconfig user, valid until Jan 2 00:00:00 2027 GMT + static final String CLIENT_CERTIFICATE_DATA = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnRENDQVNhZ0F3SUJBZ0lKQUtpUVNqdnZrSkN1TUFvR0NDcUdTTTQ5QkFNRE1EUXhGekFWQmdOVkJBb1QNCkRuTjVjM1JsYlRwdFlYTjBaWEp6TVJrd0Z3WURWUVFERXhCcmRXSmxjbTVsZEdWekxXRmtiV2x1TUI0WERUSTINCk1ERXdNVEF3TURBd01Gb1hEVEkzTURFd01qQXdNREF3TUZvd05ERVhNQlVHQTFVRUNoTU9jM2x6ZEdWdE9tMWgNCmMzUmxjbk14R1RBWEJnTlZCQU1URUd0MVltVnlibVYwWlhNdFlXUnRhVzR3V1RBVEJnY3Foa2pPUFFJQkJnZ3ENCmhrak9QUU1CQndOQ0FBUktSTGpVaDlYUGh4R0JvdUFNaFd5MmdmUWJZaG9HbEFwMVFkRDZKQ2tYTTM2YnBCSmINCjB0NGJRblJJU2pmUkpnWDVETHpBOFZhNC96Q3ZhVDdETW1jTG95RXdIekFkQmdOVkhRNEVGZ1FVVndXTXdmU0oNClRHQjBQLzJweDF4Nzh6eEVSQzR3Q2dZSUtvWkl6ajBFQXdNRFNBQXdSUUlnY1F6SGtsNk9WQm52T0tHNUNkNnMNClNxZmtMYU1oMFk1M0ovcHlBOFAwMm9FQ0lRRHB4ZkxEUFl5QzhLRVZvaW5YSUN2clo1SWVJNG5IRFUwYUtGZ2sNCml4ZGIwZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"; + static final Date CLIENT_CERTIFICATE_EXPIRY = utc(2027, 1, 2, 0, 0); + + static Date utc(int year, int month, int day, int hour, int minute) { + return Date.from(ZonedDateTime.parse(String.format("%04d-%02d-%02dT%02d:%02d:00Z", year, month, day, hour, minute)).toInstant()); + } + + static String kubeConfig(String certificateData) { + return "apiVersion: v1\n" + + "clusters:\n" + + "- cluster:\n" + + " certificate-authority-data: Y2E=\n" + + " server: https://10.1.1.10:6443\n" + + " name: kubernetes\n" + + "kind: Config\n" + + "users:\n" + + "- name: kubernetes-admin\n" + + " user:\n" + + " client-certificate-data: " + certificateData + "\n" + + " client-key-data: a2V5\n"; + } + + private CertificateExpiry find(List entries, String name) { + return entries.stream().filter(e -> e.getName().equals(name)).findFirst().orElse(null); + } + + @Test + public void parseCheckExpirationOutputReadsBothTables() { + List entries = KubernetesClusterCertificateUtil.parseCheckExpirationOutput(CHECK_EXPIRATION_OUTPUT); + Assert.assertEquals(14, entries.size()); + Assert.assertEquals(11, entries.stream().filter(e -> !e.isAuthority()).count()); + Assert.assertEquals(3, entries.stream().filter(CertificateExpiry::isAuthority).count()); + + CertificateExpiry admin = find(entries, "admin.conf"); + Assert.assertFalse(admin.isAuthority()); + Assert.assertFalse(admin.isMissing()); + Assert.assertFalse(admin.isExternallyManaged()); + Assert.assertEquals(utc(2027, 11, 9, 19, 52), admin.getExpires()); + + Assert.assertEquals(utc(2027, 11, 1, 8, 5), find(entries, "front-proxy-client").getExpires()); + CertificateExpiry ca = find(entries, "ca"); + Assert.assertTrue(ca.isAuthority()); + Assert.assertEquals(utc(2036, 11, 6, 19, 52), ca.getExpires()); + Assert.assertEquals(utc(2035, 3, 3, 10, 0), find(entries, "front-proxy-ca").getExpires()); + } + + @Test + public void parseCheckExpirationOutputHandlesMissingExpiredAndExternalCertificates() { + String output = "CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED\n" + + "admin.conf Jan 05, 2026 10:00 UTC ca no\n" + + "apiserver Feb 07, 2027 10:00 UTC 89d ca no\n" + + "!MISSING! apiserver-etcd-client\n" + + "front-proxy-client Dec 24, 2026 10:00 UTC 44d front-proxy-ca yes\n" + + "\n" + + "CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED\n" + + "ca Nov 06, 2036 19:52 UTC 9y no\n" + + "!MISSING! etcd-ca\n" + + "some unexpected line\n"; + List entries = KubernetesClusterCertificateUtil.parseCheckExpirationOutput(output); + Assert.assertEquals(6, entries.size()); + Assert.assertEquals(utc(2026, 1, 5, 10, 0), find(entries, "admin.conf").getExpires()); + CertificateExpiry missing = find(entries, "apiserver-etcd-client"); + Assert.assertTrue(missing.isMissing()); + Assert.assertFalse(missing.isAuthority()); + Assert.assertTrue(find(entries, "front-proxy-client").isExternallyManaged()); + Assert.assertTrue(find(entries, "etcd-ca").isMissing()); + Assert.assertTrue(find(entries, "etcd-ca").isAuthority()); + } + + @Test + public void parseCheckExpirationOutputOfAFailureIsEmpty() { + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput(null).isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput("").isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput("sudo: /opt/bin/kubeadm: command not found").isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseCheckExpirationOutput( + "error execution phase check-expiration: unknown command \"certs\" for \"kubeadm\"").isEmpty()); + } + + @Test + public void reportPicksTheFirstExpiryOfEachKind() { + CertificateExpiryReport report = new CertificateExpiryReport(); + report.add("control-1", KubernetesClusterCertificateUtil.parseCheckExpirationOutput(CHECK_EXPIRATION_OUTPUT)); + report.add("control-2", Arrays.asList( + new CertificateExpiry("apiserver", utc(2027, 2, 1, 0, 0), false, false), + new CertificateExpiry("front-proxy-client", utc(2026, 12, 1, 0, 0), false, true), + new CertificateExpiry("apiserver-etcd-client", null, false, false))); + report.addUncheckedNode("control-3"); + + Assert.assertFalse(report.isEmpty()); + Assert.assertEquals(Collections.singletonList("control-3"), report.getUncheckedNodes()); + Assert.assertEquals("front-proxy-client on control-2", report.getFirstLeafExpiry().getName()); + Assert.assertEquals("apiserver on control-2", report.getFirstRenewableLeafExpiry().getName()); + Assert.assertEquals(utc(2027, 2, 1, 0, 0), report.getFirstRenewableLeafExpiry().getExpires()); + Assert.assertEquals("front-proxy-ca on control-1", report.getFirstAuthorityExpiry().getName()); + } + + @Test + public void emptyReportHasNoExpiry() { + CertificateExpiryReport report = new CertificateExpiryReport(); + Assert.assertTrue(report.isEmpty()); + Assert.assertNull(report.getFirstLeafExpiry()); + Assert.assertNull(report.getFirstRenewableLeafExpiry()); + Assert.assertNull(report.getFirstAuthorityExpiry()); + } + + @Test + public void kubeConfigClientCertificateExpiry() { + Assert.assertEquals(CLIENT_CERTIFICATE_EXPIRY, KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(kubeConfig(CLIENT_CERTIFICATE_DATA))); + // what ROOT_KUBECONFIG_CERTIFICATE_COMMAND prints + Assert.assertEquals(CLIENT_CERTIFICATE_EXPIRY, KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry( + " client-certificate-data: " + CLIENT_CERTIFICATE_DATA + "\n")); + } + + @Test + public void kubeConfigWithoutUsableClientCertificate() { + Assert.assertNull(KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(null)); + Assert.assertNull(KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry("users:\n- name: x\n user:\n token: abc\n")); + Assert.assertNull(KubernetesClusterCertificateUtil.getKubeConfigClientCertificateExpiry(kubeConfig("bm90IGEgY2VydGlmaWNhdGU="))); + } + + @Test + public void parseAlertThresholds() { + Assert.assertEquals(Arrays.asList(60, 30), KubernetesClusterCertificateUtil.parseAlertThresholds("60,30")); + Assert.assertEquals(Arrays.asList(90, 30, 7), KubernetesClusterCertificateUtil.parseAlertThresholds(" 7, 30 ,90,30")); + Assert.assertEquals(Collections.singletonList(14), KubernetesClusterCertificateUtil.parseAlertThresholds("abc,0,-5,14,")); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseAlertThresholds("").isEmpty()); + Assert.assertTrue(KubernetesClusterCertificateUtil.parseAlertThresholds(null).isEmpty()); + } + + @Test + public void thresholdToAlert() { + List thresholds = Arrays.asList(60, 30); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(61, thresholds, null)); + Assert.assertEquals(Integer.valueOf(60), KubernetesClusterCertificateUtil.getThresholdToAlert(60, thresholds, null)); + // already alerted for 60: nothing until 30 is reached + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(45, thresholds, 60)); + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getThresholdToAlert(30, thresholds, 60)); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(10, thresholds, 30)); + // first seen inside both windows: one alert, for the smaller threshold + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getThresholdToAlert(12, thresholds, null)); + // expiry itself alerts once more, after the thresholds + Assert.assertEquals(Integer.valueOf(0), KubernetesClusterCertificateUtil.getThresholdToAlert(-3, thresholds, null)); + Assert.assertEquals(Integer.valueOf(0), KubernetesClusterCertificateUtil.getThresholdToAlert(-3, thresholds, 30)); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(-10, thresholds, 0)); + Assert.assertNull(KubernetesClusterCertificateUtil.getThresholdToAlert(5, Collections.emptyList(), null)); + } + + @Test + public void alertStateIsTiedToTheExpiryDate() { + Date expiry = utc(2027, 1, 2, 0, 0); + String state = KubernetesClusterCertificateUtil.buildAlertState(expiry, 30); + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, expiry)); + // renewed: the expiry moved, so the thresholds start over + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, utc(2028, 1, 2, 0, 0))); + // another control node's copy, renewed minutes apart, is the same expiry + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, utc(2027, 1, 2, 0, 7))); + Assert.assertEquals(Integer.valueOf(30), KubernetesClusterCertificateUtil.getLastAlertedThreshold(state, utc(2026, 12, 30, 0, 0))); + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold(null, expiry)); + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold("garbage", expiry)); + Assert.assertNull(KubernetesClusterCertificateUtil.getLastAlertedThreshold("1:x", expiry)); + } + + @Test + public void daysUntilAndAutoRenew() { + Date now = utc(2026, 11, 9, 12, 0); + Assert.assertEquals(30, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 12, 9, 12, 0), now)); + Assert.assertEquals(29, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 12, 9, 11, 59), now)); + Assert.assertEquals(0, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 11, 9, 13, 0), now)); + Assert.assertEquals(-1, KubernetesClusterCertificateUtil.daysUntil(utc(2026, 11, 9, 11, 0), now)); + + Assert.assertTrue(KubernetesClusterCertificateUtil.isAutoRenewDue(30, 30)); + Assert.assertTrue(KubernetesClusterCertificateUtil.isAutoRenewDue(-2, 30)); + Assert.assertFalse(KubernetesClusterCertificateUtil.isAutoRenewDue(31, 30)); + Assert.assertFalse(KubernetesClusterCertificateUtil.isAutoRenewDue(5, 0)); + // capped, so a freshly renewed certificate (about 365 days) is never due again right away + Assert.assertFalse(KubernetesClusterCertificateUtil.isAutoRenewDue(364, 400)); + Assert.assertTrue(KubernetesClusterCertificateUtil.isAutoRenewDue(180, 400)); + } +}