CKS: check and renew the control plane certificates

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java

@@ -47,6 +47,7 @@ enum Event {
         AddNodeRequested,
         RemoveNodeRequested,
         UpgradeRequested,
+        CertificateRotationRequested,
         OperationSucceeded,
         OperationFailed,
         CreateFailed,
@@ -62,6 +63,7 @@ enum State {
         Scaling("Transient state in which resources are either getting scaled up/down"),
         ScalingStoppedCluster("Transient state in which the service offerings of stopped clusters are getting scaled"),
         Upgrading("Transient state in which cluster is getting upgraded"),
+        RotatingCertificates("Transient state in which the control plane certificates of the cluster are getting renewed"),
         Importing("Transient state in which additional nodes are added as worker nodes to a cluster"),
         RemovingNodes("Transient state in which additional nodes are removed from a cluster"),
         Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."),
@@ -106,6 +108,12 @@ enum State {
             s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert);
 
+            s_fsm.addTransition(State.Running, Event.CertificateRotationRequested, State.RotatingCertificates);
+            // lets a failed or interrupted rotation be retried, and rescues a cluster whose certificates already expired
+            s_fsm.addTransition(State.Alert, Event.CertificateRotationRequested, State.RotatingCertificates);
+            s_fsm.addTransition(State.RotatingCertificates, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.RotatingCertificates, Event.OperationFailed, State.Alert);
+
             s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing);
             s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing);
             s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running);

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -86,6 +86,7 @@ private AlertType(short type, String name, boolean isDefault) {
         public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_FAILED = new AlertType((short)35, "ALERT.BACKUP.VALIDATION.FAILED", true, true);
         public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_UNABLE_TO_VALIDATE = new AlertType((short)36, "ALERT.BACKUP.VALIDATION.UNABLE.TO.VALIDATE", true, true);
         public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_CLEANUP_FAILED = new AlertType((short)37, "ALERT.BACKUP.VALIDATION.CLEANUP_FAILED", true, true);
+        public static final AlertType ALERT_TYPE_KUBERNETES_CLUSTER_CERTIFICATE = new AlertType((short)38, "ALERT.KUBERNETES.CLUSTER.CERTIFICATE", true, true);
 
         public short getType() {
             return type;

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java

@@ -26,4 +26,6 @@ public class KubernetesClusterEventTypes {
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD";
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE";
     public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE";
+    public static final String EVENT_KUBERNETES_CLUSTER_CERTIFICATES_ROTATE = "KUBERNETES.CLUSTER.CERTIFICATES.ROTATE";
+    public static final String EVENT_KUBERNETES_CLUSTER_CERTIFICATES_EXPIRY = "KUBERNETES.CLUSTER.CERTIFICATES.EXPIRY";
 }

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java

@@ -29,6 +29,7 @@
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd;
+import org.apache.cloudstack.api.command.user.kubernetes.cluster.RotateKubernetesClusterCertificatesCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd;
@@ -143,6 +144,43 @@ public interface KubernetesClusterService extends PluggableService, Configurable
             "Start port for Port forwarding rules for etcd nodes",
             true,
             KubernetesServiceEnabled.key());
+    ConfigKey<Integer> KubernetesClusterCertificateCheckInterval = new ConfigKey<>("Advanced", Integer.class,
+            "cloud.kubernetes.cluster.certificate.check.interval",
+            "6",
+            "Interval (in hours) at which the control plane certificate expiry of each running CloudManaged Kubernetes cluster " +
+                    "is checked with 'kubeadm certs check-expiration'. 0 disables the check and the automatic renewal",
+            true,
+            KubernetesServiceEnabled.key());
+    ConfigKey<String> KubernetesClusterCertificateExpiryAlertDays = new ConfigKey<>("Advanced", String.class,
+            "cloud.kubernetes.cluster.certificate.expiry.alert.days",
+            "60,30",
+            "Comma-separated list of days before a Kubernetes cluster control plane certificate (or certificate authority) " +
+                    "expires at which an alert and an event are raised. Each threshold alerts once per expiry date",
+            true,
+            KubernetesServiceEnabled.key());
+    ConfigKey<Boolean> KubernetesClusterCertificateAutoRenewEnabled = new ConfigKey<>("Advanced", Boolean.class,
+            "cloud.kubernetes.cluster.certificate.auto.renew.enabled",
+            "true",
+            "Indicates whether the control plane certificates of a CloudManaged Kubernetes cluster are renewed automatically, " +
+                    "one control node at a time, when they are about to expire. Requires Kubernetes 1.20 or later",
+            true,
+            ConfigKey.Scope.Account,
+            KubernetesServiceEnabled.key());
+    ConfigKey<Integer> KubernetesClusterCertificateAutoRenewDays = new ConfigKey<>("Advanced", Integer.class,
+            "cloud.kubernetes.cluster.certificate.auto.renew.days",
+            "30",
+            "Number of days before the first control plane certificate of a Kubernetes cluster expires at which the " +
+                    "certificates are renewed automatically, when cloud.kubernetes.cluster.certificate.auto.renew.enabled is true. " +
+                    "Values above 180 are treated as 180",
+            true,
+            ConfigKey.Scope.Account,
+            KubernetesServiceEnabled.key());
+    ConfigKey<Long> KubernetesClusterCertificateRotationTimeout = new ConfigKey<>("Advanced", Long.class,
+            "cloud.kubernetes.cluster.certificate.rotation.timeout",
+            "3600",
+            "Timeout interval (in seconds) in which the certificate rotation of a Kubernetes cluster should be completed",
+            true,
+            KubernetesServiceEnabled.key());
 
     KubernetesCluster findById(final Long id);
 
@@ -172,6 +210,8 @@ public interface KubernetesClusterService extends PluggableService, Configurable
 
     boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException;
 
+    boolean rotateKubernetesClusterCertificates(RotateKubernetesClusterCertificatesCmd cmd) throws CloudRuntimeException;
+
     boolean updateKubernetesClusterAffinityGroups(UpdateKubernetesClusterAffinityGroupCmd cmd) throws CloudRuntimeException;
 
     boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

@@ -158,6 +158,8 @@ public class KubernetesClusterActionWorker {
     public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
     public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes";
     public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks";
+    public static final String KUBE_CONFIG_DATA_DETAIL = "kubeConfigData";
+    protected static final long KUBE_CONFIG_REFRESH_TIMEOUT_MS = 2 * 60 * 1000L;
 
     protected Logger logger = LogManager.getLogger(getClass());
 
@@ -756,6 +758,83 @@ protected List<UserVm> getKubernetesClusterVMs() {
         return vmList;
     }
 
+    /**
+     * @return the control node VMs of the cluster, ordered by VM ID, which is the order they were created and
+     * received their SSH port forwarding rules in
+     */
+    protected List<UserVm> getControlNodeVMs() {
+        List<UserVm> controlVms = new ArrayList<>();
+        List<KubernetesClusterVmMapVO> vmMaps = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), CONTROL);
+        List<Long> controlVmIds;
+        if (CollectionUtils.isNotEmpty(vmMaps)) {
+            controlVmIds = vmMaps.stream().map(KubernetesClusterVmMapVO::getVmId).sorted().collect(Collectors.toList());
+        } else {
+            // clusters created before the control node flag was recorded: by position, after any etcd nodes
+            List<KubernetesClusterVmMapVO> allVmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
+            if (CollectionUtils.isEmpty(allVmMaps)) {
+                return controlVms;
+            }
+            long etcdNodes = Objects.requireNonNullElse(kubernetesCluster.getEtcdNodeCount(), 0L);
+            controlVmIds = allVmMaps.stream().map(KubernetesClusterVmMapVO::getVmId).sorted()
+                    .skip(etcdNodes).limit(Math.max(1, kubernetesCluster.getControlNodeCount())).collect(Collectors.toList());
+        }
+        for (Long vmId : controlVmIds) {
+            UserVm vm = userVmDao.findById(vmId);
+            if (vm != null) {
+                controlVms.add(vm);
+            }
+        }
+        return controlVms;
+    }
+
+    /**
+     * Returns the address and port to SSH to one control node. {@link #publicIpAddress} and {@link #sshPort} must be
+     * set. On directly reachable (shared or routed) networks it is the VM's own address. Otherwise it is the cluster's
+     * public IP with the port of the SSH port forwarding rule of the VM, falling back to the port assigned by position
+     * (control nodes are the first nodes behind {@link #CLUSTER_NODES_DEFAULT_START_SSH_PORT}) when there is no rule,
+     * e.g. behind an external load balancer.
+     */
+    protected Pair<String, Integer> getControlNodeSshEndpoint(UserVm vm, int controlNodeIndex) {
+        if (sshPort == DEFAULT_SSH_PORT) {
+            return new Pair<>(vm.getPrivateIpAddress(), DEFAULT_SSH_PORT);
+        }
+        List<PortForwardingRuleVO> rules = portForwardingRulesDao.listByVm(vm.getId());
+        if (CollectionUtils.isNotEmpty(rules)) {
+            for (PortForwardingRuleVO rule : rules) {
+                if (rule.getDestinationPortStart() != DEFAULT_SSH_PORT || rule.getState() == FirewallRule.State.Revoke) {
+                    continue;
+                }
+                // only the cluster's own rule, not one a user added on another public IP
+                IPAddressVO ruleIp = ipAddressDao.findById(rule.getSourceIpAddressId());
+                if (ruleIp != null && publicIpAddress.equals(ruleIp.getAddress().addr())) {
+                    return new Pair<>(publicIpAddress, rule.getSourcePortStart());
+                }
+            }
+        }
+        return new Pair<>(publicIpAddress, sshPort + controlNodeIndex);
+    }
+
+    /**
+     * Reads the kubeconfig from the control node reachable at {@link #publicIpAddress}:{@link #sshPort}, points it at
+     * {@link #apiPublicIpAddress} and stores it as the cluster's kubeConfigData, which getKubernetesClusterConfig
+     * returns. Called after every operation that can issue a new admin certificate (start, upgrade, certificate
+     * rotation), as the stored copy otherwise keeps the old certificate until it expires.
+     * @return true if a kubeconfig was read and stored
+     */
+    protected boolean refreshKubeConfigData(final long timeoutTime) {
+        String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), timeoutTime);
+        if (StringUtils.isEmpty(kubeConfig)) {
+            return false;
+        }
+        final String controlVMPrivateIpAddress = getControlVmPrivateIp();
+        if (StringUtils.isNotEmpty(controlVMPrivateIpAddress) && StringUtils.isNotEmpty(apiPublicIpAddress)) {
+            kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", controlVMPrivateIpAddress, CLUSTER_API_PORT),
+                    String.format("server: https://%s:%d", apiPublicIpAddress, CLUSTER_API_PORT));
+        }
+        kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KUBE_CONFIG_DATA_DETAIL, Base64.encodeBase64String(kubeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())), false);
+        return true;
+    }
+
     protected void updateLoginUserDetails(List<Long> clusterVMs) {
         if (clusterVMs == null) {
             clusterVMs = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList());