From: Todd Malsbary Date: Fri, 29 Oct 2021 20:20:19 +0000 (-0700) Subject: Capture creation of cluster into Helm chart X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=34b869955b20515bc44e5bac1b39ee0ffca65727;p=icn.git Capture creation of cluster into Helm chart This is a rudimentary chart, meant primarily to support the ICN e2etest cases. Signed-off-by: Todd Malsbary Change-Id: Ib5e6b9e4a0779d43a6891e1d8926d6e612c7aee8 --- diff --git a/.gitignore b/.gitignore index fd44a4d..564a7ac 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ deploy/baremetal-operator/logs/ deploy/cert-manager/logs/ deploy/metal3/scripts/machines-values.yaml deploy/cluster-api/logs/ +deploy/clusters/logs/ env/metal3/logs/ +deploy/clusters/addons/ diff --git a/deploy/clusters/.helmignore b/deploy/clusters/.helmignore new file mode 100644 index 0000000..e7ce5db --- /dev/null +++ b/deploy/clusters/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +addons/ +clusters.sh +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/clusters/Chart.yaml b/deploy/clusters/Chart.yaml new file mode 100644 index 0000000..1d71d39 --- /dev/null +++ b/deploy/clusters/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: clusters +type: application +version: 0.1.0 diff --git a/deploy/clusters/clusters.sh b/deploy/clusters/clusters.sh new file mode 100755 index 0000000..b10d2ca --- /dev/null +++ b/deploy/clusters/clusters.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -eux -o pipefail + +SCRIPTDIR="$(readlink -f $(dirname ${BASH_SOURCE[0]}))" +LIBDIR="$(dirname $(dirname ${SCRIPTDIR}))/env/lib" + +source $LIBDIR/logging.sh +source $LIBDIR/common.sh + +FLANNEL_VERSION="v0.15.0" + +# This may be used to update the in-place addon YAML files from the +# upstream projects +function build_source { + mkdir -p ${SCRIPTDIR}/addons + + # Flannel + curl -sL https://raw.githubusercontent.com/coreos/flannel/${FLANNEL_VERSION}/Documentation/kube-flannel.yml -o ${SCRIPTDIR}/addons/flannel.yaml + cat <${SCRIPTDIR}/templates/flannel-addon.yaml +{{- range \$clusterName, \$cluster := .Values.clusters }} +{{- if eq \$cluster.cni "flannel" }} +--- +$(kubectl create configmap flannel-addon --from-file=${SCRIPTDIR}/addons/flannel.yaml -o yaml --dry-run=client) +{{- end }} +{{- end }} +EOF + sed -i -e 's/ name: flannel-addon/ name: {{ $clusterName }}-flannel-addon/' ${SCRIPTDIR}/templates/flannel-addon.yaml + sed -i -e 's/10.244.0.0\/16/{{ $cluster.podCidr }}/' ${SCRIPTDIR}/templates/flannel-addon.yaml + + # Flux + flux install --export >${SCRIPTDIR}/addons/flux-system.yaml + # The name "sync" must be sorted after "flux-system" to ensure + # Flux CRDs are instantiated first + cat <<'EOF' >${SCRIPTDIR}/addons/sync.yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: {{ $cluster.flux.repositoryName }} + namespace: flux-system +spec: + gitImplementation: go-git + interval: 1m0s + ref: + branch: {{ $cluster.flux.branch }} + timeout: 20s + url: {{ $cluster.flux.url }} +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: {{ $clusterName }}-flux-sync + namespace: flux-system +spec: + interval: 10m0s + path: {{ $cluster.flux.path }} + prune: true + sourceRef: + kind: GitRepository + name: {{ $cluster.flux.repositoryName }} +EOF + cat <${SCRIPTDIR}/templates/flux-addon.yaml +{{- range \$clusterName, \$cluster := .Values.clusters }} +{{- if \$cluster.flux }} +--- +$(kubectl create configmap flux-addon --from-file=${SCRIPTDIR}/addons/flux-system.yaml,${SCRIPTDIR}/addons/sync.yaml -o yaml --dry-run=client) +{{- end }} +{{- end }} +EOF + sed -i -e 's/ name: flux-addon/ name: {{ $clusterName }}-flux-addon/' ${SCRIPTDIR}/templates/flux-addon.yaml +} + +case $1 in + "build-source") build_source ;; + *) cat </. + type: string + lastAttemptedRevision: + description: LastAttemptedRevision is the revision of the last reconciliation + attempt. + type: string + lastHandledReconcileAt: + description: LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change can be detected. + type: string + observedGeneration: + description: ObservedGeneration is the last reconciled generation. + format: int64 + type: integer + snapshot: + description: The last successfully applied revision metadata. + properties: + checksum: + description: The manifests sha1 checksum. + type: string + entries: + description: A list of Kubernetes kinds grouped by namespace. + items: + description: Snapshot holds the metadata of namespaced Kubernetes + objects + properties: + kinds: + additionalProperties: + type: string + description: The list of Kubernetes kinds. + type: object + namespace: + description: The namespace of this entry. + type: string + required: + - kinds + type: object + type: array + required: + - checksum + - entries + type: object + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: Kustomization is the Schema for the kustomizations API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KustomizationSpec defines the configuration to calculate + the desired state from a Source using Kustomize. + properties: + decryption: + description: Decrypt Kubernetes secrets before applying them on the + cluster. + properties: + provider: + description: Provider is the name of the decryption engine. + enum: + - sops + type: string + secretRef: + description: The secret name containing the private OpenPGP keys + used for decryption. + properties: + name: + description: Name of the referent + type: string + required: + - name + type: object + required: + - provider + type: object + dependsOn: + description: DependsOn may contain a dependency.CrossNamespaceDependencyReference + slice with references to Kustomization resources that must be ready + before this Kustomization can be reconciled. + items: + description: CrossNamespaceDependencyReference holds the reference + to a dependency. + properties: + name: + description: Name holds the name reference of a dependency. + type: string + namespace: + description: Namespace holds the namespace reference of a dependency. + type: string + required: + - name + type: object + type: array + force: + default: false + description: Force instructs the controller to recreate resources + when patching fails due to an immutable field change. + type: boolean + healthChecks: + description: A list of resources to be included in the health assessment. + items: + description: NamespacedObjectKindReference contains enough information + to let you locate the typed referenced object in any namespace + properties: + apiVersion: + description: API version of the referent, if not specified the + Kubernetes preferred version will be used + type: string + kind: + description: Kind of the referent + type: string + name: + description: Name of the referent + type: string + namespace: + description: Namespace of the referent, when not specified it + acts as LocalObjectReference + type: string + required: + - kind + - name + type: object + type: array + images: + description: Images is a list of (image name, new name, new tag or + digest) for changing image names, tags or digests. This can also + be achieved with a patch, but this operator is simpler to specify. + items: + description: Image contains an image name, a new name, a new tag + or digest, which will replace the original name and tag. + properties: + digest: + description: Digest is the value used to replace the original + image tag. If digest is present NewTag value is ignored. + type: string + name: + description: Name is a tag-less image name. + type: string + newName: + description: NewName is the value used to replace the original + name. + type: string + newTag: + description: NewTag is the value used to replace the original + tag. + type: string + required: + - name + type: object + type: array + interval: + description: The interval at which to reconcile the Kustomization. + type: string + kubeConfig: + description: The KubeConfig for reconciling the Kustomization on a + remote cluster. When specified, KubeConfig takes precedence over + ServiceAccountName. + properties: + secretRef: + description: SecretRef holds the name to a secret that contains + a 'value' key with the kubeconfig file as the value. It must + be in the same namespace as the Kustomization. It is recommended + that the kubeconfig is self-contained, and the secret is regularly + updated if credentials such as a cloud-access-token expire. + Cloud specific `cmd-path` auth helpers will not function without + adding binaries and credentials to the Pod that is responsible + for reconciling the Kustomization. + properties: + name: + description: Name of the referent + type: string + required: + - name + type: object + type: object + patches: + description: Strategic merge and JSON patches, defined as inline YAML + objects, capable of targeting objects based on kind, label and annotation + selectors. + items: + description: Patch contains either a StrategicMerge or a JSON6902 + patch, either a file or inline, and the target the patch should + be applied to. + properties: + patch: + description: Patch contains the JSON6902 patch document with + an array of operation objects. + type: string + target: + description: Target points to the resources that the patch document + should be applied to. + properties: + annotationSelector: + description: AnnotationSelector is a string that follows + the label selection expression https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource annotations. + type: string + group: + description: Group is the API group to select resources + from. Together with Version and Kind it is capable of + unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + kind: + description: Kind of the API Group to select resources from. + Together with Group and Version it is capable of unambiguously + identifying and/or selecting resources. https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + labelSelector: + description: LabelSelector is a string that follows the + label selection expression https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource labels. + type: string + name: + description: Name to match resources with. + type: string + namespace: + description: Namespace to select resources from. + type: string + version: + description: Version of the API Group to select resources + from. Together with Group and Kind it is capable of unambiguously + identifying and/or selecting resources. https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + type: object + type: object + type: array + patchesJson6902: + description: 'JSON 6902 patches, defined as inline YAML objects. Deprecated: + Use Patches instead.' + items: + description: JSON6902Patch contains a JSON6902 patch and the target + the patch should be applied to. + properties: + patch: + description: Patch contains the JSON6902 patch document with + an array of operation objects. + items: + description: JSON6902 is a JSON6902 operation object. https://tools.ietf.org/html/rfc6902#section-4 + properties: + from: + type: string + op: + enum: + - test + - remove + - add + - replace + - move + - copy + type: string + path: + type: string + value: + x-kubernetes-preserve-unknown-fields: true + required: + - op + - path + type: object + type: array + target: + description: Target points to the resources that the patch document + should be applied to. + properties: + annotationSelector: + description: AnnotationSelector is a string that follows + the label selection expression https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource annotations. + type: string + group: + description: Group is the API group to select resources + from. Together with Version and Kind it is capable of + unambiguously identifying and/or selecting resources. + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + kind: + description: Kind of the API Group to select resources from. + Together with Group and Version it is capable of unambiguously + identifying and/or selecting resources. https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + labelSelector: + description: LabelSelector is a string that follows the + label selection expression https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api + It matches with the resource labels. + type: string + name: + description: Name to match resources with. + type: string + namespace: + description: Namespace to select resources from. + type: string + version: + description: Version of the API Group to select resources + from. Together with Group and Kind it is capable of unambiguously + identifying and/or selecting resources. https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md + type: string + type: object + required: + - patch + - target + type: object + type: array + patchesStrategicMerge: + description: 'Strategic merge patches, defined as inline YAML objects. + Deprecated: Use Patches instead.' + items: + x-kubernetes-preserve-unknown-fields: true + type: array + path: + description: Path to the directory containing the kustomization.yaml + file, or the set of plain YAMLs a kustomization.yaml should be generated + for. Defaults to 'None', which translates to the root path of the + SourceRef. + type: string + postBuild: + description: PostBuild describes which actions to perform on the YAML + manifest generated by building the kustomize overlay. + properties: + substitute: + additionalProperties: + type: string + description: Substitute holds a map of key/value pairs. The variables + defined in your YAML manifests that match any of the keys defined + in the map will be substituted with the set value. Includes + support for bash string replacement functions e.g. ${var:=default}, + ${var:position} and ${var/substring/replacement}. + type: object + substituteFrom: + description: SubstituteFrom holds references to ConfigMaps and + Secrets containing the variables and their values to be substituted + in the YAML manifests. The ConfigMap and the Secret data keys + represent the var names and they must match the vars declared + in the manifests for the substitution to happen. + items: + description: SubstituteReference contains a reference to a resource + containing the variables name and value. + properties: + kind: + description: Kind of the values referent, valid values are + ('Secret', 'ConfigMap'). + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the values referent. Should reside + in the same namespace as the referring resource. + maxLength: 253 + minLength: 1 + type: string + required: + - kind + - name + type: object + type: array + type: object + prune: + description: Prune enables garbage collection. + type: boolean + retryInterval: + description: The interval at which to retry a previously failed reconciliation. + When not specified, the controller uses the KustomizationSpec.Interval + value to retry failures. + type: string + serviceAccountName: + description: The name of the Kubernetes service account to impersonate + when reconciling this Kustomization. + type: string + sourceRef: + description: Reference of the source where the kustomization file + is. + properties: + apiVersion: + description: API version of the referent. + type: string + kind: + description: Kind of the referent. + enum: + - GitRepository + - Bucket + type: string + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, defaults to the namespace + of the Kubernetes resource object that contains the reference. + type: string + required: + - kind + - name + type: object + suspend: + description: This flag tells the controller to suspend subsequent + kustomize executions, it does not apply to already started executions. + Defaults to false. + type: boolean + targetNamespace: + description: TargetNamespace sets or overrides the namespace in the + kustomization.yaml file. + maxLength: 63 + minLength: 1 + type: string + timeout: + description: Timeout for validation, apply and health checking operations. + Defaults to 'Interval' duration. + type: string + validation: + description: 'Deprecated: Not used in v1beta2.' + enum: + - none + - client + - server + type: string + wait: + description: Wait instructs the controller to check the health of + all the reconciled resources. When enabled, the HealthChecks are + ignored. Defaults to false. + type: boolean + required: + - interval + - prune + - sourceRef + type: object + status: + default: + observedGeneration: -1 + description: KustomizationStatus defines the observed state of a kustomization. + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: + \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // +listMapKey=type + \ Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + inventory: + description: Inventory contains the list of Kubernetes resource object + references that have been successfully applied. + properties: + entries: + description: Entries of Kubernetes resource object references. + items: + description: ResourceRef contains the information necessary + to locate a resource within a cluster. + properties: + id: + description: ID is the string representation of the Kubernetes + resource object's metadata, in the format '___'. + type: string + v: + description: Version is the API version of the Kubernetes + resource object's kind. + type: string + required: + - id + - v + type: object + type: array + required: + - entries + type: object + lastAppliedRevision: + description: The last successfully applied revision. The revision + format for Git sources is /. + type: string + lastAttemptedRevision: + description: LastAttemptedRevision is the revision of the last reconciliation + attempt. + type: string + lastHandledReconcileAt: + description: LastHandledReconcileAt holds the value of the most recent + reconcile request value, so a change can be detected. + type: string + observedGeneration: + description: ObservedGeneration is the last reconciled generation. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + creationTimestamp: null + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: providers.notification.toolkit.fluxcd.io + spec: + group: notification.toolkit.fluxcd.io + names: + kind: Provider + listKind: ProviderList + plural: providers + singular: provider + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Provider is the Schema for the providers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ProviderSpec defines the desired state of Provider + properties: + address: + description: HTTP/S webhook address of this provider + pattern: ^(http|https):// + type: string + certSecretRef: + description: CertSecretRef can be given the name of a secret containing + a PEM-encoded CA certificate (`caFile`) + properties: + name: + description: Name of the referent + type: string + required: + - name + type: object + channel: + description: Alert channel for this provider + type: string + proxy: + description: HTTP/S address of the proxy + pattern: ^(http|https):// + type: string + secretRef: + description: Secret reference containing the provider webhook URL + using "address" as data key + properties: + name: + description: Name of the referent + type: string + required: + - name + type: object + type: + description: Type of provider + enum: + - slack + - discord + - msteams + - rocket + - generic + - github + - gitlab + - bitbucket + - azuredevops + - googlechat + - webex + - sentry + - azureeventhub + - telegram + - lark + - matrix + - opsgenie + - alertmanager + type: string + username: + description: Bot username for this provider + type: string + required: + - type + type: object + status: + default: + observedGeneration: -1 + description: ProviderStatus defines the observed state of Provider + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: + \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // +listMapKey=type + \ Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + observedGeneration: + description: ObservedGeneration is the last reconciled generation. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + creationTimestamp: null + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: receivers.notification.toolkit.fluxcd.io + spec: + group: notification.toolkit.fluxcd.io + names: + kind: Receiver + listKind: ReceiverList + plural: receivers + singular: receiver + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Receiver is the Schema for the receivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ReceiverSpec defines the desired state of Receiver + properties: + events: + description: A list of events to handle, e.g. 'push' for GitHub or + 'Push Hook' for GitLab. + items: + type: string + type: array + resources: + description: A list of resources to be notified about changes. + items: + description: CrossNamespaceObjectReference contains enough information + to let you locate the typed referenced object at cluster level + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: Kind of the referent + enum: + - Bucket + - GitRepository + - Kustomization + - HelmRelease + - HelmChart + - HelmRepository + - ImageRepository + - ImagePolicy + - ImageUpdateAutomation + type: string + name: + description: Name of the referent + maxLength: 53 + minLength: 1 + type: string + namespace: + description: Namespace of the referent + maxLength: 53 + minLength: 1 + type: string + required: + - name + type: object + type: array + secretRef: + description: Secret reference containing the token used to validate + the payload authenticity + properties: + name: + description: Name of the referent + type: string + required: + - name + type: object + suspend: + description: This flag tells the controller to suspend subsequent + events handling. Defaults to false. + type: boolean + type: + description: Type of webhook sender, used to determine the validation + procedure and payload deserialization. + enum: + - generic + - generic-hmac + - github + - gitlab + - bitbucket + - harbor + - dockerhub + - quay + - gcr + - nexus + - acr + type: string + required: + - resources + - type + type: object + status: + default: + observedGeneration: -1 + description: ReceiverStatus defines the observed state of Receiver + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: + \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // +listMapKey=type + \ Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + observedGeneration: + description: ObservedGeneration is the last observed generation. + format: int64 + type: integer + url: + description: Generated webhook URL in the format of '/hook/sha256sum(token+name+namespace)'. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: helm-controller + namespace: flux-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: kustomize-controller + namespace: flux-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: notification-controller + namespace: flux-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: source-controller + namespace: flux-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: crd-controller-flux-system + rules: + - apiGroups: + - source.toolkit.fluxcd.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - kustomize.toolkit.fluxcd.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - helm.toolkit.fluxcd.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - notification.toolkit.fluxcd.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - image.toolkit.fluxcd.io + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - namespaces + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - configmaps + - configmaps/status + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: cluster-reconciler-flux-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: kustomize-controller + namespace: flux-system + - kind: ServiceAccount + name: helm-controller + namespace: flux-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: crd-controller-flux-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: crd-controller-flux-system + subjects: + - kind: ServiceAccount + name: kustomize-controller + namespace: flux-system + - kind: ServiceAccount + name: helm-controller + namespace: flux-system + - kind: ServiceAccount + name: source-controller + namespace: flux-system + - kind: ServiceAccount + name: notification-controller + namespace: flux-system + - kind: ServiceAccount + name: image-reflector-controller + namespace: flux-system + - kind: ServiceAccount + name: image-automation-controller + namespace: flux-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: notification-controller + namespace: flux-system + spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + app: notification-controller + type: ClusterIP + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: source-controller + namespace: flux-system + spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + app: source-controller + type: ClusterIP + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: webhook-receiver + namespace: flux-system + spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http-webhook + selector: + app: notification-controller + type: ClusterIP + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: helm-controller + namespace: flux-system + spec: + replicas: 1 + selector: + matchLabels: + app: helm-controller + template: + metadata: + annotations: + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: helm-controller + spec: + containers: + - args: + - --events-addr=http://notification-controller/ + - --watch-all-namespaces=true + - --log-level=info + - --log-encoding=json + - --enable-leader-election + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/fluxcd/helm-controller:v0.12.1 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: healthz + name: manager + ports: + - containerPort: 8080 + name: http-prom + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: temp + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: helm-controller + terminationGracePeriodSeconds: 600 + volumes: + - emptyDir: {} + name: temp + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: kustomize-controller + namespace: flux-system + spec: + replicas: 1 + selector: + matchLabels: + app: kustomize-controller + template: + metadata: + annotations: + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: kustomize-controller + spec: + containers: + - args: + - --events-addr=http://notification-controller/ + - --watch-all-namespaces=true + - --log-level=info + - --log-encoding=json + - --enable-leader-election + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/fluxcd/kustomize-controller:v0.16.0 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: healthz + name: manager + ports: + - containerPort: 8080 + name: http-prom + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: temp + nodeSelector: + kubernetes.io/os: linux + securityContext: + fsGroup: 1337 + serviceAccountName: kustomize-controller + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: {} + name: temp + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: notification-controller + namespace: flux-system + spec: + replicas: 1 + selector: + matchLabels: + app: notification-controller + template: + metadata: + annotations: + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: notification-controller + spec: + containers: + - args: + - --watch-all-namespaces=true + - --log-level=info + - --log-encoding=json + - --enable-leader-election + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/fluxcd/notification-controller:v0.18.1 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: healthz + name: manager + ports: + - containerPort: 9090 + name: http + - containerPort: 9292 + name: http-webhook + - containerPort: 8080 + name: http-prom + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: temp + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: notification-controller + terminationGracePeriodSeconds: 10 + volumes: + - emptyDir: {} + name: temp + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + control-plane: controller + name: source-controller + namespace: flux-system + spec: + replicas: 1 + selector: + matchLabels: + app: source-controller + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: source-controller + spec: + containers: + - args: + - --events-addr=http://notification-controller/ + - --watch-all-namespaces=true + - --log-level=info + - --log-encoding=json + - --enable-leader-election + - --storage-path=/data + - --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local. + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/fluxcd/source-controller:v0.17.0 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: healthz + name: manager + ports: + - containerPort: 9090 + name: http + - containerPort: 8080 + name: http-prom + - containerPort: 9440 + name: healthz + readinessProbe: + httpGet: + path: / + port: http + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /tmp + name: tmp + nodeSelector: + kubernetes.io/os: linux + securityContext: + fsGroup: 1337 + serviceAccountName: source-controller + terminationGracePeriodSeconds: 10 + volumes: + - emptyDir: {} + name: data + - emptyDir: {} + name: tmp + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: allow-egress + namespace: flux-system + spec: + egress: + - {} + ingress: + - from: + - podSelector: {} + podSelector: {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: allow-scraping + namespace: flux-system + spec: + ingress: + - from: + - namespaceSelector: {} + ports: + - port: 8080 + protocol: TCP + podSelector: {} + policyTypes: + - Ingress + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/instance: flux-system + app.kubernetes.io/part-of: flux + app.kubernetes.io/version: v0.20.0 + name: allow-webhooks + namespace: flux-system + spec: + ingress: + - from: + - namespaceSelector: {} + podSelector: + matchLabels: + app: notification-controller + policyTypes: + - Ingress + --- + sync.yaml: | + --- + apiVersion: source.toolkit.fluxcd.io/v1beta1 + kind: GitRepository + metadata: + name: {{ $cluster.flux.repositoryName }} + namespace: flux-system + spec: + gitImplementation: go-git + interval: 1m0s + ref: + branch: {{ $cluster.flux.branch }} + timeout: 20s + url: {{ $cluster.flux.url }} + --- + apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 + kind: Kustomization + metadata: + name: {{ $clusterName }}-flux-sync + namespace: flux-system + spec: + interval: 10m0s + path: {{ $cluster.flux.path }} + prune: true + sourceRef: + kind: GitRepository + name: {{ $cluster.flux.repositoryName }} +kind: ConfigMap +metadata: + creationTimestamp: null + name: {{ $clusterName }}-flux-addon +{{- end }} +{{- end }} diff --git a/deploy/clusters/templates/kubeadmconfigtemplate.yaml b/deploy/clusters/templates/kubeadmconfigtemplate.yaml new file mode 100644 index 0000000..2bfc97d --- /dev/null +++ b/deploy/clusters/templates/kubeadmconfigtemplate.yaml @@ -0,0 +1,57 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: {{ $clusterName }}-workers +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + node-labels: metal3.io/uuid={{ "{{" }} ds.meta_data.uuid {{ "}}" }} + cpu-manager-policy: static + topology-manager-policy: best-effort + kube-reserved: cpu=100m,memory=256Mi + name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}' + preKubeadmCommands: +{{- if $cluster.networks }} + # Without touching up /etc/hosts, kubeadm may pick the wrong + # (i.e. provisioning network) address for the node IP + - sed -i "1i $(ip -4 addr show dev {{ $cluster.networks.baremetal.interface }} | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1) $(hostname)" /etc/hosts +{{- end }} + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + - add-apt-repository "deb https://apt.kubernetes.io/ kubernetes-xenial main" + - apt update -y + - apt-get install -y ca-certificates + - /usr/local/bin/install-container-runtime.sh + - apt-get install -y kubelet={{ $cluster.kubeVersion }} kubeadm={{ $cluster.kubeVersion }} kubectl={{ $cluster.kubeVersion }} + - systemctl enable --now kubelet + postKubeadmCommands: + # This must be done after kubeadm as the cabpk provider relies + # on files in /var/run, which won't persist after a reboot + - /usr/local/bin/set_kernel_cmdline.sh + files: +{{ include "clusters.containerRuntime" $cluster | indent 6 }} + - path: /etc/systemd/system/containerd.service.d/override.conf + content: | +{{ $.Files.Get "resources/override.conf" | indent 10 }} + - path: /usr/local/bin/set_kernel_cmdline.sh + permissions: '0777' + content: | +{{ $.Files.Get "resources/set_kernel_cmdline.sh" | indent 10 }} + users: + - name: {{ $cluster.userData.name }} + shell: /bin/bash + lockPassword: False # Necessary to allow password login + passwd: {{ $cluster.userData.hashedPassword }} + sshAuthorizedKeys: + - {{ $cluster.userData.sshAuthorizedKey }} + sudo: "ALL=(ALL) NOPASSWD:ALL" + - name: root + sshAuthorizedKeys: + - {{ $cluster.userData.sshAuthorizedKey }} +{{- end }} diff --git a/deploy/clusters/templates/kubeadmcontrolplane.yaml b/deploy/clusters/templates/kubeadmcontrolplane.yaml new file mode 100644 index 0000000..a3881b6 --- /dev/null +++ b/deploy/clusters/templates/kubeadmcontrolplane.yaml @@ -0,0 +1,82 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 +kind: KubeadmControlPlane +metadata: + name: {{ $clusterName }} +spec: + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + node-labels: metal3.io/uuid={{ "{{" }} ds.meta_data.uuid {{ "}}" }} + cpu-manager-policy: static + topology-manager-policy: best-effort + kube-reserved: cpu=200m,memory=512Mi + name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}' + joinConfiguration: + controlPlane: {} + nodeRegistration: + kubeletExtraArgs: + node-labels: metal3.io/uuid={{ "{{" }} ds.meta_data.uuid {{ "}}" }} + name: '{{ "{{" }} ds.meta_data.name {{ "}}" }}' + preKubeadmCommands: +{{- if $cluster.networks }} + # Without touching up /etc/hosts, kubeadm may pick the wrong + # (i.e. provisioning network) address for the node IP + - sed -i "1i $(ip -4 addr show dev {{ $cluster.networks.baremetal.interface }} | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | head -1) $(hostname)" /etc/hosts +{{- end }} + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + - add-apt-repository "deb https://apt.kubernetes.io/ kubernetes-xenial main" + - apt update -y + - apt-get install -y ca-certificates +{{- if $cluster.keepalived }} + - apt-get install -y keepalived + - systemctl enable --now keepalived +{{- end }} + - /usr/local/bin/install-container-runtime.sh + - apt-get install -y kubelet={{ $cluster.kubeVersion }} kubeadm={{ $cluster.kubeVersion }} kubectl={{ $cluster.kubeVersion }} + - systemctl enable --now kubelet + postKubeadmCommands: + - mkdir -p /home/ubuntu/.kube + - cp /etc/kubernetes/admin.conf /home/ubuntu/.kube/config + - chown ubuntu:ubuntu /home/ubuntu/.kube/config + # This must be done after kubeadm as the cabpk provider relies on + # files in /var/run, which won't persist after a reboot + - /usr/local/bin/set_kernel_cmdline.sh + files: +{{ include "clusters.keepalived" $cluster | indent 4 }} +{{ include "clusters.containerRuntime" $cluster | indent 4 }} + - path: /etc/systemd/system/containerd.service.d/override.conf + content: | +{{ $.Files.Get "resources/override.conf" | indent 8 }} + - path: /usr/local/bin/set_kernel_cmdline.sh + permissions: '0777' + content: | +{{ $.Files.Get "resources/set_kernel_cmdline.sh" | indent 8 }} + users: + - name: {{ $cluster.userData.name }} + shell: /bin/bash + lockPassword: False # Necessary to allow password login + passwd: {{ $cluster.userData.hashedPassword }} + sshAuthorizedKeys: + - {{ $cluster.userData.sshAuthorizedKey }} + sudo: "ALL=(ALL) NOPASSWD:ALL" + - name: root + sshAuthorizedKeys: + - {{ $cluster.userData.sshAuthorizedKey }} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 + kind: Metal3MachineTemplate + name: {{ $clusterName }}-controlplane + nodeDrainTimeout: 0s + replicas: {{ $cluster.numControlPlaneMachines }} + rolloutStrategy: + rollingUpdate: + maxSurge: 1 + type: RollingUpdate + version: {{ $cluster.k8sVersion }} +{{- end }} diff --git a/deploy/clusters/templates/machinedeployment.yaml b/deploy/clusters/templates/machinedeployment.yaml new file mode 100644 index 0000000..b294ffb --- /dev/null +++ b/deploy/clusters/templates/machinedeployment.yaml @@ -0,0 +1,32 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +--- +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + name: {{ $clusterName }} +spec: + clusterName: {{ $clusterName }} + replicas: {{ $cluster.numWorkerMachines }} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ $clusterName }} + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 + kind: KubeadmConfigTemplate + name: {{ $clusterName }}-workers + clusterName: {{ $clusterName }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 + kind: Metal3MachineTemplate + name: {{ $clusterName }}-workers + nodeDrainTimeout: 0s + version: {{ $cluster.k8sVersion }} +{{- end }} diff --git a/deploy/clusters/templates/metal3cluster.yaml b/deploy/clusters/templates/metal3cluster.yaml new file mode 100644 index 0000000..be52259 --- /dev/null +++ b/deploy/clusters/templates/metal3cluster.yaml @@ -0,0 +1,12 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3Cluster +metadata: + name: {{ $clusterName }} +spec: + controlPlaneEndpoint: + host: {{ $cluster.controlPlaneEndpoint }} + port: 6443 + noCloudProvider: true +{{- end }} diff --git a/deploy/clusters/templates/metal3datatemplate.yaml b/deploy/clusters/templates/metal3datatemplate.yaml new file mode 100644 index 0000000..a1d3b7f --- /dev/null +++ b/deploy/clusters/templates/metal3datatemplate.yaml @@ -0,0 +1,26 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +{{- if $cluster.networks }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3DataTemplate +metadata: + name: {{ $clusterName }}-nodepool +spec: + clusterName: {{ $clusterName }} + networkData: + links: + ethernets: +{{- range $name, $network := $cluster.networks }} + - id: {{ $name }}_nic + macAddress: + fromHostInterface: {{ $network.interface }} + type: phy +{{- end }} + networks: + ipv4DHCP: +{{- range $name, $network := $cluster.networks }} + - id: {{ $name }} + link: {{ $name }}_nic +{{- end }} +{{- end }} +{{- end }} diff --git a/deploy/clusters/templates/metal3machinetemplate-controlplane.yaml b/deploy/clusters/templates/metal3machinetemplate-controlplane.yaml new file mode 100644 index 0000000..6ba6ebf --- /dev/null +++ b/deploy/clusters/templates/metal3machinetemplate-controlplane.yaml @@ -0,0 +1,23 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3MachineTemplate +metadata: + name: {{ $clusterName }}-controlplane +spec: + nodeReuse: false + template: + spec: + automatedCleaningMode: metadata +{{- if $cluster.controlPlaneHostSelector }} + hostSelector: + {{- toYaml $cluster.controlPlaneHostSelector | nindent 8 }} +{{- end }} +{{- if $cluster.networks }} + dataTemplate: + name: {{ $clusterName }}-nodepool +{{- end }} + image: + checksum: http://172.22.0.1:6180/images/{{ $cluster.imageName }}.md5sum + url: http://172.22.0.1:6180/images/{{ $cluster.imageName }} +{{- end }} diff --git a/deploy/clusters/templates/metal3machinetemplate-workers.yaml b/deploy/clusters/templates/metal3machinetemplate-workers.yaml new file mode 100644 index 0000000..bbb0dc5 --- /dev/null +++ b/deploy/clusters/templates/metal3machinetemplate-workers.yaml @@ -0,0 +1,23 @@ +{{- range $clusterName, $cluster := .Values.clusters }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: Metal3MachineTemplate +metadata: + name: {{ $clusterName }}-workers +spec: + nodeReuse: false + template: + spec: + automatedCleaningMode: metadata +{{- if $cluster.workersHostSelector }} + hostSelector: + {{- toYaml $cluster.workersHostSelector | nindent 8 }} +{{- end }} +{{- if $cluster.networks }} + dataTemplate: + name: {{ $clusterName }}-nodepool +{{- end }} + image: + checksum: http://172.22.0.1:6180/images/{{ $cluster.imageName }}.md5sum + url: http://172.22.0.1:6180/images/{{ $cluster.imageName }} +{{- end }}