From: chengli3 Date: Wed, 19 Feb 2020 14:30:06 +0000 (+0800) Subject: Add Sdewan Mwan3Conf CRD and controller X-Git-Tag: v1.0~42 X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=33ab8afd32ec31400200e4d26d6df5a0b7475b36;p=icn%2Fsdwan.git Add Sdewan Mwan3Conf CRD and controller The sdewan operator is developed under kubebuilder framework We define two CRDs in this patch: Sdewan and Mwan3Conf Sdewan defines the CNF base info, which node we should deploy the CNF on, which network should the CNF use with multus CNI, etc. The Mwan3Conf defines the mwan3 rules. In the next step, we are going to develop the firewall and the ipsec functions. Mwan3Conf is validated by k8s api admission webhook. For each created Sdewan instance, the controller creates a pod, a configmap and a service for the instance. The pod runs openswrt which provides network services, i.e. sdwan, firewall, ipsec etc. The configmap stores the network interface information and the entrypoint.sh. The network interface information has the following format: ``` [ { "name": "ovn-priv-net", "isProvider": false, "interface": "net0", "defaultGateway": false } ] ``` The service created by the controller is used for openwrt api access. We call this svc to apply rules, get openwrt info, restart openwrt service. After the openwrt pod ready, the Sdewan controller apply the configured mwan3 rules. mwan3 rule details are configured in Mwan3Conf CR, which is referenced by Sdewan.Spec.Mwan3Conf Every time the Mwan3Conf instance changes, the controller re-apply the new rules by calling opwnrt api. We can also change the rule refernce at the runtime. Signed-off-by: chengli3 Change-Id: Ic6fa4e8c61da5a560d69f749cd40d8f3b9320e81 --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf52783 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Kubernetes Generated files - skip generated files, except for vendored files + +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ +my-sdewan-deploy.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..018b7b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Build the manager binary +FROM golang:1.13 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api/ api/ +COPY controllers/ controllers/ +COPY openwrt/ openwrt/ +COPY wrtprovider/ wrtprovider/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER nonroot:nonroot + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..33196ac --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ + +# Image URL to use all building/pushing image targets +IMG ?= integratedcloudnative/sdewan-controller:dev +# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) +CRD_OPTIONS ?= "crd:trivialVersions=true" + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +all: manager + +# Run tests +test: generate fmt vet manifests + go test ./... -coverprofile cover.out + +# Build manager binary +manager: generate fmt vet + go build -o bin/manager main.go + +# Run against the configured Kubernetes cluster in ~/.kube/config +run: generate fmt vet manifests + go run ./main.go + +# Install CRDs into a cluster +install: manifests + kustomize build config/crd | kubectl apply -f - + +# Uninstall CRDs from a cluster +uninstall: manifests + kustomize build config/crd | kubectl delete -f - + +# Deploy controller in the configured Kubernetes cluster in ~/.kube/config +yaml-gen: manifests + cd config/manager && kustomize edit set image controller=${IMG} + kustomize build config/default > my-sdewan-deploy.yaml + +# Deploy controller in the configured Kubernetes cluster in ~/.kube/config +deploy: yaml-gen + kubectl apply -f my-sdewan-deploy.yaml + +# Generate manifests e.g. CRD, RBAC etc. +manifests: controller-gen + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +# Run go fmt against code +fmt: + go fmt ./... + +# Run go vet against code +vet: + go vet ./... + +# Generate code +generate: controller-gen + $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..." + +# Build the docker image +docker-build: test + docker build . -t ${IMG} + +# Push the docker image +docker-push: + docker push ${IMG} + +# find or download controller-gen +# download controller-gen if necessary +controller-gen: +ifeq (, $(shell which controller-gen)) + @{ \ + set -e ;\ + CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ + cd $$CONTROLLER_GEN_TMP_DIR ;\ + go mod init tmp ;\ + go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5 ;\ + rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ + } +CONTROLLER_GEN=$(GOBIN)/controller-gen +else +CONTROLLER_GEN=$(shell which controller-gen) +endif diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..6c51d5d --- /dev/null +++ b/PROJECT @@ -0,0 +1,10 @@ +domain: sdewan.akraino.org +repo: sdewan.akraino.org/sdewan +resources: +- group: batch + kind: Sdewan + version: v1alpha1 +- group: batch + kind: Mwan3Conf + version: v1alpha1 +version: "2" diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..2d9b3fe --- /dev/null +++ b/api/v1alpha1/groupversion_info.go @@ -0,0 +1,35 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the batch v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=batch.sdewan.akraino.org +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "batch.sdewan.akraino.org", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha1/mwan3conf_types.go b/api/v1alpha1/mwan3conf_types.go new file mode 100644 index 0000000..c9cf444 --- /dev/null +++ b/api/v1alpha1/mwan3conf_types.go @@ -0,0 +1,100 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +type PolicyMember struct { + Network string `json:"network"` + Metric int `json:"metric"` + Weight int `json:"weight"` +} + +type PolicyMembers struct { + Members []PolicyMember `json:"members"` +} +type Rule struct { + UsePolicy string `json:"use_policy"` + + // +optional + DestIP string `json:"dest_ip"` + + // +optional + DestPort string `json:"dest_port"` + + // +optional + SrcIP string `json:"src_IP"` + + // +optional + SrcPort string `json:"src_port"` + + // +optional + Proto string `json:"proto"` + + // +optional + Family string `json:"family"` + + // +optional + Sticky string `json:"sticky"` + + // +optional + Timeout string `json:"timeout"` +} + +// Mwan3ConfSpec defines the desired state of Mwan3Conf +type Mwan3ConfSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + Policies map[string]PolicyMembers `json:"policy"` + Rules map[string]Rule `json:"rule"` +} + +// Mwan3ConfStatus defines the observed state of Mwan3Conf +type Mwan3ConfStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// Mwan3Conf is the Schema for the mwan3confs API +type Mwan3Conf struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec Mwan3ConfSpec `json:"spec,omitempty"` + Status Mwan3ConfStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// Mwan3ConfList contains a list of Mwan3Conf +type Mwan3ConfList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Mwan3Conf `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Mwan3Conf{}, &Mwan3ConfList{}) +} diff --git a/api/v1alpha1/mwan3conf_webhook.go b/api/v1alpha1/mwan3conf_webhook.go new file mode 100644 index 0000000..661af76 --- /dev/null +++ b/api/v1alpha1/mwan3conf_webhook.go @@ -0,0 +1,93 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// log is for logging in this package. +var mwan3conflog = logf.Log.WithName("mwan3conf-resource") + +func (r *Mwan3Conf) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +// +kubebuilder:webhook:path=/mutate-batch-sdewan-akraino-org-v1alpha1-mwan3conf,mutating=true,failurePolicy=fail,groups=batch.sdewan.akraino.org,resources=mwan3confs,verbs=create;update,versions=v1alpha1,name=mmwan3conf.kb.io + +var _ webhook.Defaulter = &Mwan3Conf{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *Mwan3Conf) Default() { + mwan3conflog.Info("default", "name", r.Name) + + // TODO(user): fill in your defaulting logic. +} + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +// +kubebuilder:webhook:verbs=create;update,path=/validate-batch-sdewan-akraino-org-v1alpha1-mwan3conf,mutating=false,failurePolicy=fail,groups=batch.sdewan.akraino.org,resources=mwan3confs,versions=v1alpha1,name=vmwan3conf.kb.io + +var _ webhook.Validator = &Mwan3Conf{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *Mwan3Conf) ValidateCreate() error { + mwan3conflog.Info("validate create", "name", r.Name) + + // TODO(user): fill in your validation logic upon object creation. + return r.ValidateMwan3() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *Mwan3Conf) ValidateUpdate(old runtime.Object) error { + mwan3conflog.Info("validate update", "name", r.Name) + + // TODO(user): fill in your validation logic upon object update. + return r.ValidateMwan3() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *Mwan3Conf) ValidateDelete() error { + mwan3conflog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil +} + +func (r *Mwan3Conf) ValidateMwan3() error { + policies := r.Spec.Policies + for _, rOptions := range r.Spec.Rules { + if _, ok := policies[rOptions.UsePolicy]; !ok { + return &UnmatchError{"policy", rOptions.UsePolicy} + } + } + return nil +} + +type UnmatchError struct { + RType string + Name string +} + +func (err *UnmatchError) Error() string { + return err.RType + " :" + err.Name + " doesnt' exist" +} diff --git a/api/v1alpha1/sdewan_types.go b/api/v1alpha1/sdewan_types.go new file mode 100644 index 0000000..8b39bc4 --- /dev/null +++ b/api/v1alpha1/sdewan_types.go @@ -0,0 +1,91 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +type Network struct { + Name string `json:"name"` + + // +optional + IsProvider bool `json:"isProvider"` + + // +optional + Interface string `json:"interface"` + + // +optional + DefaultGateway bool `json:"defaultGateway"` +} + +// SdewanSpec defines the desired state of Sdewan +type SdewanSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of Sdewan. Edit Sdewan_types.go to remove/update + Node string `json:"node"` + Networks []Network `json:"networks"` + // +optional + Mwan3Conf string `json:"mwan3Conf"` +} + +type Mwan3Status struct { + Name string `json:"name"` + + IsApplied bool `json:"isApplied"` + + // +optional + // +nullable + AppliedTime *metav1.Time `json:"appliedTime"` +} + +// SdewanStatus defines the observed state of Sdewan +type SdewanStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + // +optional + Mwan3Status Mwan3Status `json:"mwan3Status"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// Sdewan is the Schema for the sdewans API +type Sdewan struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SdewanSpec `json:"spec,omitempty"` + Status SdewanStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// SdewanList contains a list of Sdewan +type SdewanList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Sdewan `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Sdewan{}, &SdewanList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..3d59282 --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,321 @@ +// +build !ignore_autogenerated + +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Mwan3Conf) DeepCopyInto(out *Mwan3Conf) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Mwan3Conf. +func (in *Mwan3Conf) DeepCopy() *Mwan3Conf { + if in == nil { + return nil + } + out := new(Mwan3Conf) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Mwan3Conf) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Mwan3ConfList) DeepCopyInto(out *Mwan3ConfList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Mwan3Conf, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Mwan3ConfList. +func (in *Mwan3ConfList) DeepCopy() *Mwan3ConfList { + if in == nil { + return nil + } + out := new(Mwan3ConfList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Mwan3ConfList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Mwan3ConfSpec) DeepCopyInto(out *Mwan3ConfSpec) { + *out = *in + if in.Policies != nil { + in, out := &in.Policies, &out.Policies + *out = make(map[string]PolicyMembers, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make(map[string]Rule, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Mwan3ConfSpec. +func (in *Mwan3ConfSpec) DeepCopy() *Mwan3ConfSpec { + if in == nil { + return nil + } + out := new(Mwan3ConfSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Mwan3ConfStatus) DeepCopyInto(out *Mwan3ConfStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Mwan3ConfStatus. +func (in *Mwan3ConfStatus) DeepCopy() *Mwan3ConfStatus { + if in == nil { + return nil + } + out := new(Mwan3ConfStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Mwan3Status) DeepCopyInto(out *Mwan3Status) { + *out = *in + if in.AppliedTime != nil { + in, out := &in.AppliedTime, &out.AppliedTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Mwan3Status. +func (in *Mwan3Status) DeepCopy() *Mwan3Status { + if in == nil { + return nil + } + out := new(Mwan3Status) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Network) DeepCopyInto(out *Network) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. +func (in *Network) DeepCopy() *Network { + if in == nil { + return nil + } + out := new(Network) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PolicyMember) DeepCopyInto(out *PolicyMember) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicyMember. +func (in *PolicyMember) DeepCopy() *PolicyMember { + if in == nil { + return nil + } + out := new(PolicyMember) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PolicyMembers) DeepCopyInto(out *PolicyMembers) { + *out = *in + if in.Members != nil { + in, out := &in.Members, &out.Members + *out = make([]PolicyMember, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicyMembers. +func (in *PolicyMembers) DeepCopy() *PolicyMembers { + if in == nil { + return nil + } + out := new(PolicyMembers) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Rule) DeepCopyInto(out *Rule) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Rule. +func (in *Rule) DeepCopy() *Rule { + if in == nil { + return nil + } + out := new(Rule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Sdewan) DeepCopyInto(out *Sdewan) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Sdewan. +func (in *Sdewan) DeepCopy() *Sdewan { + if in == nil { + return nil + } + out := new(Sdewan) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Sdewan) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SdewanList) DeepCopyInto(out *SdewanList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Sdewan, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SdewanList. +func (in *SdewanList) DeepCopy() *SdewanList { + if in == nil { + return nil + } + out := new(SdewanList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SdewanList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SdewanSpec) DeepCopyInto(out *SdewanSpec) { + *out = *in + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]Network, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SdewanSpec. +func (in *SdewanSpec) DeepCopy() *SdewanSpec { + if in == nil { + return nil + } + out := new(SdewanSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SdewanStatus) DeepCopyInto(out *SdewanStatus) { + *out = *in + in.Mwan3Status.DeepCopyInto(&out.Mwan3Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SdewanStatus. +func (in *SdewanStatus) DeepCopy() *SdewanStatus { + if in == nil { + return nil + } + out := new(SdewanStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UnmatchError) DeepCopyInto(out *UnmatchError) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnmatchError. +func (in *UnmatchError) DeepCopy() *UnmatchError { + if in == nil { + return nil + } + out := new(UnmatchError) + in.DeepCopyInto(out) + return out +} diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 0000000..237c937 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,25 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 0000000..bebea5a --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..90d7c31 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/crd/bases/batch.sdewan.akraino.org_mwan3confs.yaml b/config/crd/bases/batch.sdewan.akraino.org_mwan3confs.yaml new file mode 100644 index 0000000..547f4e9 --- /dev/null +++ b/config/crd/bases/batch.sdewan.akraino.org_mwan3confs.yaml @@ -0,0 +1,104 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: mwan3confs.batch.sdewan.akraino.org +spec: + group: batch.sdewan.akraino.org + names: + kind: Mwan3Conf + listKind: Mwan3ConfList + plural: mwan3confs + singular: mwan3conf + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Mwan3Conf is the Schema for the mwan3confs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Mwan3ConfSpec defines the desired state of Mwan3Conf + properties: + policy: + additionalProperties: + properties: + members: + items: + properties: + metric: + type: integer + network: + type: string + weight: + type: integer + required: + - metric + - network + - weight + type: object + type: array + required: + - members + type: object + type: object + rule: + additionalProperties: + properties: + dest_ip: + type: string + dest_port: + type: string + family: + type: string + proto: + type: string + src_IP: + type: string + src_port: + type: string + sticky: + type: string + timeout: + type: string + use_policy: + type: string + required: + - use_policy + type: object + type: object + required: + - policy + - rule + type: object + status: + description: Mwan3ConfStatus defines the observed state of Mwan3Conf + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/batch.sdewan.akraino.org_sdewans.yaml b/config/crd/bases/batch.sdewan.akraino.org_sdewans.yaml new file mode 100644 index 0000000..991b8cc --- /dev/null +++ b/config/crd/bases/batch.sdewan.akraino.org_sdewans.yaml @@ -0,0 +1,96 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: sdewans.batch.sdewan.akraino.org +spec: + group: batch.sdewan.akraino.org + names: + kind: Sdewan + listKind: SdewanList + plural: sdewans + singular: sdewan + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Sdewan is the Schema for the sdewans API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SdewanSpec defines the desired state of Sdewan + properties: + mwan3Conf: + type: string + networks: + items: + properties: + defaultGateway: + type: boolean + interface: + type: string + isProvider: + type: boolean + name: + type: string + required: + - name + type: object + type: array + node: + description: Foo is an example field of Sdewan. Edit Sdewan_types.go + to remove/update + type: string + required: + - networks + - node + type: object + status: + description: SdewanStatus defines the observed state of Sdewan + properties: + mwan3Status: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + properties: + appliedTime: + format: date-time + nullable: true + type: string + isApplied: + type: boolean + name: + type: string + required: + - isApplied + - name + type: object + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..d8fda98 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,24 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/batch.sdewan.akraino.org_sdewans.yaml +- bases/batch.sdewan.akraino.org_mwan3confs.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_sdewans.yaml +#- patches/webhook_in_mwan3confs.yaml +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_sdewans.yaml +#- patches/cainjection_in_mwan3confs.yaml +# +kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 0000000..6f83d9a --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,17 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_mwan3confs.yaml b/config/crd/patches/cainjection_in_mwan3confs.yaml new file mode 100644 index 0000000..a375d59 --- /dev/null +++ b/config/crd/patches/cainjection_in_mwan3confs.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: mwan3confs.batch.sdewan.akraino.org diff --git a/config/crd/patches/cainjection_in_sdewans.yaml b/config/crd/patches/cainjection_in_sdewans.yaml new file mode 100644 index 0000000..7e9c2c3 --- /dev/null +++ b/config/crd/patches/cainjection_in_sdewans.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: sdewans.batch.sdewan.akraino.org diff --git a/config/crd/patches/webhook_in_mwan3confs.yaml b/config/crd/patches/webhook_in_mwan3confs.yaml new file mode 100644 index 0000000..42c4580 --- /dev/null +++ b/config/crd/patches/webhook_in_mwan3confs.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: mwan3confs.batch.sdewan.akraino.org +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/crd/patches/webhook_in_sdewans.yaml b/config/crd/patches/webhook_in_sdewans.yaml new file mode 100644 index 0000000..faaf7bc --- /dev/null +++ b/config/crd/patches/webhook_in_sdewans.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sdewans.batch.sdewan.akraino.org +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..7f8dc21 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,75 @@ +# Adds namespace to all resources. +namespace: sdwan-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: sdwan- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: +- manager_image_patch.yaml + # Protect the /metrics endpoint by putting it behind auth. + # Only one of manager_auth_proxy_patch.yaml and + # manager_prometheus_metrics_patch.yaml should be enabled. +- manager_auth_proxy_patch.yaml + # If you want your controller-manager to expose the /metrics + # endpoint w/o any authn/z, uncomment the following line and + # comment manager_auth_proxy_patch.yaml. + # Only one of manager_auth_proxy_patch.yaml and + # manager_prometheus_metrics_patch.yaml should be enabled. +#- manager_prometheus_metrics_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml +- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1alpha2 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace +- name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1alpha2 + name: serving-cert # this name should match the one in certificate.yaml +- name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace +- name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 0000000..61cb5e7 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,25 @@ +# This patch inject a sidecar container which is a HTTP proxy for the controller manager, +# it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https + - name: manager + args: + - "--metrics-addr=127.0.0.1:8080" + - "--enable-leader-election" diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml new file mode 100644 index 0000000..d5c887f --- /dev/null +++ b/config/default/manager_image_patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent + image: integratedcloudnative/sdewan-controller:dev + env: + - name: OPENWRT_IMAGE + value: "integratedcloudnative/openwrt:dev" diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000..738de35 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 0000000..7e79bf9 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 0000000..1b69499 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: integratedcloudnative/sdewan-controller + newTag: dev diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 0000000..b6c85a5 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - command: + - /manager + args: + - --enable-leader-election + image: controller:latest + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..ed13716 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..e2d9b08 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,15 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + selector: + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 0000000..618f5e4 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 0000000..48ed1e4 --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 0000000..6cf656b --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..817f1fe --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 3 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..eaa7915 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,32 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..eed1690 --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/mwan3conf_editor_role.yaml b/config/rbac/mwan3conf_editor_role.yaml new file mode 100644 index 0000000..3e57fb7 --- /dev/null +++ b/config/rbac/mwan3conf_editor_role.yaml @@ -0,0 +1,26 @@ +# permissions to do edit mwan3confs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mwan3conf-editor-role +rules: +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs/status + verbs: + - get + - patch + - update diff --git a/config/rbac/mwan3conf_viewer_role.yaml b/config/rbac/mwan3conf_viewer_role.yaml new file mode 100644 index 0000000..2d4e1c4 --- /dev/null +++ b/config/rbac/mwan3conf_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions to do viewer mwan3confs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mwan3conf-viewer-role +rules: +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs + verbs: + - get + - list + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..fcf1614 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,48 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs/status + verbs: + - get + - patch + - update +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 0000000..8f26587 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/sdewan_editor_role.yaml b/config/rbac/sdewan_editor_role.yaml new file mode 100644 index 0000000..154cdae --- /dev/null +++ b/config/rbac/sdewan_editor_role.yaml @@ -0,0 +1,26 @@ +# permissions to do edit sdewans. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sdewan-editor-role +rules: +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans/status + verbs: + - get + - patch + - update diff --git a/config/rbac/sdewan_viewer_role.yaml b/config/rbac/sdewan_viewer_role.yaml new file mode 100644 index 0000000..b953aee --- /dev/null +++ b/config/rbac/sdewan_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions to do viewer sdewans. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sdewan-viewer-role +rules: +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans + verbs: + - get + - list + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans/status + verbs: + - get diff --git a/config/samples/batch_v1alpha1_mwan3conf.yaml b/config/samples/batch_v1alpha1_mwan3conf.yaml new file mode 100644 index 0000000..f3b5921 --- /dev/null +++ b/config/samples/batch_v1alpha1_mwan3conf.yaml @@ -0,0 +1,16 @@ +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: Mwan3Conf +metadata: + name: example-mwan3conf +spec: + # Add fields here + policy: + testpolicy1: + members: + - network: ovn-priv-net + metric: 1 + weight: 4 + rule: + https: + use_policy: testpolicy1 + dest_ip: 10.10.10.16 diff --git a/config/samples/batch_v1alpha1_sdewan.yaml b/config/samples/batch_v1alpha1_sdewan.yaml new file mode 100644 index 0000000..336ce04 --- /dev/null +++ b/config/samples/batch_v1alpha1_sdewan.yaml @@ -0,0 +1,10 @@ +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: Sdewan +metadata: + name: example-sdewan +spec: + # Add fields here + node: "ubuntu18" + networks: + - name: ovn-priv-net + mwan3Conf: example-mwan3conf diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..25e21e3 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 0000000..486d07f --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,52 @@ + +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /mutate-batch-sdewan-akraino-org-v1alpha1-mwan3conf + failurePolicy: Fail + name: mmwan3conf.kb.io + rules: + - apiGroups: + - batch.sdewan.akraino.org + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - mwan3confs + +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /validate-batch-sdewan-akraino-org-v1alpha1-mwan3conf + failurePolicy: Fail + name: vmwan3conf.kb.io + rules: + - apiGroups: + - batch.sdewan.akraino.org + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - mwan3confs diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..31e0f82 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,12 @@ + +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/controllers/mwan3conf_controller.go b/controllers/mwan3conf_controller.go new file mode 100644 index 0000000..8a92e3d --- /dev/null +++ b/controllers/mwan3conf_controller.go @@ -0,0 +1,91 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + batchv1alpha1 "sdewan.akraino.org/sdewan/api/v1alpha1" +) + +// Mwan3ConfReconciler reconciles a Mwan3Conf object +type Mwan3ConfReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=batch.sdewan.akraino.org,resources=mwan3confs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=batch.sdewan.akraino.org,resources=mwan3confs/status,verbs=get;update;patch + +func (r *Mwan3ConfReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + ctx := context.Background() + log := r.Log.WithValues("mwan3conf", req.NamespacedName) + + instance := &batchv1alpha1.Mwan3Conf{} + err := r.Get(ctx, req.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + foundSdewans := &batchv1alpha1.SdewanList{} + err = r.List(ctx, foundSdewans, client.MatchingFields{".spec.mwan3Conf": instance.Name}) + if err != nil && errors.IsNotFound(err) { + log.Info("No sdewan using this mwan3 conf", "mwan3conf", instance.Name) + return ctrl.Result{}, nil + } else if err != nil { + log.Error(err, "Failed to get the sdewan list using this mwan3 conf", "mwan3conf", instance.Name) + return ctrl.Result{}, nil + } else { + log.Info("Applying mwan3 conf for multiple sdewan instances as the conf changes", "mwan3conf", instance.Name) + for _, sdewan := range foundSdewans.Items { + // Updating sdewan to set status isapplied = false + // this will trigger sdewan controller to apply the new conf + sdewan.Status.Mwan3Status = batchv1alpha1.Mwan3Status{Name: instance.Name, IsApplied: false} + err := r.Status().Update(ctx, &sdewan) + if err != nil { + log.Error(err, "Failed to update the sdewan instance status", "sdewan", sdewan.Name) + } + } + } + return ctrl.Result{}, nil +} + +func (r *Mwan3ConfReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(&batchv1alpha1.Sdewan{}, ".spec.mwan3Conf", func(rawObj runtime.Object) []string { + // grab the job object, extract the owner... + sdewan := rawObj.(*batchv1alpha1.Sdewan) + return []string{sdewan.Spec.Mwan3Conf} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). + For(&batchv1alpha1.Mwan3Conf{}). + Complete(r) +} diff --git a/controllers/sdewan_controller.go b/controllers/sdewan_controller.go new file mode 100644 index 0000000..72a6788 --- /dev/null +++ b/controllers/sdewan_controller.go @@ -0,0 +1,365 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + batchv1alpha1 "sdewan.akraino.org/sdewan/api/v1alpha1" + "sdewan.akraino.org/sdewan/wrtprovider" +) + +var OpenwrtTag = "hle2/openwrt-1806-mwan3:latest" + +func init() { + if img := os.Getenv("OPENWRT_IMAGE"); img != "" { + OpenwrtTag = img + } +} + +// SdewanReconciler reconciles a Sdewan object +type SdewanReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=batch.sdewan.akraino.org,resources=sdewans,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=batch.sdewan.akraino.org,resources=sdewans/status,verbs=get;update;patch + +func (r *SdewanReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + ctx := context.Background() + log := r.Log.WithValues("sdewan", req.NamespacedName) + + // your logic here + // Fetch the Sdewan instance + instance := &batchv1alpha1.Sdewan{} + err := r.Get(ctx, req.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + for i, network := range instance.Spec.Networks { + if network.Interface == "" { + instance.Spec.Networks[i].Interface = fmt.Sprintf("net%d", i) + } + } + + cm := newConfigmapForCR(instance) + if err := ctrl.SetControllerReference(instance, cm, r.Scheme); err != nil { + log.Error(err, "Failed to set configmap controller reference", "Configmap.Name", cm.Name) + return ctrl.Result{}, nil + } + foundcm := &corev1.ConfigMap{} + err = r.Get(ctx, types.NamespacedName{Name: cm.Name, Namespace: cm.Namespace}, foundcm) + if err != nil && errors.IsNotFound(err) { + log.Info("Creating a new Configmap", "Configmap.Namespace", cm.Namespace, "Configmap.Name", cm.Name) + err = r.Create(ctx, cm) + if err != nil { + log.Error(err, "Failed to create configmap", "Configmap.Name", cm.Name) + return ctrl.Result{}, nil + } + } else if err != nil { + log.Error(err, "Error happends when fetch the configmap", "Configmap.Name", cm.Name) + return ctrl.Result{}, nil + } else if reflect.DeepEqual(foundcm.Data, cm.Data) { + log.Info("Updating Configmap", "Configmap.Namespace", cm.Namespace, "Configmap.Name", cm.Name) + err = r.Update(ctx, cm) + if err != nil { + log.Error(err, "Failed to update configmap", "Configmap.Name", cm.Name) + return ctrl.Result{}, nil + } + } else { + log.Info("Configmap not changed", "Configmap.Namespace", foundcm.Namespace, "Configmap.Name", foundcm.Name) + } + // Define a new Pod object + pod := newPodForCR(instance) + + // Set Sdewan instance as the owner and controller + if err := ctrl.SetControllerReference(instance, pod, r.Scheme); err != nil { + return ctrl.Result{}, nil + } + + // Check if this Pod already exists + foundpod := &corev1.Pod{} + err = r.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, foundpod) + if err != nil && errors.IsNotFound(err) { + log.Info("Creating a new Pod", "Pod.Namespace", pod.Namespace, "Pod.Name", pod.Name) + err = r.Create(ctx, pod) + if err != nil { + log.Error(err, "Failed to create the new pod", "Name", pod.Name) + return ctrl.Result{}, nil + } + + // Pod created successfully - don't requeue + log.Info("A new Pod created", "Pod.Namespace", pod.Namespace, "Pod.Name", pod.Name) + } else if err != nil { + return ctrl.Result{}, nil + } else { + // Pod already exists - don't requeue + log.Info("Pod already exists", "Pod.Namespace", foundpod.Namespace, "Pod.Name", foundpod.Name) + } + + svc := newSvcForCR(instance) + // Set Sdewan instance as the owner and controller + if err := ctrl.SetControllerReference(instance, svc, r.Scheme); err != nil { + return ctrl.Result{}, nil + } + // Check if this svc already exists + foundsvc := &corev1.Service{} + err = r.Get(ctx, types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, foundsvc) + if err != nil && errors.IsNotFound(err) { + log.Info("Creating a new Service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err = r.Create(ctx, svc) + if err != nil { + return ctrl.Result{}, nil + } + log.Info("A new Service created", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + } else if err != nil { + return ctrl.Result{}, nil + } else { + log.Info("Service already exists", "Service.Namespace", foundsvc.Namespace, "Service.Name", foundsvc.Name) + } + + // Apply rules if the pod is ready + if len(foundpod.Status.ContainerStatuses) > 0 && foundpod.Status.ContainerStatuses[0].Ready { + mwan3conf := &batchv1alpha1.Mwan3Conf{} + err = r.Get(ctx, types.NamespacedName{Name: instance.Spec.Mwan3Conf, Namespace: instance.Namespace}, mwan3conf) + if err != nil { + log.Error(err, "unable to find the mwan3conf", "namespace", instance.Namespace, "mwan3 name", instance.Spec.Mwan3Conf) + instance.Status.Mwan3Status = batchv1alpha1.Mwan3Status{Name: instance.Spec.Mwan3Conf, IsApplied: false} + err = r.Status().Update(ctx, instance) + if err != nil { + log.Error(err, "Failed to update Sdewan status") + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil + } + if (instance.Status.Mwan3Status.Name != instance.Spec.Mwan3Conf) || !instance.Status.Mwan3Status.IsApplied { + err = wrtprovider.Mwan3Apply(mwan3conf, instance) + if err != nil { + log.Error(err, "Failed to apply the mwan3conf", "namespace", instance.Namespace, "mwan3 name", instance.Spec.Mwan3Conf) + instance.Status.Mwan3Status = batchv1alpha1.Mwan3Status{Name: instance.Spec.Mwan3Conf, IsApplied: false} + r.Status().Update(ctx, instance) + return ctrl.Result{}, nil + } + instance.Status.Mwan3Status = batchv1alpha1.Mwan3Status{Name: instance.Spec.Mwan3Conf, IsApplied: true, AppliedTime: &metav1.Time{Time: time.Now()}} + err = r.Status().Update(ctx, instance) + if err != nil { + log.Error(err, "Failed to update Sdewan status") + return ctrl.Result{}, nil + } + log.Info("sdewan config applied") + } else { + log.Info("mwan3 conf not chnaged, so not re-apply", "sdewan", instance.Name) + } + } else { + log.Info("Don't apply conf as the pod is not ready", "sdewan", instance.Name) + } + + return ctrl.Result{}, nil +} + +func (r *SdewanReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&batchv1alpha1.Sdewan{}). + Owns(&corev1.Pod{}). + Complete(r) +} + +// newSvcForCR returns a svc with the same name/namespace as the cr +func newSvcForCR(cr *batchv1alpha1.Sdewan) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name, + Namespace: cr.Namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "api", + Port: 80, + }, + }, + Selector: map[string]string{ + "app": cr.Name, + }, + }, + } +} + +// newConfigmapForCR returns a configmap with the same name/namespace as the cr +func newConfigmapForCR(cr *batchv1alpha1.Sdewan) *corev1.ConfigMap { + netjson, _ := json.MarshalIndent(cr.Spec.Networks, "", " ") + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name, + Namespace: cr.Namespace, + }, + Data: map[string]string{ + "networks.json": string(netjson), + "entrypoint.sh": `#!/bin/bash +# Always exit on errors. +set -e +echo "" > /etc/config/network +cat > /etc/config/mwan3 <> /etc/config/network <> /etc/config/mwan3 <= 400 { + if resp.StatusCode == 403 { + // token expired, retry + o.token = "" + continue + } else { + // error request + return "", &OpenwrtError{Code: resp.StatusCode, Message: string(body)} + } + } + + return string(body), nil + } + + return "", nil +} + +// call openwrt Get restful API +func (o *openwrtClient) Get(url string) (string, error) { + return o.call("GET", url, "") +} + +// call openwrt restful API +func (o *openwrtClient) Post(url string, request string) (string, error) { + return o.call("POST", url, request) +} + +// call openwrt restful API +func (o *openwrtClient) Put(url string, request string) (string, error) { + return o.call("PUT", url, request) +} + +// call openwrt restful API +func (o *openwrtClient) Delete(url string) (string, error) { + return o.call("DELETE", url, "") +} diff --git a/openwrt/service.go b/openwrt/service.go new file mode 100644 index 0000000..7580e94 --- /dev/null +++ b/openwrt/service.go @@ -0,0 +1,54 @@ +package openwrt + +import ( + "encoding/json" +) + +const ( + serviceBaseURL = "sdewan/v1/" +) + +var available_Services = []string{"mwan3", "firewall", "ipsec"} + +type ServiceClient struct { + OpenwrtClient *openwrtClient +} + +// Service API struct +type AvailableServices struct { + Services []string `json:"services"` +} + +// get available services +func (s *ServiceClient) GetAvailableServices() (*AvailableServices, error) { + response, err := s.OpenwrtClient.Get(serviceBaseURL + "services") + if err != nil { + return nil, err + } + + var servs AvailableServices + err2 := json.Unmarshal([]byte(response), &servs) + if err2 != nil { + return nil, err2 + } + + return &servs, nil +} + +func (s *ServiceClient) formatExecuteServiceBody(operation string) string { + return "{\"action\":\"" + operation + "\"}" +} + +// execute operation on service +func (s *ServiceClient) ExecuteService(service string, operation string) (bool, error) { + if !IsContained(available_Services, service) { + return false, &OpenwrtError{Code: 400, Message: "Bad Request: not supported service(" + service + ")"} + } + + _, err := s.OpenwrtClient.Put(serviceBaseURL+"service/"+service, s.formatExecuteServiceBody(operation)) + if err != nil { + return false, err + } + + return true, nil +} diff --git a/openwrt/utils.go b/openwrt/utils.go new file mode 100644 index 0000000..c37631d --- /dev/null +++ b/openwrt/utils.go @@ -0,0 +1,22 @@ +package openwrt + +import ( + "reflect" +) + +// util function to check whether items contains item +func IsContained(items interface{}, item interface{}) bool { + switch reflect.TypeOf(items).Kind() { + case reflect.Slice: + v := reflect.ValueOf(items) + for i := 0; i < v.Len(); i++ { + if reflect.DeepEqual(item, v.Index(i).Interface()) { + return true + } + } + default: + return false + } + + return false +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..04e35aa --- /dev/null +++ b/readme.md @@ -0,0 +1,57 @@ +# Sdewan operator + +The sdewan operator is developed under kubebuilder framework + +We define two CRDs in this patch: Sdewan and Mwan3Conf. + +Sdewan defines the CNF base info, which node we should deploy the CNF on, +which network should the CNF use with multus CNI, etc. + +The Mwan3Conf defines the mwan3 rules. In the next step, we are going to +develop the firewall and the ipsec functions. Mwan3Conf is validated by k8s +api admission webhook. + +For each created Sdewan instance, the controller creates a pod, a configmap +and a service for the instance. The pod runs openswrt which provides network +services, i.e. sdwan, firewall, ipsec etc. + +The configmap stores the network interface information and the entrypoint.sh. +The network interface information has the following format: +``` +[ + { + "name": "ovn-priv-net", + "isProvider": false, + "interface": "net0", + "defaultGateway": false + } +] +``` + +The service created by the controller is used for openwrt api access. +We call this svc to apply rules, get openwrt info, restart openwrt service. + +After the openwrt pod ready, the Sdewan controller apply the configured mwan3 rules. +mwan3 rule details are configured in Mwan3Conf CR, which is referenced by Sdewan.Spec.Mwan3Conf +Every time the Mwan3Conf instance changes, the controller re-apply the new rules by calling opwnrt +api. We can also change the rule refernce at the runtime. + +## Deployment + +The API admission webhook depends on cert-manager so we need to install cert-manager first. + +To install the CRD and the controller, we can follow this guide. +https://book.kubebuilder.io/cronjob-tutorial/running-webhook.html + +We have the image built and published at `integratedcloudnative/sdewan-controller:dev`. The openwrt +docker image we used for test is at `integratedcloudnative/openwrt:dev`. To use some other images, +we need to make configuration in `config/default/manager_image_patch.yaml` + +The simple installation steps: +1. kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v0.11.0/cert-manager.yaml +2. kubectl apply -f sdewan-deploy.yaml + +## References + +- https://book.kubebuilder.io/ +- https://openwrt.org/ diff --git a/sdewan-deploy.yaml b/sdewan-deploy.yaml new file mode 100644 index 0000000..8b583ca --- /dev/null +++ b/sdewan-deploy.yaml @@ -0,0 +1,506 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: sdwan-system +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: mwan3confs.batch.sdewan.akraino.org +spec: + group: batch.sdewan.akraino.org + names: + kind: Mwan3Conf + listKind: Mwan3ConfList + plural: mwan3confs + singular: mwan3conf + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Mwan3Conf is the Schema for the mwan3confs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Mwan3ConfSpec defines the desired state of Mwan3Conf + properties: + policy: + additionalProperties: + properties: + members: + items: + properties: + metric: + type: integer + network: + type: string + weight: + type: integer + required: + - metric + - network + - weight + type: object + type: array + required: + - members + type: object + type: object + rule: + additionalProperties: + properties: + dest_ip: + type: string + dest_port: + type: string + family: + type: string + proto: + type: string + src_IP: + type: string + src_port: + type: string + sticky: + type: string + timeout: + type: string + use_policy: + type: string + required: + - use_policy + type: object + type: object + required: + - policy + - rule + type: object + status: + description: Mwan3ConfStatus defines the observed state of Mwan3Conf + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: sdewans.batch.sdewan.akraino.org +spec: + group: batch.sdewan.akraino.org + names: + kind: Sdewan + listKind: SdewanList + plural: sdewans + singular: sdewan + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Sdewan is the Schema for the sdewans API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SdewanSpec defines the desired state of Sdewan + properties: + mwan3Conf: + type: string + networks: + items: + properties: + defaultGateway: + type: boolean + interface: + type: string + isProvider: + type: boolean + name: + type: string + required: + - name + type: object + type: array + node: + description: Foo is an example field of Sdewan. Edit Sdewan_types.go + to remove/update + type: string + required: + - networks + - node + type: object + status: + description: SdewanStatus defines the observed state of Sdewan + properties: + mwan3Status: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + properties: + appliedTime: + format: date-time + nullable: true + type: string + isApplied: + type: boolean + name: + type: string + required: + - isApplied + - name + type: object + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: sdwan-system/sdwan-serving-cert + creationTimestamp: null + name: sdwan-mutating-webhook-configuration +webhooks: +- clientConfig: + caBundle: Cg== + service: + name: sdwan-webhook-service + namespace: sdwan-system + path: /mutate-batch-sdewan-akraino-org-v1alpha1-mwan3conf + failurePolicy: Fail + name: mmwan3conf.kb.io + rules: + - apiGroups: + - batch.sdewan.akraino.org + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - mwan3confs +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sdwan-leader-election-role + namespace: sdwan-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: sdwan-manager-role +rules: +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - mwan3confs/status + verbs: + - get + - patch + - update +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch.sdewan.akraino.org + resources: + - sdewans/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sdwan-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sdwan-leader-election-rolebinding + namespace: sdwan-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sdwan-leader-election-role +subjects: +- kind: ServiceAccount + name: default + namespace: sdwan-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sdwan-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sdwan-manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: sdwan-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sdwan-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sdwan-proxy-role +subjects: +- kind: ServiceAccount + name: default + namespace: sdwan-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: sdwan-controller-manager-metrics-service + namespace: sdwan-system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + name: sdwan-webhook-service + namespace: sdwan-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: controller-manager + name: sdwan-controller-manager + namespace: sdwan-system +spec: + replicas: 1 + selector: + matchLabels: + control-plane: controller-manager + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=10 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + - args: + - --metrics-addr=127.0.0.1:8080 + - --enable-leader-election + command: + - /manager + env: + - name: OPENWRT_IMAGE + value: integratedcloudnative/openwrt:dev + image: integratedcloudnative/sdewan-controller:dev + imagePullPolicy: IfNotPresent + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: sdwan-serving-cert + namespace: sdwan-system +spec: + dnsNames: + - sdwan-webhook-service.sdwan-system.svc + - sdwan-webhook-service.sdwan-system.svc.cluster.local + issuerRef: + kind: Issuer + name: sdwan-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: sdwan-selfsigned-issuer + namespace: sdwan-system +spec: + selfSigned: {} +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: sdwan-system/sdwan-serving-cert + creationTimestamp: null + name: sdwan-validating-webhook-configuration +webhooks: +- clientConfig: + caBundle: Cg== + service: + name: sdwan-webhook-service + namespace: sdwan-system + path: /validate-batch-sdewan-akraino-org-v1alpha1-mwan3conf + failurePolicy: Fail + name: vmwan3conf.kb.io + rules: + - apiGroups: + - batch.sdewan.akraino.org + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - mwan3confs diff --git a/wrtprovider/wrtprovider.go b/wrtprovider/wrtprovider.go new file mode 100644 index 0000000..634932e --- /dev/null +++ b/wrtprovider/wrtprovider.go @@ -0,0 +1,179 @@ +package wrtprovider + +import ( + "fmt" + "reflect" + sdewanv1alpha1 "sdewan.akraino.org/sdewan/api/v1alpha1" + "sdewan.akraino.org/sdewan/openwrt" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var log = logf.Log.WithName("controller_sdewan") + +func NetworkInterfaceMap(instance *sdewanv1alpha1.Sdewan) map[string]string { + ifMap := make(map[string]string) + for i, network := range instance.Spec.Networks { + prefix := "lan_" + if network.IsProvider { + prefix = "wan_" + } + if network.Interface == "" { + network.Interface = fmt.Sprintf("net%d", i) + } + ifMap[network.Name] = prefix + fmt.Sprintf("net%d", i) + } + return ifMap +} + +func Mwan3ReplacePolicies(policies []openwrt.SdewanPolicy, existOnes []openwrt.SdewanPolicy, client *openwrt.Mwan3Client) error { + // create/update new policies + for _, policy := range policies { + found := false + for _, p := range existOnes { + if p.Name == policy.Name { + if !reflect.DeepEqual(policy, p) { + _, err := client.UpdatePolicy(policy) + if err != nil { + return err + } + } + found = true + break + } + } + if found == false { + _, err := client.CreatePolicy(policy) + if err != nil { + return err + } + } + } + + // remove old policies + for _, p := range existOnes { + found := false + for _, policy := range policies { + if p.Name == policy.Name { + found = true + break + } + } + if found == false { + err := client.DeletePolicy(p.Name) + if err != nil { + return err + } + } + } + + return nil +} + +func Mwan3ReplaceRules(rules []openwrt.SdewanRule, existOnes []openwrt.SdewanRule, client *openwrt.Mwan3Client) error { + // create/update new rules + for _, rule := range rules { + found := false + for _, r := range existOnes { + if r.Name == rule.Name { + if !reflect.DeepEqual(rule, r) { + _, err := client.UpdateRule(rule) + if err != nil { + return err + } + } + found = true + break + } + } + if found == false { + _, err := client.CreateRule(rule) + if err != nil { + return err + } + } + } + + // remove old rules + for _, r := range existOnes { + found := false + for _, rule := range rules { + if r.Name == rule.Name { + found = true + break + } + } + if found == false { + err := client.DeleteRule(r.Name) + if err != nil { + return err + } + } + } + + return nil +} + +// apply policy and rules +func Mwan3Apply(mwan3Conf *sdewanv1alpha1.Mwan3Conf, sdewan *sdewanv1alpha1.Sdewan) error { + reqLogger := log.WithValues("Mwan3Provider", mwan3Conf.Name, "Sdewan", sdewan.Name) + openwrtClient := openwrt.NewOpenwrtClient(sdewan.Name, "root", "") + mwan3 := openwrt.Mwan3Client{OpenwrtClient: openwrtClient} + service := openwrt.ServiceClient{OpenwrtClient: openwrtClient} + netMap := NetworkInterfaceMap(sdewan) + var policies []openwrt.SdewanPolicy + for policyName, members := range mwan3Conf.Spec.Policies { + openwrtMembers := make([]openwrt.SdewanMember, len(members.Members)) + for i, member := range members.Members { + openwrtMembers[i] = openwrt.SdewanMember{ + Interface: netMap[member.Network], + Metric: fmt.Sprintf("%d", member.Metric), + Weight: fmt.Sprintf("%d", member.Weight), + } + } + policies = append(policies, openwrt.SdewanPolicy{ + Name: policyName, + Members: openwrtMembers}) + } + existPolicies, err := mwan3.GetPolicies() + if err != nil { + reqLogger.Error(err, "Failed to fetch existing policies") + return err + } + err = Mwan3ReplacePolicies(policies, existPolicies.Policies, &mwan3) + if err != nil { + reqLogger.Error(err, "Failed to apply Policies") + return err + } + var rules []openwrt.SdewanRule + for ruleName, rule := range mwan3Conf.Spec.Rules { + openwrtRule := openwrt.SdewanRule{ + Name: ruleName, + Policy: rule.UsePolicy, + SrcIp: rule.SrcIP, + SrcPort: rule.SrcPort, + DestIp: rule.DestIP, + DestPort: rule.DestPort, + Proto: rule.Proto, + Family: rule.Family, + Sticky: rule.Sticky, + Timeout: rule.Timeout, + } + rules = append(rules, openwrtRule) + } + existRules, err := mwan3.GetRules() + if err != nil { + reqLogger.Error(err, "Failed to fetch existing rules") + return err + } + err = Mwan3ReplaceRules(rules, existRules.Rules, &mwan3) + if err != nil { + reqLogger.Error(err, "Failed to apply rules") + return err + } + _, err = service.ExecuteService("mwan3", "restart") + if err != nil { + reqLogger.Error(err, "Failed to restart mwan3 service") + return err + } + return nil +}