From c98b421b037c606fef72bbaaa8543a419a66dab2 Mon Sep 17 00:00:00 2001 From: garrettmills Date: Sat, 28 Sep 2024 01:44:56 -0400 Subject: [PATCH] [WIP] Start very early implementation --- .gitignore | 19 ++++ Dockerfile | 12 +++ Makefile | 49 ++++++++++ PROJECT | 7 ++ deploy/clusterrole.yaml | 65 +++++++++++++ deploy/clusterrolebinding.yaml | 25 +++++ deploy/csidriver.yaml | 7 ++ deploy/daemonset.yaml | 104 ++++++++++++++++++++ deploy/serviceaccount.yaml | 11 +++ deploy/statefulset.yaml | 86 +++++++++++++++++ go.mod | 20 ++++ go.sum | 21 +++++ hack/boilerplate.go.txt | 15 +++ main.go | 63 +++++++++++++ pkg/csi/controller.go | 167 ++++++++++++++++++++++++++++++++ pkg/csi/driver.go | 121 ++++++++++++++++++++++++ pkg/csi/identity.go | 53 +++++++++++ pkg/csi/node.go | 168 +++++++++++++++++++++++++++++++++ pkg/csi/p5x.go | 97 +++++++++++++++++++ pkg/csi/version.go | 62 ++++++++++++ 20 files changed, 1172 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 PROJECT create mode 100644 deploy/clusterrole.yaml create mode 100644 deploy/clusterrolebinding.yaml create mode 100644 deploy/csidriver.yaml create mode 100644 deploy/daemonset.yaml create mode 100644 deploy/serviceaccount.yaml create mode 100644 deploy/statefulset.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate.go.txt create mode 100644 main.go create mode 100644 pkg/csi/controller.go create mode 100644 pkg/csi/driver.go create mode 100644 pkg/csi/identity.go create mode 100644 pkg/csi/node.go create mode 100644 pkg/csi/p5x.go create mode 100644 pkg/csi/version.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4326f36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin/ + +# Test binary, build with ` + "`go test -c`" + ` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..748f10e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.18-buster + +ARG GOPROXY + +WORKDIR /workspace +COPY . . +ENV GOPROXY=${GOPROXY:-https://proxy.golang.org} + +RUN make csi +RUN chmod u+x /workspace/bin/csi + +ENTRYPOINT ["/workspace/bin/csi"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5ea7ac9 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +# Image URL to use all building/pushing image targets +IMG ?= csi:latest + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +##@ Build + +.PHONY: build +build: fmt vet ## Build manager binary. + go build -o bin/manager main.go + +.PHONY: run +run: fmt vet ## Run a csi driver from your host. + go run ./main.go + +.PHONY: docker-build +docker-build: test ## Build docker image with the manager. + docker build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..a41033a --- /dev/null +++ b/PROJECT @@ -0,0 +1,7 @@ +version: + number: 1 + stage: 0 + goversion: "" +name: "" +repo: csi-driver +goversion: "1.18" diff --git a/deploy/clusterrole.yaml b/deploy/clusterrole.yaml new file mode 100644 index 0000000..90915d5 --- /dev/null +++ b/deploy/clusterrole.yaml @@ -0,0 +1,65 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-node +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-controller +rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch \ No newline at end of file diff --git a/deploy/clusterrolebinding.yaml b/deploy/clusterrolebinding.yaml new file mode 100644 index 0000000..5224174 --- /dev/null +++ b/deploy/clusterrolebinding.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-node +subjects: + - kind: ServiceAccount + name: csi-node + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: csi-provisioner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-provisioner +subjects: + - kind: ServiceAccount + name: csi-controller + namespace: default \ No newline at end of file diff --git a/deploy/csidriver.yaml b/deploy/csidriver.yaml new file mode 100644 index 0000000..070e096 --- /dev/null +++ b/deploy/csidriver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: p5x +spec: + attachRequired: false + podInfoOnMount: false \ No newline at end of file diff --git a/deploy/daemonset.yaml b/deploy/daemonset.yaml new file mode 100644 index 0000000..92bf6e0 --- /dev/null +++ b/deploy/daemonset.yaml @@ -0,0 +1,104 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: p5x-csi-node + namespace: default +spec: + selector: + matchLabels: + app: p5x-csi-node + template: + metadata: + labels: + app: p5x-csi-node + spec: + serviceAccountName: csi-node + tolerations: + - operator: Exists + priorityClassName: system-node-critical + dnsPolicy: ClusterFirstWithHostNet + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --nodeid=$(NODE_NAME) + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: csi-image + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm /csi/csi.sock + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: csi-plugin + ports: + - containerPort: 9909 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/csi-plugins/demo.csi.com/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v2.1.0 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=$(ADDRESS) + - --health-port=$(HEALTH_PORT) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: HEALTH_PORT + value: "9909" + image: quay.io/k8scsi/livenessprobe:v1.1.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/csi-plugins/demo.csi.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir \ No newline at end of file diff --git a/deploy/serviceaccount.yaml b/deploy/serviceaccount.yaml new file mode 100644 index 0000000..2b0e651 --- /dev/null +++ b/deploy/serviceaccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-controller + namespace: default +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-node + namespace: default \ No newline at end of file diff --git a/deploy/statefulset.yaml b/deploy/statefulset.yaml new file mode 100644 index 0000000..024d1c5 --- /dev/null +++ b/deploy/statefulset.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: p5x-controller + name: p5x-controller + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: p5x-csi-controller + serviceName: csi-controller + template: + metadata: + labels: + app: p5x-csi-controller + spec: + priorityClassName: system-cluster-critical + serviceAccountName: csi-controller + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --nodeid=$(NODE_NAME) + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: csi-image + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: csi-plugin + ports: + - containerPort: 9909 + name: healthz + protocol: TCP + securityContext: + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --timeout=60s + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: quay.io/k8scsi/csi-provisioner:v1.6.0 + name: csi-provisioner + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --health-port=$(HEALTH_PORT) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: HEALTH_PORT + value: "9909" + image: quay.io/k8scsi/livenessprobe:v1.1.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - emptyDir: {} + name: socket-dir \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9d8a705 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module csi-driver + +go 1.21 + +toolchain go1.21.12 + +require ( + github.com/container-storage-interface/spec v1.10.0 + google.golang.org/grpc v1.67.0 + k8s.io/klog v1.0.0 +) + +require ( + github.com/kubernetes-csi/csi-test v1.1.1 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..690ac6b --- /dev/null +++ b/go.sum @@ -0,0 +1,21 @@ +github.com/container-storage-interface/spec v1.10.0 h1:YkzWPV39x+ZMTa6Ax2czJLLwpryrQ+dPesB34mrRMXA= +github.com/container-storage-interface/spec v1.10.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kubernetes-csi/csi-test v1.1.1 h1:L4RPre34ICeoQW7ez4X5t0PnFKaKs8K5q0c1XOrvXEM= +github.com/kubernetes-csi/csi-test v1.1.1/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= +google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..d8fe6c1 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..8349046 --- /dev/null +++ b/main.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + + "k8s.io/klog" + + "csi-driver/pkg/csi" +) + +var ( + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI Endpoint") + version = flag.Bool("version", false, "Print the version and exit.") + nodeID = flag.String("nodeid", "", "Node ID") + p5xEndpoint = flag.String("p5x-endpoint", "", "HTTPS endpoint of the P5x API server (e.g. https://p5x.example.local)") + p5xToken = flag.String("p5x-token", "", "P5x API access token") + p5xPort = flag.Int64("p5x-port", 3450, "P5x API port") +) + +func main() { + flag.Parse() + + if *version { + info, err := csi.GetVersionJSON() + if err != nil { + klog.Fatalln(err) + } + fmt.Println(info) + os.Exit(0) + } + if *nodeID == "" { + klog.Fatalln("--nodeid must be provided") + } + if *p5xEndpoint == "" { + klog.Fatalln("--p5x-endpoint must be provided") + } + if *p5xToken == "" { + klog.Fatalln("--p5x-token must be provided") + } + + drv := csi.NewDriver(*endpoint, *nodeID, *p5xEndpoint, *p5xToken, *p5xPort) + if err := drv.Run(); err != nil { + klog.Fatalln(err) + } +} diff --git a/pkg/csi/controller.go b/pkg/csi/controller.go new file mode 100644 index 0000000..4d84da1 --- /dev/null +++ b/pkg/csi/controller.go @@ -0,0 +1,167 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + controllerCaps = []csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + } +) + +type controllerService struct { + p5x *p5xApi + csi.UnimplementedControllerServer +} + +var _ csi.ControllerServer = &controllerService{} + +func newControllerService(p5x *p5xApi) controllerService { + return controllerService{ + p5x: p5x, + } +} + +// CreateVolume creates a volume +func (d *controllerService) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + if len(request.Name) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty") + } + if request.VolumeCapabilities == nil { + return nil, status.Error(codes.InvalidArgument, "Volume Capabilities cannot be empty") + } + + requiredCap := request.CapacityRange.GetRequiredBytes() + + vol, err := d.p5x.CreateVolume(request.Name, requiredCap) + if err != nil { + return nil, err + } + + csiVol := csi.Volume{ + VolumeId: vol.Name, + CapacityBytes: vol.SizeInBytes, + } + + return &csi.CreateVolumeResponse{Volume: &csiVol}, nil + + // volCtx := make(map[string]string) + // for k, v := range request.Parameters { + // volCtx[k] = v + // } + // + // volCtx["subPath"] = request.Name + // + // volume := csi.Volume{ + // VolumeId: request.Name, + // CapacityBytes: requiredCap, + // VolumeContext: volCtx, + // } +} + +// DeleteVolume deletes a volume +func (d *controllerService) DeleteVolume(ctx context.Context, request *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + vol, err := d.p5x.GetVolumeByName(request.VolumeId) + if err != nil { + return nil, err + } + + err = d.p5x.DeleteVolume(vol) + if err != nil { + return nil, err + } + + return &csi.DeleteVolumeResponse{}, nil +} + +// ControllerGetCapabilities get controller capabilities +func (d *controllerService) ControllerGetCapabilities(ctx context.Context, request *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + var caps []*csi.ControllerServiceCapability + for _, cap := range controllerCaps { + c := &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: cap, + }, + }, + } + caps = append(caps, c) + } + return &csi.ControllerGetCapabilitiesResponse{Capabilities: caps}, nil +} + +// ControllerPublishVolume publish a volume +func (d *controllerService) ControllerPublishVolume(ctx context.Context, request *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerUnpublishVolume unpublish a volume +func (d *controllerService) ControllerUnpublishVolume(ctx context.Context, request *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ValidateVolumeCapabilities validate volume capabilities +func (d *controllerService) ValidateVolumeCapabilities(ctx context.Context, request *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ListVolumes list volumes +func (d *controllerService) ListVolumes(ctx context.Context, request *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// GetCapacity get capacity +func (d *controllerService) GetCapacity(ctx context.Context, request *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// CreateSnapshot create a snapshot +func (d *controllerService) CreateSnapshot(ctx context.Context, request *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// DeleteSnapshot delete a snapshot +func (d *controllerService) DeleteSnapshot(ctx context.Context, request *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ListSnapshots list snapshots +func (d *controllerService) ListSnapshots(ctx context.Context, request *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerExpandVolume expand a volume +func (d *controllerService) ControllerExpandVolume(ctx context.Context, request *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerGetVolume get a volume +func (d *controllerService) ControllerGetVolume(ctx context.Context, request *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// ControllerModifyVolume modify a volume +func (d *controllerService) ControllerModifyVolume(ctx context.Context, request *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/pkg/csi/driver.go b/pkg/csi/driver.go new file mode 100644 index 0000000..cc47877 --- /dev/null +++ b/pkg/csi/driver.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + "net" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc" + "k8s.io/klog" +) + +const ( + // DriverName to be registered + DriverName = "p5x" +) + +type Driver struct { + controllerService + nodeService + + srv *grpc.Server + endpoint string + + P5x *p5xApi + + csi.UnimplementedIdentityServer +} + +// NewDriver creates a new driver +func NewDriver(endpoint string, nodeID string, p5xEndpoint string, p5xToken string, p5xPort int64) *Driver { + klog.Infof("Driver: %v version %v commit %v date %v", DriverName, driverVersion, gitCommit, buildDate) + + p5x := &p5xApi{ + endpoint: p5xEndpoint, + token: p5xToken, + port: p5xPort, + } + + return &Driver{ + endpoint: endpoint, + controllerService: newControllerService(p5x), + nodeService: newNodeService(nodeID), + P5x: p5x, + } +} + +func (d *Driver) Run() error { + scheme, addr, err := ParseEndpoint(d.endpoint) + if err != nil { + return err + } + + listener, err := net.Listen(scheme, addr) + if err != nil { + return err + } + + logErr := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + resp, err := handler(ctx, req) + if err != nil { + klog.Errorf("GRPC error: %v", err) + } + return resp, err + } + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(logErr), + } + d.srv = grpc.NewServer(opts...) + + csi.RegisterIdentityServer(d.srv, d) + csi.RegisterControllerServer(d.srv, d) + csi.RegisterNodeServer(d.srv, d) + + klog.Infof("Listening for connection on address: %#v", listener.Addr()) + return d.srv.Serve(listener) +} + +func ParseEndpoint(endpoint string) (string, string, error) { + u, err := url.Parse(endpoint) + if err != nil { + return "", "", fmt.Errorf("could not parse endpoint: %v", err) + } + + addr := path.Join(u.Host, filepath.FromSlash(u.Path)) + + scheme := strings.ToLower(u.Scheme) + switch scheme { + case "tcp": + case "unix": + addr = path.Join("/", addr) + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + return "", "", fmt.Errorf("could not remove unix domain socket %q: %v", addr, err) + } + default: + return "", "", fmt.Errorf("unsupported protocol: %s", scheme) + } + + return scheme, addr, nil +} diff --git a/pkg/csi/identity.go b/pkg/csi/identity.go new file mode 100644 index 0000000..6b1f93d --- /dev/null +++ b/pkg/csi/identity.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +// GetPluginInfo returns the name and version of the plugin +func (d *Driver) GetPluginInfo(ctx context.Context, request *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + resp := &csi.GetPluginInfoResponse{ + Name: DriverName, + VendorVersion: "v1", + } + return resp, nil +} + +// GetPluginCapabilities returns the capabilities of the plugin +func (d *Driver) GetPluginCapabilities(ctx context.Context, request *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + resp := &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + }, + } + return resp, nil +} + +// Probe returns the health and readiness of the plugin +func (d *Driver) Probe(ctx context.Context, request *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{}, nil +} diff --git a/pkg/csi/node.go b/pkg/csi/node.go new file mode 100644 index 0000000..e074e3c --- /dev/null +++ b/pkg/csi/node.go @@ -0,0 +1,168 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + volumeCaps = []csi.VolumeCapability_AccessMode{ + { + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + { + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, + }, + { + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + } +) + +type nodeService struct { + nodeID string + csi.UnimplementedNodeServer +} + +var _ csi.NodeServer = &nodeService{} + +func newNodeService(nodeID string) nodeService { + return nodeService{ + nodeID: nodeID, + } +} + +// NodeStageVolume is called by the CO when a workload that wants to use the specified volume is placed (scheduled) on a node. +func (n *nodeService) NodeStageVolume(ctx context.Context, request *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodeUnstageVolume is called by the CO when a workload that was using the specified volume is being moved to a different node. +func (n *nodeService) NodeUnstageVolume(ctx context.Context, request *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodePublishVolume mounts the volume on the node. +func (n *nodeService) NodePublishVolume(ctx context.Context, request *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + volumeID := request.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume id not provided") + } + + target := request.GetTargetPath() + if len(target) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + volCap := request.GetVolumeCapability() + if volCap == nil { + return nil, status.Error(codes.InvalidArgument, "Volume capability not provided") + } + + if !isValidVolumeCapabilities([]*csi.VolumeCapability{volCap}) { + return nil, status.Error(codes.InvalidArgument, "Volume capability not supported") + } + + readOnly := false + if request.GetReadonly() || request.VolumeCapability.AccessMode.GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { + readOnly = true + } + + options := make(map[string]string) + if m := volCap.GetMount(); m != nil { + for _, f := range m.MountFlags { + // get mountOptions from PV.spec.mountOptions + options[f] = "" + } + } + + if readOnly { + // Todo add readonly in your mount options + } + + // TODO modify your volume mount logic here + + return &csi.NodePublishVolumeResponse{}, nil +} + +// NodeUnpublishVolume unmount the volume from the target path +func (n *nodeService) NodeUnpublishVolume(ctx context.Context, request *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + target := request.GetTargetPath() + if len(target) == 0 { + return nil, status.Error(codes.InvalidArgument, "Target path not provided") + } + + // TODO modify your volume umount logic here + + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +// NodeGetVolumeStats get the volume stats +func (n *nodeService) NodeGetVolumeStats(ctx context.Context, request *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodeExpandVolume expand the volume +func (n *nodeService) NodeExpandVolume(ctx context.Context, request *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +// NodeGetCapabilities get the node capabilities +func (n *nodeService) NodeGetCapabilities(ctx context.Context, request *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + return &csi.NodeGetCapabilitiesResponse{}, nil +} + +// NodeGetInfo get the node info +func (n *nodeService) NodeGetInfo(ctx context.Context, request *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + return &csi.NodeGetInfoResponse{NodeId: n.nodeID}, nil +} + +func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) bool { + hasSupport := func(cap *csi.VolumeCapability) bool { + if csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER == cap.AccessMode.GetMode() { + return true + } + + if csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER == cap.AccessMode.GetMode() { + return true + } + + if csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY == cap.AccessMode.GetMode() { + return true + } + + // for _, c := range volumeCaps { + // if c.GetMode() == cap.AccessMode.GetMode() { + // return true + // } + // } + return false + } + + foundAll := true + for _, c := range volCaps { + if !hasSupport(c) { + foundAll = false + } + } + return foundAll +} diff --git a/pkg/csi/p5x.go b/pkg/csi/p5x.go new file mode 100644 index 0000000..ab2764e --- /dev/null +++ b/pkg/csi/p5x.go @@ -0,0 +1,97 @@ +package csi + +import ( + "bytes" + "encoding/json" + "fmt" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "io" + "k8s.io/klog" + "net/http" +) + +type p5xApi struct { + endpoint string + port int64 + token string +} + +type p5xVolume struct { + VolumeId int64 `json:"volumeId"` + Name string `json:"name"` + SizeInBytes int64 `json:"sizeInBytes"` +} + +func (p5x *p5xApi) CreateVolume(name string, sizeInBytes int64) (*p5xVolume, error) { + vol := &p5xVolume{ + VolumeId: 0, + Name: name, + SizeInBytes: sizeInBytes, + } + + body, err := json.Marshal(vol) + if err != nil { + return nil, err + } + + resBody, err := p5x.MakeRequest(http.MethodPost, "volumes", body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(resBody, vol) + klog.Info("Successfully created volume: ", vol) + + return vol, nil +} + +func (p5x *p5xApi) GetVolumeByName(name string) (*p5xVolume, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (p5x *p5xApi) GetVolumeById(volumeId int64) (*p5xVolume, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (p5x *p5xApi) DeleteVolume(volume *p5xVolume) error { + route := fmt.Sprintf("volumes/%s", volume.Name) + resBody, err := p5x.MakeRequest(http.MethodDelete, route, []byte(`{}`)) + if err != nil { + return err + } + + klog.Infof("Successfully deleted volume %s: %s", volume.Name, resBody) + return nil +} + +func (p5x *p5xApi) MakeRequest(method string, route string, body []byte) ([]byte, error) { + bodyReader := bytes.NewReader(body) + + url := fmt.Sprintf("%s:%d/api/v1/%s", p5x.endpoint, p5x.port, route) + klog.Infof("p5x.MakeRequest: [%s] %s", method, url) + klog.Infof("p5x.MakeRequest: %s", body) + req, err := http.NewRequest(method, url, bodyReader) + if err != nil { + klog.Errorf("p5x.MakeRequest: could not create request: %s\n", err) + return nil, err + } + + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", p5x.token)) + req.Header.Set("Content-Type", "application/json") + + res, err := http.DefaultClient.Do(req) + if err != nil { + klog.Errorf("p5x.MakeRequest: error executing request: %s\n", err) + return nil, err + } + + resBody, err := io.ReadAll(res.Body) + if err != nil { + klog.Errorf("p5x.MakeRequest: could not read response body: %s\n", err) + return nil, err + } + klog.Infof("p5x.MakeRequest: response body: %s\n", resBody) + return resBody, nil +} diff --git a/pkg/csi/version.go b/pkg/csi/version.go new file mode 100644 index 0000000..5d6e649 --- /dev/null +++ b/pkg/csi/version.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 p5x. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "encoding/json" + "fmt" + "runtime" +) + +// These are set during build time via -ldflags +var ( + driverVersion string + gitCommit string + buildDate string +) + +// VersionInfo struct +type VersionInfo struct { + DriverVersion string + GitCommit string + BuildDate string + GoVersion string + Compiler string + Platform string +} + +// GetVersion returns VersionInfo +func GetVersion() VersionInfo { + return VersionInfo{ + DriverVersion: driverVersion, + GitCommit: gitCommit, + BuildDate: buildDate, + GoVersion: runtime.Version(), + Compiler: runtime.Compiler, + Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), + } +} + +// GetVersionJSON returns version in JSON +func GetVersionJSON() (string, error) { + info := GetVersion() + marshalled, err := json.MarshalIndent(&info, "", " ") + if err != nil { + return "", err + } + return string(marshalled), nil +}