[WIP] More work implementing create/delete volume and publish/unpublish volume
This commit is contained in:
parent
c98b421b03
commit
b9f2259674
1
.gitignore
vendored
1
.gitignore
vendored
@ -13,6 +13,7 @@ bin/
|
|||||||
*.out
|
*.out
|
||||||
|
|
||||||
# editor and IDE paraphernalia
|
# editor and IDE paraphernalia
|
||||||
|
.fleet
|
||||||
.idea
|
.idea
|
||||||
*.swp
|
*.swp
|
||||||
*.swo
|
*.swo
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.18-buster
|
FROM golang:1.21-bookworm
|
||||||
|
|
||||||
ARG GOPROXY
|
ARG GOPROXY
|
||||||
|
|
||||||
@ -6,7 +6,7 @@ WORKDIR /workspace
|
|||||||
COPY . .
|
COPY . .
|
||||||
ENV GOPROXY=${GOPROXY:-https://proxy.golang.org}
|
ENV GOPROXY=${GOPROXY:-https://proxy.golang.org}
|
||||||
|
|
||||||
RUN make csi
|
RUN make build
|
||||||
RUN chmod u+x /workspace/bin/csi
|
RUN chmod u+x /workspace/bin/manager
|
||||||
|
|
||||||
ENTRYPOINT ["/workspace/bin/csi"]
|
ENTRYPOINT ["/workspace/bin/manager"]
|
||||||
|
10
Makefile
10
Makefile
@ -1,5 +1,5 @@
|
|||||||
# Image URL to use all building/pushing image targets
|
# Image URL to use all building/pushing image targets
|
||||||
IMG ?= csi:latest
|
IMG ?= registry.millslan.net/p5x-csi:latest
|
||||||
|
|
||||||
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
|
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
|
||||||
ifeq (,$(shell go env GOBIN))
|
ifeq (,$(shell go env GOBIN))
|
||||||
@ -40,10 +40,10 @@ build: fmt vet ## Build manager binary.
|
|||||||
run: fmt vet ## Run a csi driver from your host.
|
run: fmt vet ## Run a csi driver from your host.
|
||||||
go run ./main.go
|
go run ./main.go
|
||||||
|
|
||||||
.PHONY: docker-build
|
.PHONY: image
|
||||||
docker-build: test ## Build docker image with the manager.
|
image: Dockerfile build ## Build docker image with the manager.
|
||||||
docker build -t ${IMG} .
|
docker build -t ${IMG} .
|
||||||
|
|
||||||
.PHONY: docker-push
|
.PHONY: push
|
||||||
docker-push: ## Push docker image with the manager.
|
push: ## Push docker image with the manager.
|
||||||
docker push ${IMG}
|
docker push ${IMG}
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
metadata:
|
metadata:
|
||||||
name: csi-node
|
name: p5x-csi-node
|
||||||
rules: []
|
rules: []
|
||||||
---
|
---
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
metadata:
|
metadata:
|
||||||
name: csi-controller
|
name: p5x-csi-controller
|
||||||
rules:
|
rules:
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- ""
|
- ""
|
||||||
@ -19,6 +19,8 @@ rules:
|
|||||||
- watch
|
- watch
|
||||||
- create
|
- create
|
||||||
- delete
|
- delete
|
||||||
|
- patch
|
||||||
|
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- ""
|
- ""
|
||||||
resources:
|
resources:
|
||||||
@ -28,6 +30,24 @@ rules:
|
|||||||
- list
|
- list
|
||||||
- watch
|
- watch
|
||||||
- update
|
- update
|
||||||
|
|
||||||
|
- apiGroups:
|
||||||
|
- storage.k8s.io
|
||||||
|
resources:
|
||||||
|
- volumeattachments
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
- patch
|
||||||
|
|
||||||
|
- apiGroups:
|
||||||
|
- storage.k8s.io
|
||||||
|
resources:
|
||||||
|
- volumeattachments/status
|
||||||
|
verbs:
|
||||||
|
- patch
|
||||||
|
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- storage.k8s.io
|
- storage.k8s.io
|
||||||
resources:
|
resources:
|
||||||
@ -36,6 +56,7 @@ rules:
|
|||||||
- get
|
- get
|
||||||
- list
|
- list
|
||||||
- watch
|
- watch
|
||||||
|
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- ""
|
- ""
|
||||||
resources:
|
resources:
|
||||||
@ -47,6 +68,7 @@ rules:
|
|||||||
- create
|
- create
|
||||||
- update
|
- update
|
||||||
- patch
|
- patch
|
||||||
|
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- storage.k8s.io
|
- storage.k8s.io
|
||||||
resources:
|
resources:
|
||||||
@ -55,6 +77,7 @@ rules:
|
|||||||
- get
|
- get
|
||||||
- list
|
- list
|
||||||
- watch
|
- watch
|
||||||
|
|
||||||
- apiGroups:
|
- apiGroups:
|
||||||
- ""
|
- ""
|
||||||
resources:
|
resources:
|
||||||
|
@ -1,25 +1,25 @@
|
|||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
kind: ClusterRoleBinding
|
kind: ClusterRoleBinding
|
||||||
metadata:
|
metadata:
|
||||||
name: csi-node
|
name: p5x-csi-node
|
||||||
roleRef:
|
roleRef:
|
||||||
apiGroup: rbac.authorization.k8s.io
|
apiGroup: rbac.authorization.k8s.io
|
||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
name: csi-node
|
name: p5x-csi-node
|
||||||
subjects:
|
subjects:
|
||||||
- kind: ServiceAccount
|
- kind: ServiceAccount
|
||||||
name: csi-node
|
name: p5x-csi-node
|
||||||
namespace: default
|
namespace: p5x-system
|
||||||
---
|
---
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
kind: ClusterRoleBinding
|
kind: ClusterRoleBinding
|
||||||
metadata:
|
metadata:
|
||||||
name: csi-provisioner
|
name: p5x-csi-controller
|
||||||
roleRef:
|
roleRef:
|
||||||
apiGroup: rbac.authorization.k8s.io
|
apiGroup: rbac.authorization.k8s.io
|
||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
name: csi-provisioner
|
name: p5x-csi-controller
|
||||||
subjects:
|
subjects:
|
||||||
- kind: ServiceAccount
|
- kind: ServiceAccount
|
||||||
name: csi-controller
|
name: p5x-csi-controller
|
||||||
namespace: default
|
namespace: p5x-system
|
@ -3,5 +3,5 @@ kind: CSIDriver
|
|||||||
metadata:
|
metadata:
|
||||||
name: p5x
|
name: p5x
|
||||||
spec:
|
spec:
|
||||||
attachRequired: false
|
attachRequired: true
|
||||||
podInfoOnMount: false
|
podInfoOnMount: false
|
@ -2,7 +2,7 @@ kind: DaemonSet
|
|||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
metadata:
|
metadata:
|
||||||
name: p5x-csi-node
|
name: p5x-csi-node
|
||||||
namespace: default
|
namespace: p5x-system
|
||||||
spec:
|
spec:
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
@ -12,7 +12,7 @@ spec:
|
|||||||
labels:
|
labels:
|
||||||
app: p5x-csi-node
|
app: p5x-csi-node
|
||||||
spec:
|
spec:
|
||||||
serviceAccountName: csi-node
|
serviceAccountName: p5x-csi-node
|
||||||
tolerations:
|
tolerations:
|
||||||
- operator: Exists
|
- operator: Exists
|
||||||
priorityClassName: system-node-critical
|
priorityClassName: system-node-critical
|
||||||
@ -20,8 +20,10 @@ spec:
|
|||||||
containers:
|
containers:
|
||||||
- args:
|
- args:
|
||||||
- --endpoint=$(CSI_ENDPOINT)
|
- --endpoint=$(CSI_ENDPOINT)
|
||||||
- --logtostderr
|
# - --logtostderr
|
||||||
- --nodeid=$(NODE_NAME)
|
- --nodeid=$(NODE_NAME)
|
||||||
|
- --p5x-endpoint=http://172.20.0.22
|
||||||
|
- --p5x-token=1
|
||||||
env:
|
env:
|
||||||
- name: CSI_ENDPOINT
|
- name: CSI_ENDPOINT
|
||||||
value: unix:/csi/csi.sock
|
value: unix:/csi/csi.sock
|
||||||
@ -29,7 +31,7 @@ spec:
|
|||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
fieldPath: spec.nodeName
|
fieldPath: spec.nodeName
|
||||||
image: csi-image
|
image: registry.millslan.net/p5x-csi:latest
|
||||||
lifecycle:
|
lifecycle:
|
||||||
preStop:
|
preStop:
|
||||||
exec:
|
exec:
|
||||||
|
32
deploy/examplepod.yaml
Normal file
32
deploy/examplepod.yaml
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: example-pvc
|
||||||
|
namespace: p5x-system
|
||||||
|
spec:
|
||||||
|
storageClassName: p5x
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 5Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: example-pod
|
||||||
|
namespace: p5x-system
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: example-ct
|
||||||
|
image: fedora:39
|
||||||
|
command: ['/bin/bash', '-c', '--']
|
||||||
|
args: ['while true; do sleep 30; done;']
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: '/mnt/example-pvc'
|
||||||
|
name: example-pvc
|
||||||
|
volumes:
|
||||||
|
- name: example-pvc
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: example-pvc
|
5
deploy/namespace.yaml
Normal file
5
deploy/namespace.yaml
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: p5x-system
|
@ -1,11 +1,11 @@
|
|||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ServiceAccount
|
kind: ServiceAccount
|
||||||
metadata:
|
metadata:
|
||||||
name: csi-controller
|
name: p5x-csi-controller
|
||||||
namespace: default
|
namespace: p5x-system
|
||||||
---
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: ServiceAccount
|
kind: ServiceAccount
|
||||||
metadata:
|
metadata:
|
||||||
name: csi-node
|
name: p5x-csi-node
|
||||||
namespace: default
|
namespace: p5x-system
|
@ -5,28 +5,34 @@ metadata:
|
|||||||
app.kubernetes.io/component: controller
|
app.kubernetes.io/component: controller
|
||||||
app.kubernetes.io/name: p5x-controller
|
app.kubernetes.io/name: p5x-controller
|
||||||
name: p5x-controller
|
name: p5x-controller
|
||||||
namespace: default
|
namespace: p5x-system
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: p5x-csi-controller
|
app: p5x-csi-controller
|
||||||
serviceName: csi-controller
|
serviceName: p5x-csi-controller
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app: p5x-csi-controller
|
app: p5x-csi-controller
|
||||||
spec:
|
spec:
|
||||||
priorityClassName: system-cluster-critical
|
priorityClassName: system-cluster-critical
|
||||||
serviceAccountName: csi-controller
|
serviceAccountName: p5x-csi-controller
|
||||||
tolerations:
|
tolerations:
|
||||||
- key: CriticalAddonsOnly
|
- key: CriticalAddonsOnly
|
||||||
operator: Exists
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- emptyDir: {}
|
||||||
|
name: socket-dir
|
||||||
containers:
|
containers:
|
||||||
- args:
|
- name: csi-plugin
|
||||||
|
args:
|
||||||
- --endpoint=$(CSI_ENDPOINT)
|
- --endpoint=$(CSI_ENDPOINT)
|
||||||
- --logtostderr
|
# - --logtostderr
|
||||||
- --nodeid=$(NODE_NAME)
|
- --nodeid=$(NODE_NAME)
|
||||||
|
- --p5x-endpoint=http://172.20.0.22
|
||||||
|
- --p5x-token=1
|
||||||
env:
|
env:
|
||||||
- name: CSI_ENDPOINT
|
- name: CSI_ENDPOINT
|
||||||
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
|
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
|
||||||
@ -34,7 +40,7 @@ spec:
|
|||||||
valueFrom:
|
valueFrom:
|
||||||
fieldRef:
|
fieldRef:
|
||||||
fieldPath: spec.nodeName
|
fieldPath: spec.nodeName
|
||||||
image: csi-image
|
image: registry.millslan.net/p5x-csi:latest
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
failureThreshold: 5
|
failureThreshold: 5
|
||||||
httpGet:
|
httpGet:
|
||||||
@ -43,7 +49,6 @@ spec:
|
|||||||
initialDelaySeconds: 10
|
initialDelaySeconds: 10
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
timeoutSeconds: 3
|
timeoutSeconds: 3
|
||||||
name: csi-plugin
|
|
||||||
ports:
|
ports:
|
||||||
- containerPort: 9909
|
- containerPort: 9909
|
||||||
name: healthz
|
name: healthz
|
||||||
@ -56,19 +61,38 @@ spec:
|
|||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
||||||
name: socket-dir
|
name: socket-dir
|
||||||
- args:
|
|
||||||
|
|
||||||
|
- name: csi-provisioner
|
||||||
|
args:
|
||||||
- --csi-address=$(ADDRESS)
|
- --csi-address=$(ADDRESS)
|
||||||
- --timeout=60s
|
- --timeout=60s
|
||||||
- --v=5
|
- --v=5
|
||||||
env:
|
env:
|
||||||
- name: ADDRESS
|
- name: ADDRESS
|
||||||
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
||||||
image: quay.io/k8scsi/csi-provisioner:v1.6.0
|
image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
|
||||||
name: csi-provisioner
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
||||||
name: socket-dir
|
name: socket-dir
|
||||||
- args:
|
|
||||||
|
|
||||||
|
- name: csi-attacher
|
||||||
|
args:
|
||||||
|
- --csi-address=$(ADDRESS)
|
||||||
|
- --timeout=60s
|
||||||
|
- --v=5
|
||||||
|
env:
|
||||||
|
- name: ADDRESS
|
||||||
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
||||||
|
image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/lib/csi/sockets/pluginproxy/
|
||||||
|
name: socket-dir
|
||||||
|
|
||||||
|
|
||||||
|
- name: liveness-probe
|
||||||
|
args:
|
||||||
- --csi-address=$(ADDRESS)
|
- --csi-address=$(ADDRESS)
|
||||||
- --health-port=$(HEALTH_PORT)
|
- --health-port=$(HEALTH_PORT)
|
||||||
env:
|
env:
|
||||||
@ -77,10 +101,6 @@ spec:
|
|||||||
- name: HEALTH_PORT
|
- name: HEALTH_PORT
|
||||||
value: "9909"
|
value: "9909"
|
||||||
image: quay.io/k8scsi/livenessprobe:v1.1.0
|
image: quay.io/k8scsi/livenessprobe:v1.1.0
|
||||||
name: liveness-probe
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- mountPath: /csi
|
- mountPath: /csi
|
||||||
name: socket-dir
|
name: socket-dir
|
||||||
volumes:
|
|
||||||
- emptyDir: {}
|
|
||||||
name: socket-dir
|
|
6
deploy/storageclass.yaml
Normal file
6
deploy/storageclass.yaml
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
---
|
||||||
|
apiVersion: storage.k8s.io/v1
|
||||||
|
kind: StorageClass
|
||||||
|
metadata:
|
||||||
|
name: p5x
|
||||||
|
provisioner: p5x
|
@ -18,6 +18,7 @@ package csi
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"k8s.io/klog"
|
||||||
|
|
||||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
@ -27,6 +28,7 @@ import (
|
|||||||
var (
|
var (
|
||||||
controllerCaps = []csi.ControllerServiceCapability_RPC_Type{
|
controllerCaps = []csi.ControllerServiceCapability_RPC_Type{
|
||||||
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
||||||
|
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -45,6 +47,7 @@ func newControllerService(p5x *p5xApi) controllerService {
|
|||||||
|
|
||||||
// CreateVolume creates a volume
|
// CreateVolume creates a volume
|
||||||
func (d *controllerService) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
|
func (d *controllerService) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
|
||||||
|
klog.Infof("controller.CreateVolume: %s (size: %d)", request.Name, request.CapacityRange.GetRequiredBytes())
|
||||||
if len(request.Name) == 0 {
|
if len(request.Name) == 0 {
|
||||||
return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty")
|
return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty")
|
||||||
}
|
}
|
||||||
@ -64,6 +67,7 @@ func (d *controllerService) CreateVolume(ctx context.Context, request *csi.Creat
|
|||||||
CapacityBytes: vol.SizeInBytes,
|
CapacityBytes: vol.SizeInBytes,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
klog.Info("controller.CreateVolume: Created volume: ", vol)
|
||||||
return &csi.CreateVolumeResponse{Volume: &csiVol}, nil
|
return &csi.CreateVolumeResponse{Volume: &csiVol}, nil
|
||||||
|
|
||||||
// volCtx := make(map[string]string)
|
// volCtx := make(map[string]string)
|
||||||
@ -82,7 +86,9 @@ func (d *controllerService) CreateVolume(ctx context.Context, request *csi.Creat
|
|||||||
|
|
||||||
// DeleteVolume deletes a volume
|
// DeleteVolume deletes a volume
|
||||||
func (d *controllerService) DeleteVolume(ctx context.Context, request *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
|
func (d *controllerService) DeleteVolume(ctx context.Context, request *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
|
||||||
vol, err := d.p5x.GetVolumeByName(request.VolumeId)
|
klog.Infof("controller.DeleteVolume: %s", request.GetVolumeId())
|
||||||
|
|
||||||
|
vol, err := d.p5x.GetVolumeByName(request.GetVolumeId())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -92,11 +98,13 @@ func (d *controllerService) DeleteVolume(ctx context.Context, request *csi.Delet
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
klog.Info("controller.DeleteVolume: Successfully deleted volume: ", vol)
|
||||||
return &csi.DeleteVolumeResponse{}, nil
|
return &csi.DeleteVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerGetCapabilities get controller capabilities
|
// ControllerGetCapabilities get controller capabilities
|
||||||
func (d *controllerService) ControllerGetCapabilities(ctx context.Context, request *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
|
func (d *controllerService) ControllerGetCapabilities(ctx context.Context, request *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
|
||||||
|
klog.Info("controller.ControllerGetCapabilities: called")
|
||||||
var caps []*csi.ControllerServiceCapability
|
var caps []*csi.ControllerServiceCapability
|
||||||
for _, cap := range controllerCaps {
|
for _, cap := range controllerCaps {
|
||||||
c := &csi.ControllerServiceCapability{
|
c := &csi.ControllerServiceCapability{
|
||||||
@ -113,55 +121,66 @@ func (d *controllerService) ControllerGetCapabilities(ctx context.Context, reque
|
|||||||
|
|
||||||
// ControllerPublishVolume publish a volume
|
// ControllerPublishVolume publish a volume
|
||||||
func (d *controllerService) ControllerPublishVolume(ctx context.Context, request *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
func (d *controllerService) ControllerPublishVolume(ctx context.Context, request *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ControllerPublishVolume: %s on node %s", request.GetVolumeId(), request.GetNodeId())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ControllerPublishVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerUnpublishVolume unpublish a volume
|
// ControllerUnpublishVolume unpublish a volume
|
||||||
func (d *controllerService) ControllerUnpublishVolume(ctx context.Context, request *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
|
func (d *controllerService) ControllerUnpublishVolume(ctx context.Context, request *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ControllerUnpublishVolume: %s on node %s", request.GetVolumeId(), request.GetNodeId())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ControllerUnpublishVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateVolumeCapabilities validate volume capabilities
|
// ValidateVolumeCapabilities validate volume capabilities
|
||||||
func (d *controllerService) ValidateVolumeCapabilities(ctx context.Context, request *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
|
func (d *controllerService) ValidateVolumeCapabilities(ctx context.Context, request *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ValidateVolumeCapabilities: %s", request.GetVolumeId())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ValidateVolumeCapabilities")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListVolumes list volumes
|
// ListVolumes list volumes
|
||||||
func (d *controllerService) ListVolumes(ctx context.Context, request *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
|
func (d *controllerService) ListVolumes(ctx context.Context, request *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ListVolumes: called")
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ListVolumes")
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCapacity get capacity
|
// GetCapacity get capacity
|
||||||
func (d *controllerService) GetCapacity(ctx context.Context, request *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
|
func (d *controllerService) GetCapacity(ctx context.Context, request *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.GetCapacity: called")
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.GetCapacity")
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateSnapshot create a snapshot
|
// CreateSnapshot create a snapshot
|
||||||
func (d *controllerService) CreateSnapshot(ctx context.Context, request *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
|
func (d *controllerService) CreateSnapshot(ctx context.Context, request *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.CreateSnapshot: called")
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.CreateSnapshot")
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteSnapshot delete a snapshot
|
// DeleteSnapshot delete a snapshot
|
||||||
func (d *controllerService) DeleteSnapshot(ctx context.Context, request *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
|
func (d *controllerService) DeleteSnapshot(ctx context.Context, request *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.DeleteSnapshot: called")
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.DeleteSnapshot")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListSnapshots list snapshots
|
// ListSnapshots list snapshots
|
||||||
func (d *controllerService) ListSnapshots(ctx context.Context, request *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
|
func (d *controllerService) ListSnapshots(ctx context.Context, request *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ListSnapshots: called")
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ListSnapshots")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerExpandVolume expand a volume
|
// ControllerExpandVolume expand a volume
|
||||||
func (d *controllerService) ControllerExpandVolume(ctx context.Context, request *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
|
func (d *controllerService) ControllerExpandVolume(ctx context.Context, request *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ControllerExpandVolume: %s (size: %d)", request.GetVolumeId(), request.GetCapacityRange().GetRequiredBytes())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ControllerExpandVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerGetVolume get a volume
|
// ControllerGetVolume get a volume
|
||||||
func (d *controllerService) ControllerGetVolume(ctx context.Context, request *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {
|
func (d *controllerService) ControllerGetVolume(ctx context.Context, request *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ControllerGetVolume: %s", request.GetVolumeId())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ControllerGetVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ControllerModifyVolume modify a volume
|
// ControllerModifyVolume modify a volume
|
||||||
func (d *controllerService) ControllerModifyVolume(ctx context.Context, request *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) {
|
func (d *controllerService) ControllerModifyVolume(ctx context.Context, request *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("controller.ControllerModifyVolume: %s", request.GetVolumeId())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "controller.ControllerModifyVolume")
|
||||||
}
|
}
|
||||||
|
@ -50,7 +50,7 @@ type Driver struct {
|
|||||||
|
|
||||||
// NewDriver creates a new driver
|
// NewDriver creates a new driver
|
||||||
func NewDriver(endpoint string, nodeID string, p5xEndpoint string, p5xToken string, p5xPort int64) *Driver {
|
func NewDriver(endpoint string, nodeID string, p5xEndpoint string, p5xToken string, p5xPort int64) *Driver {
|
||||||
klog.Infof("Driver: %v version %v commit %v date %v", DriverName, driverVersion, gitCommit, buildDate)
|
klog.Infof("Driver: %v | Version: %v | Commit: %v | Date: %v", DriverName, driverVersion, gitCommit, buildDate)
|
||||||
|
|
||||||
p5x := &p5xApi{
|
p5x := &p5xApi{
|
||||||
endpoint: p5xEndpoint,
|
endpoint: p5xEndpoint,
|
||||||
|
@ -18,6 +18,7 @@ package csi
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"k8s.io/klog"
|
||||||
|
|
||||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
@ -53,16 +54,20 @@ func newNodeService(nodeID string) nodeService {
|
|||||||
|
|
||||||
// NodeStageVolume is called by the CO when a workload that wants to use the specified volume is placed (scheduled) on a node.
|
// NodeStageVolume is called by the CO when a workload that wants to use the specified volume is placed (scheduled) on a node.
|
||||||
func (n *nodeService) NodeStageVolume(ctx context.Context, request *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
|
func (n *nodeService) NodeStageVolume(ctx context.Context, request *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("node.NodeStageVolume: Staging volume %s -> %s", request.GetVolumeId(), request.GetStagingTargetPath())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "node.NodeStageVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeUnstageVolume is called by the CO when a workload that was using the specified volume is being moved to a different node.
|
// NodeUnstageVolume is called by the CO when a workload that was using the specified volume is being moved to a different node.
|
||||||
func (n *nodeService) NodeUnstageVolume(ctx context.Context, request *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
|
func (n *nodeService) NodeUnstageVolume(ctx context.Context, request *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("node.NodeUnstageVolume: Staging volume %s -> %s", request.GetVolumeId(), request.GetStagingTargetPath())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "node.NodeUnstageVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodePublishVolume mounts the volume on the node.
|
// NodePublishVolume mounts the volume on the node.
|
||||||
func (n *nodeService) NodePublishVolume(ctx context.Context, request *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
func (n *nodeService) NodePublishVolume(ctx context.Context, request *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
||||||
|
klog.Infof("node.NodePublishVolume: Publishing volume %s -> %s", request.GetVolumeId(), request.GetTargetPath())
|
||||||
|
|
||||||
volumeID := request.GetVolumeId()
|
volumeID := request.GetVolumeId()
|
||||||
if len(volumeID) == 0 {
|
if len(volumeID) == 0 {
|
||||||
return nil, status.Error(codes.InvalidArgument, "Volume id not provided")
|
return nil, status.Error(codes.InvalidArgument, "Volume id not provided")
|
||||||
@ -101,11 +106,14 @@ func (n *nodeService) NodePublishVolume(ctx context.Context, request *csi.NodePu
|
|||||||
|
|
||||||
// TODO modify your volume mount logic here
|
// TODO modify your volume mount logic here
|
||||||
|
|
||||||
return &csi.NodePublishVolumeResponse{}, nil
|
return nil, status.Error(codes.Unimplemented, "node.NodePublishVolume")
|
||||||
|
// return &csi.NodePublishVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeUnpublishVolume unmount the volume from the target path
|
// NodeUnpublishVolume unmount the volume from the target path
|
||||||
func (n *nodeService) NodeUnpublishVolume(ctx context.Context, request *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
|
func (n *nodeService) NodeUnpublishVolume(ctx context.Context, request *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
|
||||||
|
klog.Infof("node.NodeUnpublishVolume: Unpublishing volume %s -> %s", request.GetVolumeId(), request.GetTargetPath())
|
||||||
|
|
||||||
target := request.GetTargetPath()
|
target := request.GetTargetPath()
|
||||||
if len(target) == 0 {
|
if len(target) == 0 {
|
||||||
return nil, status.Error(codes.InvalidArgument, "Target path not provided")
|
return nil, status.Error(codes.InvalidArgument, "Target path not provided")
|
||||||
@ -113,26 +121,31 @@ func (n *nodeService) NodeUnpublishVolume(ctx context.Context, request *csi.Node
|
|||||||
|
|
||||||
// TODO modify your volume umount logic here
|
// TODO modify your volume umount logic here
|
||||||
|
|
||||||
return &csi.NodeUnpublishVolumeResponse{}, nil
|
return nil, status.Error(codes.Unimplemented, "node.NodeUnpublishVolume")
|
||||||
|
// return &csi.NodeUnpublishVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeGetVolumeStats get the volume stats
|
// NodeGetVolumeStats get the volume stats
|
||||||
func (n *nodeService) NodeGetVolumeStats(ctx context.Context, request *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
|
func (n *nodeService) NodeGetVolumeStats(ctx context.Context, request *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("node.NodeGetVolumeStats: For volume %s -> %s", request.GetVolumeId(), request.GetVolumePath())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "node.NodeGetVolumeStats")
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeExpandVolume expand the volume
|
// NodeExpandVolume expand the volume
|
||||||
func (n *nodeService) NodeExpandVolume(ctx context.Context, request *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
|
func (n *nodeService) NodeExpandVolume(ctx context.Context, request *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("node.NodeExpandVolume: %s -> %s (min bytes: %d)", request.GetVolumeId(), request.GetVolumePath(), request.GetCapacityRange().GetRequiredBytes())
|
||||||
|
return nil, status.Error(codes.Unimplemented, "node.NodeExpandVolume")
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeGetCapabilities get the node capabilities
|
// NodeGetCapabilities get the node capabilities
|
||||||
func (n *nodeService) NodeGetCapabilities(ctx context.Context, request *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
|
func (n *nodeService) NodeGetCapabilities(ctx context.Context, request *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
|
||||||
|
klog.Infof("node.NodeGetCapabilities: Called")
|
||||||
return &csi.NodeGetCapabilitiesResponse{}, nil
|
return &csi.NodeGetCapabilitiesResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeGetInfo get the node info
|
// NodeGetInfo get the node info
|
||||||
func (n *nodeService) NodeGetInfo(ctx context.Context, request *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
|
func (n *nodeService) NodeGetInfo(ctx context.Context, request *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
|
||||||
|
klog.Infof("node.NodeGetInfo: Called")
|
||||||
return &csi.NodeGetInfoResponse{NodeId: n.nodeID}, nil
|
return &csi.NodeGetInfoResponse{NodeId: n.nodeID}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -4,8 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"google.golang.org/grpc/codes"
|
|
||||||
"google.golang.org/grpc/status"
|
|
||||||
"io"
|
"io"
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -29,6 +27,7 @@ func (p5x *p5xApi) CreateVolume(name string, sizeInBytes int64) (*p5xVolume, err
|
|||||||
Name: name,
|
Name: name,
|
||||||
SizeInBytes: sizeInBytes,
|
SizeInBytes: sizeInBytes,
|
||||||
}
|
}
|
||||||
|
klog.Info("p5x.CreateVolume: Attempting to create: ", vol)
|
||||||
|
|
||||||
body, err := json.Marshal(vol)
|
body, err := json.Marshal(vol)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -41,47 +40,67 @@ func (p5x *p5xApi) CreateVolume(name string, sizeInBytes int64) (*p5xVolume, err
|
|||||||
}
|
}
|
||||||
|
|
||||||
err = json.Unmarshal(resBody, vol)
|
err = json.Unmarshal(resBody, vol)
|
||||||
klog.Info("Successfully created volume: ", vol)
|
klog.Info("p5x.CreateVolume: Successfully created volume: ", vol)
|
||||||
|
|
||||||
return vol, nil
|
return vol, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p5x *p5xApi) GetVolumeByName(name string) (*p5xVolume, error) {
|
func (p5x *p5xApi) GetVolumeByName(name string) (*p5xVolume, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
klog.Infof("p5x.GetVolumeByName: %s", name)
|
||||||
}
|
route := fmt.Sprintf("volumes/%s", name)
|
||||||
|
resBody, err := p5x.MakeRequest(http.MethodGet, route, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
func (p5x *p5xApi) GetVolumeById(volumeId int64) (*p5xVolume, error) {
|
vol := &p5xVolume{}
|
||||||
return nil, status.Error(codes.Unimplemented, "")
|
err = json.Unmarshal(resBody, vol)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
klog.Info("p5x.GetVolumeByName: Retrieved volume: ", vol)
|
||||||
|
return vol, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p5x *p5xApi) DeleteVolume(volume *p5xVolume) error {
|
func (p5x *p5xApi) DeleteVolume(volume *p5xVolume) error {
|
||||||
|
klog.Info("p5x.DeleteVolume: ", volume)
|
||||||
route := fmt.Sprintf("volumes/%s", volume.Name)
|
route := fmt.Sprintf("volumes/%s", volume.Name)
|
||||||
resBody, err := p5x.MakeRequest(http.MethodDelete, route, []byte(`{}`))
|
resBody, err := p5x.MakeRequest(http.MethodDelete, route, []byte(`{}`))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
klog.Infof("Successfully deleted volume %s: %s", volume.Name, resBody)
|
klog.Infof("p5x.DeleteVolume: Successfully deleted volume %s: %s", volume.Name, resBody)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p5x *p5xApi) MakeRequest(method string, route string, body []byte) ([]byte, error) {
|
func (p5x *p5xApi) MakeRequest(method string, route string, body []byte) ([]byte, error) {
|
||||||
bodyReader := bytes.NewReader(body)
|
|
||||||
|
|
||||||
url := fmt.Sprintf("%s:%d/api/v1/%s", p5x.endpoint, p5x.port, route)
|
url := fmt.Sprintf("%s:%d/api/v1/%s", p5x.endpoint, p5x.port, route)
|
||||||
klog.Infof("p5x.MakeRequest: [%s] %s", method, url)
|
klog.Infof("p5x.MakeRequest: [%s] %s", method, url)
|
||||||
klog.Infof("p5x.MakeRequest: %s", body)
|
|
||||||
req, err := http.NewRequest(method, url, bodyReader)
|
var res *http.Response
|
||||||
|
var err error
|
||||||
|
if method == http.MethodGet {
|
||||||
|
res, err = http.Get(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Errorf("p5x.MakeRequest: could not create request: %s\n", err)
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
klog.Infof("p5x.MakeRequest: body: %s", body)
|
||||||
|
bodyReader := bytes.NewReader(body)
|
||||||
|
req, err2 := http.NewRequest(method, url, bodyReader)
|
||||||
|
if err2 != nil {
|
||||||
|
klog.Errorf("p5x.MakeRequest: could not create request: %s\n", err)
|
||||||
|
return nil, err2
|
||||||
|
}
|
||||||
|
|
||||||
req.Header.Set("Accept", "application/json")
|
req.Header.Set("Accept", "application/json")
|
||||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", p5x.token))
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", p5x.token))
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
res, err = http.DefaultClient.Do(req)
|
||||||
|
}
|
||||||
|
|
||||||
res, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Errorf("p5x.MakeRequest: error executing request: %s\n", err)
|
klog.Errorf("p5x.MakeRequest: error executing request: %s\n", err)
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -24,9 +24,9 @@ import (
|
|||||||
|
|
||||||
// These are set during build time via -ldflags
|
// These are set during build time via -ldflags
|
||||||
var (
|
var (
|
||||||
driverVersion string
|
driverVersion string = "v0.0.1"
|
||||||
gitCommit string
|
gitCommit string = "master"
|
||||||
buildDate string
|
buildDate string = "2024-09-28"
|
||||||
)
|
)
|
||||||
|
|
||||||
// VersionInfo struct
|
// VersionInfo struct
|
||||||
|
Loading…
Reference in New Issue
Block a user