ceph
镜像拉取
由于下载不到国外的镜像,只能使用这个笨办法了
下载国内镜像(所有节点执行)
docker pull xxx/cephcsi:v3.6.1-xxx2.8.3.1216
docker pull registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0
docker pull registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1
docker pull registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0docker tag xxx/cephcsi:v3.6.1-csp2.8.3.1216 quay.io/cephcsi/cephcsi:v3.6.1
docker tag registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
docker tag registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
docker tag registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
下载源码
git clone .git -b release-v3.4
cd ceph-csi/deploy/cephfs/kubernetes
修改yaml文件
把文件csi-config-map.yaml修改成
---
apiVersion: v1
kind: ConfigMap
data:config.json: |-[{"clusterID": "a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e","monitors": ["172.27.16.11:6789","172.27.16.3:6789","172.27.16.7:6789"]}]
metadata:name: ceph-csi-config
创建ceph-conf.yaml
---
# This is a sample configmap that helps define a Ceph configuration as required
# by the CSI plugins.# Sample ceph.conf available at
# .ceph.conf Detailed
# documentation is available at
# /
apiVersion: v1
kind: ConfigMap
data:ceph.conf: |[global]auth_cluster_required = cephxauth_service_required = cephxauth_client_required = cephx# Workaround for = false# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB# adding 'fuse_big_writes = true' option by default to override this limit# see = true# keyring is a required key and its value should be emptykeyring: |
metadata:name: ceph-config
创建ceph-csi-encryption-kms-config.yaml
---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-{"vault-test": {"encryptionKMSType": "vault","vaultAddress": ":8200","vaultAuthPath": "/v1/auth/kubernetes/login","vaultRole": "csi-kubernetes","vaultBackend": "kv-v2","vaultDestroyKeys": "true","vaultPassphraseRoot": "/v1/secret","vaultPassphrasePath": "ceph-csi/","vaultCAVerify": "false"},"vault-tokens-test": {"encryptionKMSType": "vaulttokens","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "secret/","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantTokenName": "ceph-csi-kms-token","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantTokenName": "storage-encryption-token","vaultDestroyKeys": "false"}}},"vault-tenant-sa-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultDestroyKeys": "false","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantSAName": "storage-encryption-sa"}}},"vault-tenant-sa-ns-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultAuthNamespace": "devops","vaultNamespace": "devops/homepage","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"webservers": {"vaultAddress": "","vaultAuthNamespace": "webservers","vaultNamespace": "webservers/homepage","vaultCAVerify": "true"},"homepage-db": {"vaultNamespace": "devops/homepage/database","tenantSAName": "storage-encryption-sa"}}},"secrets-metadata-test": {"encryptionKMSType": "metadata"},"user-ns-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret","secretNamespace": "default"},"user-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret"},"ibmkeyprotect-test": {"encryptionKMSType": "ibmkeyprotect","secretName": "ceph-csi-kp-credentials","keyProtectRegionKey": "us-south-2","keyProtectServiceInstanceID": "7abef064-01dd-4237-9ea5-8b3890970be3"},"aws-sts-metadata-test": {"encryptionKMSType": "aws-sts-metadata","secretName": "ceph-csi-aws-credentials"},"kmip-test": {"KMS_PROVIDER": "kmip","KMIP_ENDPOINT": "kmip:5696","KMIP_SECRET_NAME": "ceph-csi-kmip-credentials","TLS_SERVER_NAME": "kmip.ciphertrustmanager.local","READ_TIMEOUT": 10,"WRITE_TIMEOUT": 10}}
metadata:
name: ceph-csi-encryption-kms-config
Footer
© 2023 GitHub, Inc.
Footer navigation
Terms
Privacy
在主节点执行
kubectl taint nodes k8s-master node-role.kubernetes.io/master-
不执行这句,会导致调度到主节点的pod处于appending状态
创建pod
kubectl apply -f ceph-csi/deploy/rbd/kubernetes/
创建成功
[root@VM-16-3-centos data]# kubectl get pods
NAME READY STATUS RESTARTS AGE
csi-rbdplugin-jw8v4 3/3 Running 0 43m
csi-rbdplugin-pncb4 3/3 Running 0 43m
csi-rbdplugin-provisioner-58ff6984fd-947m2 7/7 Running 0 3m58s
csi-rbdplugin-provisioner-58ff6984fd-9mwtl 7/7 Running 0 43m
csi-rbdplugin-provisioner-58ff6984fd-mz8r8 7/7 Running 0 43m
csi-rbdplugin-wzmlp 3/3 Running 0 43m
创建csi-secret.yaml
---
apiVersion: v1
kind: Secret
metadata:name: csi-rbd-secretnamespace: default
stringData:userID: adminuserKey: AQDRuF1kAAAAABAAS3AdiAWbYfhVzg+EjcQqNw==
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:clusterID: a674ff7d-229c-4af1-b7b1-f4e5b0d52c2epool: rbddataimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: defaultcsi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard
创建pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: rbd-pvc
spec:accessModes:- ReadWriteOnceresources:requests:storage: 1GistorageClassName: csi-rbd-sc
[root@VM-16-3-centos rbd]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-ad185da7-d9de-4520-b51d-6e61223d9042 1Gi RWO Delete Bound default/rbd-pvc csi-rbd-sc 3m32s
[root@VM-16-3-centos rbd]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
rbd-pvc Bound pvc-ad185da7-d9de-4520-b51d-6e61223d9042 1Gi RWO csi-rbd-sc 30m
[root@VM-16-3-centos rbd]# rbd ls -p rbddata
csi-vol-5cd46f69-f0c3-11ed-bb18-6ef31c6b7f26
创建POD使用pvc
apiVersion: v1
kind: Pod
metadata:name: centos
spec:containers:- name: mypod1image: centos:centos8args:- /bin/bash- -c- sleep 10; touch /tmp/healthy; sleep 30000volumeMounts:- mountPath: "/mydata"name: mydatavolumes:- name: mydatapersistentVolumeClaim:claimName: rbd-pvc
pod创建成功
[root@k8s-node2 rbd]# kubectl get pods
NAME READY STATUS RESTARTS AGE
centos 1/1 Running 0 8m7s
csi-rbdplugin-jgrsd 3/3 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-dz6lx 7/7 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-mppdr 7/7 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-tzl6j 7/7 Running 0 14m
csi-rbdplugin-rjbnd 3/3 Running 0 14m
csi-rbdplugin-sx446 3/3 Running 0 14m
ceph
镜像拉取
由于下载不到国外的镜像,只能使用这个笨办法了
下载国内镜像(所有节点执行)
docker pull xxx/cephcsi:v3.6.1-xxx2.8.3.1216
docker pull registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0
docker pull registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1
docker pull registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0docker tag xxx/cephcsi:v3.6.1-csp2.8.3.1216 quay.io/cephcsi/cephcsi:v3.6.1
docker tag registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
docker tag registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
docker tag registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
下载源码
git clone .git -b release-v3.4
cd ceph-csi/deploy/cephfs/kubernetes
修改yaml文件
把文件csi-config-map.yaml修改成
---
apiVersion: v1
kind: ConfigMap
data:config.json: |-[{"clusterID": "a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e","monitors": ["172.27.16.11:6789","172.27.16.3:6789","172.27.16.7:6789"]}]
metadata:name: ceph-csi-config
创建ceph-conf.yaml
---
# This is a sample configmap that helps define a Ceph configuration as required
# by the CSI plugins.# Sample ceph.conf available at
# .ceph.conf Detailed
# documentation is available at
# /
apiVersion: v1
kind: ConfigMap
data:ceph.conf: |[global]auth_cluster_required = cephxauth_service_required = cephxauth_client_required = cephx# Workaround for = false# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB# adding 'fuse_big_writes = true' option by default to override this limit# see = true# keyring is a required key and its value should be emptykeyring: |
metadata:name: ceph-config
创建ceph-csi-encryption-kms-config.yaml
---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-{"vault-test": {"encryptionKMSType": "vault","vaultAddress": ":8200","vaultAuthPath": "/v1/auth/kubernetes/login","vaultRole": "csi-kubernetes","vaultBackend": "kv-v2","vaultDestroyKeys": "true","vaultPassphraseRoot": "/v1/secret","vaultPassphrasePath": "ceph-csi/","vaultCAVerify": "false"},"vault-tokens-test": {"encryptionKMSType": "vaulttokens","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "secret/","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantTokenName": "ceph-csi-kms-token","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantTokenName": "storage-encryption-token","vaultDestroyKeys": "false"}}},"vault-tenant-sa-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultDestroyKeys": "false","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantSAName": "storage-encryption-sa"}}},"vault-tenant-sa-ns-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultAuthNamespace": "devops","vaultNamespace": "devops/homepage","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"webservers": {"vaultAddress": "","vaultAuthNamespace": "webservers","vaultNamespace": "webservers/homepage","vaultCAVerify": "true"},"homepage-db": {"vaultNamespace": "devops/homepage/database","tenantSAName": "storage-encryption-sa"}}},"secrets-metadata-test": {"encryptionKMSType": "metadata"},"user-ns-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret","secretNamespace": "default"},"user-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret"},"ibmkeyprotect-test": {"encryptionKMSType": "ibmkeyprotect","secretName": "ceph-csi-kp-credentials","keyProtectRegionKey": "us-south-2","keyProtectServiceInstanceID": "7abef064-01dd-4237-9ea5-8b3890970be3"},"aws-sts-metadata-test": {"encryptionKMSType": "aws-sts-metadata","secretName": "ceph-csi-aws-credentials"},"kmip-test": {"KMS_PROVIDER": "kmip","KMIP_ENDPOINT": "kmip:5696","KMIP_SECRET_NAME": "ceph-csi-kmip-credentials","TLS_SERVER_NAME": "kmip.ciphertrustmanager.local","READ_TIMEOUT": 10,"WRITE_TIMEOUT": 10}}
metadata:
name: ceph-csi-encryption-kms-config
Footer
© 2023 GitHub, Inc.
Footer navigation
Terms
Privacy
在主节点执行
kubectl taint nodes k8s-master node-role.kubernetes.io/master-
不执行这句,会导致调度到主节点的pod处于appending状态
创建pod
kubectl apply -f ceph-csi/deploy/rbd/kubernetes/
创建成功
[root@VM-16-3-centos data]# kubectl get pods
NAME READY STATUS RESTARTS AGE
csi-rbdplugin-jw8v4 3/3 Running 0 43m
csi-rbdplugin-pncb4 3/3 Running 0 43m
csi-rbdplugin-provisioner-58ff6984fd-947m2 7/7 Running 0 3m58s
csi-rbdplugin-provisioner-58ff6984fd-9mwtl 7/7 Running 0 43m
csi-rbdplugin-provisioner-58ff6984fd-mz8r8 7/7 Running 0 43m
csi-rbdplugin-wzmlp 3/3 Running 0 43m
创建csi-secret.yaml
---
apiVersion: v1
kind: Secret
metadata:name: csi-rbd-secretnamespace: default
stringData:userID: adminuserKey: AQDRuF1kAAAAABAAS3AdiAWbYfhVzg+EjcQqNw==
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:clusterID: a674ff7d-229c-4af1-b7b1-f4e5b0d52c2epool: rbddataimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: defaultcsi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard
创建pvc
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: rbd-pvc
spec:accessModes:- ReadWriteOnceresources:requests:storage: 1GistorageClassName: csi-rbd-sc
[root@VM-16-3-centos rbd]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-ad185da7-d9de-4520-b51d-6e61223d9042 1Gi RWO Delete Bound default/rbd-pvc csi-rbd-sc 3m32s
[root@VM-16-3-centos rbd]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
rbd-pvc Bound pvc-ad185da7-d9de-4520-b51d-6e61223d9042 1Gi RWO csi-rbd-sc 30m
[root@VM-16-3-centos rbd]# rbd ls -p rbddata
csi-vol-5cd46f69-f0c3-11ed-bb18-6ef31c6b7f26
创建POD使用pvc
apiVersion: v1
kind: Pod
metadata:name: centos
spec:containers:- name: mypod1image: centos:centos8args:- /bin/bash- -c- sleep 10; touch /tmp/healthy; sleep 30000volumeMounts:- mountPath: "/mydata"name: mydatavolumes:- name: mydatapersistentVolumeClaim:claimName: rbd-pvc
pod创建成功
[root@k8s-node2 rbd]# kubectl get pods
NAME READY STATUS RESTARTS AGE
centos 1/1 Running 0 8m7s
csi-rbdplugin-jgrsd 3/3 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-dz6lx 7/7 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-mppdr 7/7 Running 0 14m
csi-rbdplugin-provisioner-58ff6984fd-tzl6j 7/7 Running 0 14m
csi-rbdplugin-rjbnd 3/3 Running 0 14m
csi-rbdplugin-sx446 3/3 Running 0 14m