最新消息: USBMI致力于为网友们分享Windows、安卓、IOS等主流手机系统相关的资讯以及评测、同时提供相关教程、应用、软件下载等服务。

ceph

IT圈 admin 9浏览 0评论

ceph

镜像拉取

由于下载不到国外的镜像,只能使用这个笨办法了

下载国内镜像(所有节点执行)

docker pull xxx/cephcsi:v3.6.1-xxx2.8.3.1216
docker pull registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0
docker pull registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1
docker pull registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0docker tag xxx/cephcsi:v3.6.1-csp2.8.3.1216 quay.io/cephcsi/cephcsi:v3.6.1
docker tag registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
docker tag registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
docker tag registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0

下载源码

git clone .git -b release-v3.4
cd ceph-csi/deploy/cephfs/kubernetes

修改yaml文件

把文件csi-config-map.yaml修改成

---
apiVersion: v1
kind: ConfigMap
data:config.json: |-[{"clusterID": "a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e","monitors": ["172.27.16.11:6789","172.27.16.3:6789","172.27.16.7:6789"]}]
metadata:name: ceph-csi-config

创建ceph-conf.yaml

---
# This is a sample configmap that helps define a Ceph configuration as required
# by the CSI plugins.# Sample ceph.conf available at
# .ceph.conf Detailed
# documentation is available at
# /
apiVersion: v1
kind: ConfigMap
data:ceph.conf: |[global]auth_cluster_required = cephxauth_service_required = cephxauth_client_required = cephx# Workaround for  = false# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB# adding 'fuse_big_writes = true' option by default to override this limit# see  = true# keyring is a required key and its value should be emptykeyring: |
metadata:name: ceph-config

创建ceph-csi-encryption-kms-config.yaml

---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-{"vault-test": {"encryptionKMSType": "vault","vaultAddress": ":8200","vaultAuthPath": "/v1/auth/kubernetes/login","vaultRole": "csi-kubernetes","vaultBackend": "kv-v2","vaultDestroyKeys": "true","vaultPassphraseRoot": "/v1/secret","vaultPassphrasePath": "ceph-csi/","vaultCAVerify": "false"},"vault-tokens-test": {"encryptionKMSType": "vaulttokens","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "secret/","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantTokenName": "ceph-csi-kms-token","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantTokenName": "storage-encryption-token","vaultDestroyKeys": "false"}}},"vault-tenant-sa-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultDestroyKeys": "false","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantSAName": "storage-encryption-sa"}}},"vault-tenant-sa-ns-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultAuthNamespace": "devops","vaultNamespace": "devops/homepage","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"webservers": {"vaultAddress": "","vaultAuthNamespace": "webservers","vaultNamespace": "webservers/homepage","vaultCAVerify": "true"},"homepage-db": {"vaultNamespace": "devops/homepage/database","tenantSAName": "storage-encryption-sa"}}},"secrets-metadata-test": {"encryptionKMSType": "metadata"},"user-ns-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret","secretNamespace": "default"},"user-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret"},"ibmkeyprotect-test": {"encryptionKMSType": "ibmkeyprotect","secretName": "ceph-csi-kp-credentials","keyProtectRegionKey": "us-south-2","keyProtectServiceInstanceID": "7abef064-01dd-4237-9ea5-8b3890970be3"},"aws-sts-metadata-test": {"encryptionKMSType": "aws-sts-metadata","secretName": "ceph-csi-aws-credentials"},"kmip-test": {"KMS_PROVIDER": "kmip","KMIP_ENDPOINT": "kmip:5696","KMIP_SECRET_NAME": "ceph-csi-kmip-credentials","TLS_SERVER_NAME": "kmip.ciphertrustmanager.local","READ_TIMEOUT": 10,"WRITE_TIMEOUT": 10}}
metadata:
name: ceph-csi-encryption-kms-config
Footer
© 2023 GitHub, Inc.
Footer navigation
Terms
Privacy

在主节点执行

kubectl taint nodes k8s-master node-role.kubernetes.io/master-

不执行这句,会导致调度到主节点的pod处于appending状态

创建pod

kubectl apply -f ceph-csi/deploy/rbd/kubernetes/

创建成功

[root@VM-16-3-centos data]# kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
csi-rbdplugin-jw8v4                          3/3     Running   0          43m
csi-rbdplugin-pncb4                          3/3     Running   0          43m
csi-rbdplugin-provisioner-58ff6984fd-947m2   7/7     Running   0          3m58s
csi-rbdplugin-provisioner-58ff6984fd-9mwtl   7/7     Running   0          43m
csi-rbdplugin-provisioner-58ff6984fd-mz8r8   7/7     Running   0          43m
csi-rbdplugin-wzmlp                          3/3     Running   0          43m                     

创建csi-secret.yaml

---
apiVersion: v1
kind: Secret
metadata:name: csi-rbd-secretnamespace: default
stringData:userID: adminuserKey: AQDRuF1kAAAAABAAS3AdiAWbYfhVzg+EjcQqNw==
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:clusterID: a674ff7d-229c-4af1-b7b1-f4e5b0d52c2epool: rbddataimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: defaultcsi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard

创建pvc

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: rbd-pvc
spec:accessModes:- ReadWriteOnceresources:requests:storage: 1GistorageClassName: csi-rbd-sc
[root@VM-16-3-centos rbd]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS   REASON   AGE
pvc-ad185da7-d9de-4520-b51d-6e61223d9042   1Gi        RWO            Delete           Bound    default/rbd-pvc   csi-rbd-sc              3m32s
[root@VM-16-3-centos rbd]# kubectl get pvc
NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
rbd-pvc   Bound    pvc-ad185da7-d9de-4520-b51d-6e61223d9042   1Gi        RWO            csi-rbd-sc     30m
[root@VM-16-3-centos rbd]# rbd ls -p rbddata
csi-vol-5cd46f69-f0c3-11ed-bb18-6ef31c6b7f26

创建POD使用pvc

apiVersion: v1
kind: Pod
metadata:name: centos
spec:containers:- name: mypod1image: centos:centos8args:- /bin/bash- -c- sleep 10; touch /tmp/healthy; sleep 30000volumeMounts:- mountPath: "/mydata"name: mydatavolumes:- name: mydatapersistentVolumeClaim:claimName: rbd-pvc

pod创建成功

[root@k8s-node2 rbd]# kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
centos                                       1/1     Running   0          8m7s
csi-rbdplugin-jgrsd                          3/3     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-dz6lx   7/7     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-mppdr   7/7     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-tzl6j   7/7     Running   0          14m
csi-rbdplugin-rjbnd                          3/3     Running   0          14m
csi-rbdplugin-sx446                          3/3     Running   0          14m

ceph

镜像拉取

由于下载不到国外的镜像,只能使用这个笨办法了

下载国内镜像(所有节点执行)

docker pull xxx/cephcsi:v3.6.1-xxx2.8.3.1216
docker pull registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0
docker pull registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1
docker pull registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0
docker pull registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0docker tag xxx/cephcsi:v3.6.1-csp2.8.3.1216 quay.io/cephcsi/cephcsi:v3.6.1
docker tag registry.aliyuncs.com/it00021hot/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
docker tag registry.aliyuncs.com/it00021hot/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-snapshotter:v5.0.1 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
docker tag registry.aliyuncs.com/it00021hot/csi-attacher:v3.4.0 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
docker tag registry.aliyuncs.com/it00021hot/csi-node-driver-registrar:v2.4.0 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0

下载源码

git clone .git -b release-v3.4
cd ceph-csi/deploy/cephfs/kubernetes

修改yaml文件

把文件csi-config-map.yaml修改成

---
apiVersion: v1
kind: ConfigMap
data:config.json: |-[{"clusterID": "a674ff7d-229c-4af1-b7b1-f4e5b0d52c2e","monitors": ["172.27.16.11:6789","172.27.16.3:6789","172.27.16.7:6789"]}]
metadata:name: ceph-csi-config

创建ceph-conf.yaml

---
# This is a sample configmap that helps define a Ceph configuration as required
# by the CSI plugins.# Sample ceph.conf available at
# .ceph.conf Detailed
# documentation is available at
# /
apiVersion: v1
kind: ConfigMap
data:ceph.conf: |[global]auth_cluster_required = cephxauth_service_required = cephxauth_client_required = cephx# Workaround for  = false# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB# adding 'fuse_big_writes = true' option by default to override this limit# see  = true# keyring is a required key and its value should be emptykeyring: |
metadata:name: ceph-config

创建ceph-csi-encryption-kms-config.yaml

---
apiVersion: v1
kind: ConfigMap
data:
config.json: |-{"vault-test": {"encryptionKMSType": "vault","vaultAddress": ":8200","vaultAuthPath": "/v1/auth/kubernetes/login","vaultRole": "csi-kubernetes","vaultBackend": "kv-v2","vaultDestroyKeys": "true","vaultPassphraseRoot": "/v1/secret","vaultPassphrasePath": "ceph-csi/","vaultCAVerify": "false"},"vault-tokens-test": {"encryptionKMSType": "vaulttokens","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "secret/","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantTokenName": "ceph-csi-kms-token","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantTokenName": "storage-encryption-token","vaultDestroyKeys": "false"}}},"vault-tenant-sa-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultDestroyKeys": "false","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"my-app": {"vaultAddress": "","vaultCAVerify": "true"},"an-other-app": {"tenantSAName": "storage-encryption-sa"}}},"vault-tenant-sa-ns-test": {"encryptionKMSType": "vaulttenantsa","vaultAddress": ":8200","vaultBackend": "kv-v2","vaultBackendPath": "shared-secrets","vaultAuthNamespace": "devops","vaultNamespace": "devops/homepage","vaultTLSServerName": "vault.default.svc.cluster.local","vaultCAVerify": "false","tenantConfigName": "ceph-csi-kms-config","tenantSAName": "ceph-csi-vault-sa","tenants": {"webservers": {"vaultAddress": "","vaultAuthNamespace": "webservers","vaultNamespace": "webservers/homepage","vaultCAVerify": "true"},"homepage-db": {"vaultNamespace": "devops/homepage/database","tenantSAName": "storage-encryption-sa"}}},"secrets-metadata-test": {"encryptionKMSType": "metadata"},"user-ns-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret","secretNamespace": "default"},"user-secrets-metadata-test": {"encryptionKMSType": "metadata","secretName": "storage-encryption-secret"},"ibmkeyprotect-test": {"encryptionKMSType": "ibmkeyprotect","secretName": "ceph-csi-kp-credentials","keyProtectRegionKey": "us-south-2","keyProtectServiceInstanceID": "7abef064-01dd-4237-9ea5-8b3890970be3"},"aws-sts-metadata-test": {"encryptionKMSType": "aws-sts-metadata","secretName": "ceph-csi-aws-credentials"},"kmip-test": {"KMS_PROVIDER": "kmip","KMIP_ENDPOINT": "kmip:5696","KMIP_SECRET_NAME": "ceph-csi-kmip-credentials","TLS_SERVER_NAME": "kmip.ciphertrustmanager.local","READ_TIMEOUT": 10,"WRITE_TIMEOUT": 10}}
metadata:
name: ceph-csi-encryption-kms-config
Footer
© 2023 GitHub, Inc.
Footer navigation
Terms
Privacy

在主节点执行

kubectl taint nodes k8s-master node-role.kubernetes.io/master-

不执行这句,会导致调度到主节点的pod处于appending状态

创建pod

kubectl apply -f ceph-csi/deploy/rbd/kubernetes/

创建成功

[root@VM-16-3-centos data]# kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
csi-rbdplugin-jw8v4                          3/3     Running   0          43m
csi-rbdplugin-pncb4                          3/3     Running   0          43m
csi-rbdplugin-provisioner-58ff6984fd-947m2   7/7     Running   0          3m58s
csi-rbdplugin-provisioner-58ff6984fd-9mwtl   7/7     Running   0          43m
csi-rbdplugin-provisioner-58ff6984fd-mz8r8   7/7     Running   0          43m
csi-rbdplugin-wzmlp                          3/3     Running   0          43m                     

创建csi-secret.yaml

---
apiVersion: v1
kind: Secret
metadata:name: csi-rbd-secretnamespace: default
stringData:userID: adminuserKey: AQDRuF1kAAAAABAAS3AdiAWbYfhVzg+EjcQqNw==
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: csi-rbd-sc
provisioner: rbd.csi.ceph.com
parameters:clusterID: a674ff7d-229c-4af1-b7b1-f4e5b0d52c2epool: rbddataimageFeatures: layeringcsi.storage.k8s.io/provisioner-secret-name: csi-rbd-secretcsi.storage.k8s.io/provisioner-secret-namespace: defaultcsi.storage.k8s.io/controller-expand-secret-name: csi-rbd-secretcsi.storage.k8s.io/controller-expand-secret-namespace: defaultcsi.storage.k8s.io/node-stage-secret-name: csi-rbd-secretcsi.storage.k8s.io/node-stage-secret-namespace: defaultcsi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:- discard

创建pvc

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: rbd-pvc
spec:accessModes:- ReadWriteOnceresources:requests:storage: 1GistorageClassName: csi-rbd-sc
[root@VM-16-3-centos rbd]# kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM             STORAGECLASS   REASON   AGE
pvc-ad185da7-d9de-4520-b51d-6e61223d9042   1Gi        RWO            Delete           Bound    default/rbd-pvc   csi-rbd-sc              3m32s
[root@VM-16-3-centos rbd]# kubectl get pvc
NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
rbd-pvc   Bound    pvc-ad185da7-d9de-4520-b51d-6e61223d9042   1Gi        RWO            csi-rbd-sc     30m
[root@VM-16-3-centos rbd]# rbd ls -p rbddata
csi-vol-5cd46f69-f0c3-11ed-bb18-6ef31c6b7f26

创建POD使用pvc

apiVersion: v1
kind: Pod
metadata:name: centos
spec:containers:- name: mypod1image: centos:centos8args:- /bin/bash- -c- sleep 10; touch /tmp/healthy; sleep 30000volumeMounts:- mountPath: "/mydata"name: mydatavolumes:- name: mydatapersistentVolumeClaim:claimName: rbd-pvc

pod创建成功

[root@k8s-node2 rbd]# kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
centos                                       1/1     Running   0          8m7s
csi-rbdplugin-jgrsd                          3/3     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-dz6lx   7/7     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-mppdr   7/7     Running   0          14m
csi-rbdplugin-provisioner-58ff6984fd-tzl6j   7/7     Running   0          14m
csi-rbdplugin-rjbnd                          3/3     Running   0          14m
csi-rbdplugin-sx446                          3/3     Running   0          14m
发布评论

评论列表 (0)

  1. 暂无评论