juicefs-csi

references

https://github.com/juicedata/juicefs-csi-driver
https://github.com/juicedata/charts/tree/main/charts/juicefs-csi-driver

prepare

installation

prepare namespace storage

kubectl get namespace storage > /dev/null 2>&1 || kubectl create namespace storage

prepare dashboard secret

kubectl -n storage create secret generic juicefs-dashboard-secret \
  --from-literal=username=admin \
  --from-literal=password=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 16)

prepare juicefs-csi.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: juicefs-csi
spec:
  syncPolicy:
    syncOptions:
    - CreateNamespace=true
  project: default
  source:
    repoURL: https://juicedata.github.io/charts
    chart: juicefs-csi-driver
    targetRevision: 0.28.2
    helm:
      releaseName: juicefs-csi
      valuesObject:
        image:
          repository: m.daocloud.io/docker.io/juicedata/juicefs-csi-driver
        dashboardImage:
          repository: m.daocloud.io/docker.io/juicedata/csi-dashboard
        sidecars:
          livenessProbeImage:
            repository: m.daocloud.io/registry.k8s.io/sig-storage/livenessprobe
          nodeDriverRegistrarImage:
            repository: m.daocloud.io/registry.k8s.io/sig-storage/csi-node-driver-registrar
          csiProvisionerImage:
            repository: m.daocloud.io/registry.k8s.io/sig-storage/csi-provisioner
          csiResizerImage:
            repository: m.daocloud.io/registry.k8s.io/sig-storage/csi-resizer
        imagePullSecrets: []
        mountMode: mountpod
        driverName: "csi.juicefs.com"
        jfsMountDir: /var/lib/juicefs/volume
        jfsConfigDir: /var/lib/juicefs/config
        immutable: false
        controller:
          enabled: true
          cacheClientConf: false
          replicas: 3
          resources:
            limits:
              cpu: 1000m
              memory: 1Gi
            requests:
              cpu: 100m
              memory: 512Mi
        node:
          enabled: true
          resources:
            limits:
              cpu: 1000m
              memory: 1Gi
            requests:
              cpu: 100m
              memory: 512Mi
        validatingWebhook:
          enabled: true
        dashboard:
          enabled: true
          enableManager: true
          auth:
            enabled: true
            existingSecret: "juicefs-dashboard-secret"
          replicas: 1
          resources:
            limits:
              cpu: 1000m
              memory: 1Gi
            requests:
              cpu: 100m
              memory: 200Mi
          ingress:
            enabled: true
            className: "nginx"
            annotations:
              cert-manager.io/cluster-issuer: self-signed-ca-issuer
            hosts:
            - host: "juice-fs-dashboard.dev.geekcity.tech"
              paths:
              - path: /
                pathType: ImplementationSpecific
            tls:
            - secretName: juice-fs-dashboard.dev.geekcity.tech-tls
              hosts:
              - juice-fs-dashboard.dev.geekcity.tech
  destination:
    server: https://kubernetes.default.svc
    namespace: storage

apply to k8s

kubectl -n argocd apply -f juicefs-csi.yaml

sync by argocd
- ```
argocd app sync argocd/juicefs-csi
```

visit juice-fs-dashboard by broswer

https://juice-fs-dashboard.dev.geekcity.tech

credentials

# username
kubectl -n storage get secret juicefs-dashboard-secret -o jsonpath='{.data.username}' | base64 -d && echo

# password
kubectl -n storage get secret juicefs-dashboard-secret -o jsonpath='{.data.password}' | base64 -d && echo

pvc

create a secret to store juice fs credentials

MINIO_ACCESS_KEY=$(kubectl -n storage get secret minio-credentials -o jsonpath='{.data.rootUser}' | base64 -d)
MINIO_SECRET_KEY=$(kubectl -n storage get secret minio-credentials -o jsonpath='{.data.rootPassword}' | base64 -d)
kubectl -n storage create secret generic juice-fs-tidb-minio-credential \
  --from-literal=name=juice-fs-tidb-minio \
  --from-literal=metaurl=tikv://basic-pd.tidb-cluster:2379/juice-fs-tidb-minio \
  --from-literal=storage=minio \
  --from-literal=bucket=http://minio-headless.storage:9000/juice-fs-tidb-minio \
  --from-literal=access-key=${MINIO_ACCESS_KEY} \
  --from-literal=secret-key=${MINIO_SECRET_KEY}
kubectl -n storage patch secret juice-fs-tidb-minio-credential -p '{"metadata":{"labels":{"juicefs.com/validate-secret":"true"}}}'

create pvc and test with a deployment

prepare busybox-juice-fs-pvc-test.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: busybox-juice-fs-pv
  labels:
    juicefs-name: juice-fs-tidb-minio
spec:
  # any valid value will work, 
  # but the capacity is not supported by JuiceFS CSI driver yet 
  capacity:
    storage: 10Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  csi:
    driver: csi.juicefs.com
    # unique, recommend to be same as pv name
    volumeHandle: busybox-juice-fs-pv
    fsType: juicefs
    nodePublishSecretRef:
      name: juice-fs-tidb-minio-credential
      namespace: storage

---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: busybox-juice-fs-pvc
  namespace: default
spec:
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  storageClassName: ""
  # any value smaller than or equal to PV capacity will work
  # but note that the capacity of JuiceFS CSI driver is not supported yet
  resources:
    requests:
      storage: 10Gi
  selector:
    matchLabels:
      juicefs-name: juice-fs-tidb-minio

---

apiVersion: batch/v1
kind: Job
metadata:
  name: busybox-juice-fs-pv-test-job
spec:
  completions: 1
  parallelism: 1
  backoffLimit: 4
  template:
    metadata:
      labels:
        app: busybox-juice-fs-pv-test
    spec:
      restartPolicy: Never
      containers:
      - name: busybox-juice-fs-pv-test
        image: m.daocloud.io/docker.io/library/busybox:latest
        command:
          - sh
          - -c
          - |
            echo "Writing to PVC..."
            RANDOM_STR=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 16)
            echo "hello local path provisioner $RANDOM_STR" > /data/hello.txt
            echo "Reading from PVC..."
            READ_STR=$(cat /data/hello.txt)
            echo "Written content: hello local path provisioner $RANDOM_STR"
            echo "Read content: $READ_STR"
            if [ "$READ_STR" = "hello local path provisioner $RANDOM_STR" ]; then
              echo "PVC test completed successfully!"
            else
              echo "PVC test failed!"
              exit 1
            fi
        volumeMounts:
        - name: data-volume
          mountPath: /data
      volumes:
      - name: data-volume
        persistentVolumeClaim:
          claimName: busybox-juice-fs-pvc

apply to k8s

kubectl -n default apply -f busybox-juice-fs-pvc-test.yaml

storage class

create storage class

prepare juice-fs-tidb-minio-test.storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: juice-fs-tidb-minio
provisioner: csi.juicefs.com
parameters:
  csi.storage.k8s.io/provisioner-secret-name: juice-fs-tidb-minio-credential
  csi.storage.k8s.io/provisioner-secret-namespace: storage
  csi.storage.k8s.io/node-publish-secret-name: juice-fs-tidb-minio-credential
  csi.storage.k8s.io/node-publish-secret-namespace: storage
  pathPattern: "${.pvc.namespace}-${.pvc.name}"
reclaimPolicy: Retain
allowVolumeExpansion: true

apply to k8s

kubectl -n default apply -f juice-fs-tidb-minio-test.storageclass.yaml

create pvc and test with a deployment

prepare busybox-storage-class-test.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: busybox-storage-class-test-pvc
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: juice-fs-tidb-minio

---

apiVersion: batch/v1
kind: Job
metadata:
  name: busybox-storage-class-test-job
spec:
  completions: 1
  parallelism: 1
  backoffLimit: 4
  template:
    metadata:
      labels:
        app: busybox-storage-class-test
    spec:
      restartPolicy: Never
      containers:
      - name: busybox-pvc-test
        image: m.daocloud.io/docker.io/library/busybox:latest
        command:
          - sh
          - -c
          - |
            echo "Writing to PVC..."
            RANDOM_STR=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 16)
            echo "hello local path provisioner $RANDOM_STR" > /data/hello.txt
            echo "Reading from PVC..."
            READ_STR=$(cat /data/hello.txt)
            echo "Written content: hello local path provisioner $RANDOM_STR"
            echo "Read content: $READ_STR"
            if [ "$READ_STR" = "hello local path provisioner $RANDOM_STR" ]; then
              echo "PVC test completed successfully!"
            else
              echo "PVC test failed!"
              exit 1
            fi
        volumeMounts:
        - name: juice-fs-tidb-minio-vol
          mountPath: /data
      volumes:
      - name: juice-fs-tidb-minio-vol
        persistentVolumeClaim:
          claimName: busybox-storage-class-test-pvc

apply to k8s

kubectl -n default apply -f busybox-storage-class-test.yaml

juicefs-csi

juicefs-csi

references

prepare

installation

pvc

storage class

advanced topics