Problem with volume mount using Ceph CSI Plugin

Hi,

I have a problem with volume mount using Ceph CSI Plugin.

The version info of ceph and nomad is:

[opc@minio-0 ~]$ sudo ceph --version
ceph version 14.2.20 (36274af6eb7f2a5055f2d53ad448f2694e9046a0) nautilus (stable)

[opc@minio-0 ~]$ nomad -version
Nomad v1.0.4 (9294f35f9aa8dbb4acb6e85fa88e3e2534a3e41a)

First, I have run jobs of ceph csi controller and nodes with this.

## creat a file, ceph-csi-plugin-controller.nomad.
cat <<EOC > ceph-csi-plugin-controller.nomad
job "ceph-csi-plugin-controller" {
  datacenters = ["dc1"]

  group "controller" {
    network {    
      port "metrics" {}
    }
    task "ceph-controller" {

      template {
        data        = <<EOF
[{
    "clusterID": "c628ebf1-d03f-4806-9941-8b5840338b14",
    "monitors": [
        "v1:10.0.0.3:6789",
		"v1:10.0.0.4:6789",
		"v1:10.0.0.5:6789",
		"v2:10.0.0.3:3300",
		"v2:10.0.0.4:3300",
		"v2:10.0.0.5:3300"
    ]
}]
EOF
        destination = "local/config.json"
        change_mode = "restart"
      }
      driver = "docker"
      config {
        image = "quay.io/cephcsi/cephcsi:v3.3.1"
        volumes = [
          "./local/config.json:/etc/ceph-csi-config/config.json"
        ]    
        args = [
          "--type=rbd",
          "--controllerserver=true",
          "--drivername=rbd.csi.ceph.com",         
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=\${node.unique.name}",
		  "--instanceid=\${node.unique.name}-controller",	
          "--pidlimit=-1",
          "--v=5",		  
          "--metricsport=\$\${NOMAD_PORT_metrics}"
        ]		
      }	  
	  resources {
        cpu    = 500
        memory = 256
      }
      service {
        name = "ceph-csi-controller"
        port = "metrics"
        tags = [ "prometheus" ]
      }

      csi_plugin {
        id        = "ceph-csi"
        type      = "controller"
        mount_dir = "/csi"
      }
    }
  }
}
EOC




## create a ceph-csi-plugin-nodes.nomad.
cat <<EOC > ceph-csi-plugin-nodes.nomad
job "ceph-csi-plugin-nodes" {
  datacenters = ["dc1"]
  type        = "system"
  group "nodes" {
    network {    
      port "metrics" {}
    }
	
    task "ceph-node" {
      driver = "docker"
      template {
        data        = <<EOF
[{
    "clusterID": "c628ebf1-d03f-4806-9941-8b5840338b14",
    "monitors": [
        "v1:10.0.0.3:6789",
		"v1:10.0.0.4:6789",
		"v1:10.0.0.5:6789",
		"v2:10.0.0.3:3300",
		"v2:10.0.0.4:3300",
		"v2:10.0.0.5:3300"
    ]
}]
EOF
        destination = "local/config.json"
        change_mode = "restart"
      }
      config {
        image = "quay.io/cephcsi/cephcsi:v3.3.1"
        volumes = [
          "./local/config.json:/etc/ceph-csi-config/config.json"
        ]
        mounts = [
          {
            type     = "tmpfs"
            target   = "/tmp/csi/keys"
            readonly = false
            tmpfs_options = {
              size = 1000000 # size in bytes
            }
          }
        ]
        args = [
          "--type=rbd",
          "--drivername=rbd.csi.ceph.com",        
          "--nodeserver=true",
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=\${node.unique.name}",
          "--instanceid=\${node.unique.name}-nodes",	
          "--pidlimit=-1",
          "--v=5",		 		  
          "--metricsport=\$\${NOMAD_PORT_metrics}"
        ]		
        privileged = true
      }	  
	  resources {
        cpu    = 500
        memory = 256
      }
      service {
        name = "ceph-csi-nodes"
        port = "metrics"
        tags = [ "prometheus" ]
      }

      csi_plugin {
        id        = "ceph-csi"
        type      = "node"
        mount_dir = "/csi"
      }
    }
  }
}
EOC




## stop jobs.
nomad stop ceph-csi-plugin-controller;
nomad stop ceph-csi-plugin-nodes;



## run ceph csi plugin job.
nomad job run ceph-csi-plugin-controller.nomad;
nomad job run ceph-csi-plugin-nodes.nomad;

And then, I have created a volume.


## create volume.
cat <<EOF > ceph-volume.hcl
type = "csi"
id   = "ceph-mysql"
name = "ceph-mysql"
external_id     = "0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca"                          
access_mode     = "single-node-writer"
attachment_mode = "block-device"
mount_options {}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBemoFg1vvcBBAAX81SSAlWEtoOWLubhnIyVA=="  
}
context {  
  clusterID = "c628ebf1-d03f-4806-9941-8b5840338b14"
  pool      = "myPool" 
  imageFeatures = "layering"
}
EOF


nomad volume deregister ceph-mysql;
nomad volume register ceph-volume.hcl;

After registering a volume, the status of it looks like this.

[opc@minio-0 ~]$ nomad volume status ceph-mysql
ID                   = ceph-mysql
Name                 = ceph-mysql
External ID          = 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca
Plugin ID            = ceph-csi
Provider             = rbd.csi.ceph.com
Version              = v3.3.1
Schedulable          = true
Controllers Healthy  = 0
Controllers Expected = 1
Nodes Healthy        = 2
Nodes Expected       = 2
Access Mode          = single-node-writer
Attachment Mode      = block-device
Mount Options        = <none>
Namespace            = default

Allocations
No allocations placed

Before registering the volume, I have already created a pool with image in ceph.

# Create a ceph pool:
sudo ceph osd pool create myPool 64 64

# Create a block device pool:
sudo rbd pool init myPool

# create image.
sudo rbd create myimage --size 4096 --pool myPool --image-feature layering;

# associate pool to application.
sudo ceph osd pool application enable myPool rbd;

Let’s list pools in ceph.

[opc@minio-0 ~]$ sudo ceph osd lspools;
1 cephfs_data
2 cephfs_metadata
3 foo
4 bar
5 .rgw.root
6 default.rgw.control
7 default.rgw.meta
8 default.rgw.log
9 myPool

Finally, I have run mysql server job.

## deploy mysql.
cat <<EOF > mysql-server.nomad
job "mysql-server4" {
  datacenters = ["dc1"]
  type        = "service"

  group "mysql-server" {
    count = 1

    volume "ceph-mysql" {
      type      = "csi"
      read_only = false
      source    = "ceph-mysql"
    }

    network {
      port "db" {
        static = 3306
      }
    }

    restart {
      attempts = 10
      interval = "5m"
      delay    = "25s"
      mode     = "delay"
    }

    task "mysql-server" {
      driver = "docker"

      volume_mount {
        volume      = "ceph-mysql"
        destination = "/srv"
        read_only   = false
      }

      env {
        MYSQL_ROOT_PASSWORD = "password"
      }

      config {
        image = "hashicorp/mysql-portworx-demo:latest"
        args  = ["--datadir", "/srv/mysql"]
        ports = ["db"]
      }

      resources {
        cpu    = 500
        memory = 1024
      }

      service {
        name = "mysql-server"
        port = "db"

        check {
          type     = "tcp"
          interval = "10s"
          timeout  = "2s"
        }
      }
    }
  }
}
EOF

# run mysql server job.
nomad job run mysql-server.nomad;

But the job has failed with the errors.

[opc@minio-0 ~]$ nomad alloc status 559fd00d
ID                     = 559fd00d-92b8-84bf-7082-3c1fb6dcf3da
Eval ID                = d4072aa6
Name                   = mysql-server4.mysql-server[0]
Node ID                = dd258138
Node Name              = nomad-client-1
Job ID                 = mysql-server4
Job Version            = 0
Client Status          = failed
Client Description     = Failed tasks
Desired Status         = run
Desired Description    = <none>
Created                = 1m36s ago
Modified               = 1m34s ago
Deployment ID          = 71762f2d
Deployment Health      = unhealthy
Reschedule Eligibility = 24s from now

Allocation Addresses
Label  Dynamic  Address
*db    yes      10.0.0.7:3306

Task "mysql-server" is "dead"
Task Resources
CPU      Memory   Disk     Addresses
500 MHz  1.0 GiB  300 MiB

CSI Volumes:
ID          Read Only
ceph-mysql  false

Task Events:
Started At     = N/A
Finished At    = 2021-04-23T14:30:21Z
Total Restarts = 0
Last Restart   = N/A

Recent Events:
Time                  Type           Description
2021-04-23T14:30:21Z  Setup Failure  failed to setup alloc: pre-run hook "csi_hook" failed: node plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = image not found: RBD image not found
2021-04-23T14:30:21Z  Received       Task received by client

I have no idea what image not found: RBD image not found means.

I have used Ceph CSI v3.3.1 for this example for now, but I have already tried using another versions of ceph csi without success.

On Kubernetes, I have succeeded with dynamic volume provisioning using Ceph CSI v3.3.1 with external Ceph 14.x.

Is there anybody who has experienced using ceph csi with ceph 14.x(nautilus) without problem on Nomad?

Finally, I got to succeed with volume mount using ceph csi plugin.

I want to mention what I did to solve my problem with the following steps.

First, I have noticed that I have to create an image in the pool of ceph before creating the volume, which I didn’t know.
I have created an image in the pool like this.

# create a image.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacaca --size 1024 --pool myPool --image-feature layering;

And then, I got the following errors with rbd module not loaded.

...
I0428 04:58:04.509357       1 utils.go:162] ID: 8333 GRPC call: /csi.v1.Identity/Probe
I0428 04:58:04.509424       1 utils.go:166] ID: 8333 GRPC request: {}
I0428 04:58:04.509452       1 utils.go:173] ID: 8333 GRPC response: {}
I0428 04:58:04.512691       1 utils.go:162] ID: 8334 GRPC call: /csi.v1.Identity/Probe
I0428 04:58:04.512754       1 utils.go:166] ID: 8334 GRPC request: {}
I0428 04:58:04.512782       1 utils.go:173] ID: 8334 GRPC response: {}
I0428 04:58:04.512864       1 utils.go:162] ID: 8335 GRPC call: /csi.v1.Node/NodeGetCapabilities
I0428 04:58:04.512911       1 utils.go:166] ID: 8335 GRPC request: {}
I0428 04:58:04.513003       1 utils.go:173] ID: 8335 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}}]}
I0428 04:58:10.866104       1 utils.go:162] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC call: /csi.v1.Node/NodeStageVolume
I0428 04:58:10.866303       1 utils.go:166] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC request: {"secrets":"***stripped***","staging_target_path":"/csi/staging/ceph-mysql/rw-block-device-single-node-writer","volume_capability":{"AccessType":{"Block":{}},"access_mode":{"mode":1}},"volume_context":{"clusterID":"c628ebf1-d03f-4806-9941-8b5840338b14","imageFeatures":"layering","pool":"myPool"},"volume_id":"0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca"}
I0428 04:58:10.867220       1 rbd_util.go:977] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca setting disableInUseChecks: false image features: [layering] mounter: rbd
I0428 04:58:10.867284       1 encryption.go:244] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca could not detect owner for myPool/
E0428 04:58:10.868357       1 omap.go:77] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca omap not found (pool="myPool", namespace="", name="csi.volume.00000000-1111-2222-bbbb-cacacacacaca"): rados: ret=-2, No such file or directory
W0428 04:58:10.868406       1 voljournal.go:649] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca unable to read omap keys: pool or key missing: key not found: rados: ret=-2, No such file or directory
E0428 04:58:10.868651       1 util.go:233] kernel 3.10.0-1160.15.2.el7.x86_64 does not support required features
I0428 04:58:10.917446       1 cephcmds.go:59] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca command succeeded: rbd [device list --format=json --device-type krbd]
I0428 04:58:10.996180       1 rbd_attach.go:242] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca rbd: map mon 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789
I0428 04:58:11.059343       1 cephcmds.go:53] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca an error (exit status 2) occurred while running rbd args: [--id admin -m 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789 --keyfile=***stripped*** map myPool/csi-vol-00000000-1111-2222-bbbb-cacacacacaca --device-type krbd --options noudev]
W0428 04:58:11.059385       1 rbd_attach.go:270] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca rbd: map error an error (exit status 2) occurred while running rbd args: [--id admin -m 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789 --keyfile=***stripped*** map myPool/csi-vol-00000000-1111-2222-bbbb-cacacacacaca --device-type krbd --options noudev], rbd output: modinfo: ERROR: Module alias rbd not found.
modprobe: FATAL: Module rbd not found in directory /lib/modules/3.10.0-1160.15.2.el7.x86_64
rbd: failed to load rbd kernel module (1)
rbd: sysfs write failed
rbd: map failed: (2) No such file or directory
E0428 04:58:11.059542       1 utils.go:171] ID: 8336 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC error: rpc error: code = Internal desc = rbd: map failed with error an error (exit status 2) occurred while running rbd args: [--id admin -m 10.0.0.3:6789,10.0.0.4:6789,10.0.0.5:6789 --keyfile=***stripped*** map myPool/csi-vol-00000000-1111-2222-bbbb-cacacacacaca --device-type krbd --options noudev], rbd error output: modinfo: ERROR: Module alias rbd not found.
modprobe: FATAL: Module rbd not found in directory /lib/modules/3.10.0-1160.15.2.el7.x86_64
rbd: failed to load rbd kernel module (1)
rbd: sysfs write failed
rbd: map failed: (2) No such file or directory
I0428 04:58:11.070397       1 utils.go:162] ID: 8337 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC call: /csi.v1.Node/NodeUnpublishVolume
I0428 04:58:11.070522       1 utils.go:166] ID: 8337 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC request: {"target_path":"/csi/per-alloc/b2dcf68e-944a-9e2a-49bd-f349fc4d8057/ceph-mysql/rw-block-device-single-node-writer","volume_id":"0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca"}
I0428 04:58:11.070705       1 nodeserver.go:580] ID: 8337 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca targetPath: /csi/per-alloc/b2dcf68e-944a-9e2a-49bd-f349fc4d8057/ceph-mysql/rw-block-device-single-node-writer has already been deleted
I0428 04:58:11.070732       1 utils.go:173] ID: 8337 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC response: {}
I0428 04:58:11.071548       1 utils.go:162] ID: 8338 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC call: /csi.v1.Node/NodeUnstageVolume
I0428 04:58:11.071625       1 utils.go:166] ID: 8338 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC request: {"staging_target_path":"/csi/staging/ceph-mysql/rw-block-device-single-node-writer","volume_id":"0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca"}
I0428 04:58:11.071695       1 nodeserver.go:667] ID: 8338 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca failed to find image metadata: missing stash: open /csi/staging/ceph-mysql/rw-block-device-single-node-writer/image-meta.json: no such file or directory
I0428 04:58:11.071728       1 utils.go:173] ID: 8338 Req-ID: 0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca GRPC response: {}
I0428 04:58:34.513487       1 utils.go:162] ID: 8339 GRPC call: /csi.v1.Identity/Probe
I0428 04:58:34.513539       1 utils.go:166] ID: 8339 GRPC request: {}
I0428 04:58:34.513560       1 utils.go:173] ID: 8339 GRPC response: {}
I0428 04:58:34.513845       1 utils.go:162] ID: 8340 GRPC call: /csi.v1.Node/NodeGetCapabilities
I0428 04:58:34.513896       1 utils.go:166] ID: 8340 GRPC request: {}
I0428 04:58:34.513974       1 utils.go:173] ID: 8340 GRPC response: {"capabilities":[{"Type":{"Rpc":{"type":1}}},{"Type":{"Rpc":{"type":2}}},{"Type":{"Rpc":{"type":3}}}]}
I0428 04:58:34.514119       1 utils.go:162] ID: 8341 GRPC call: /csi.v1.Identity/Probe
I0428 04:58:34.514144       1 utils.go:166] ID: 8341 GRPC request: {}
I0428 04:58:34.514157       1 utils.go:173] ID: 8341 GRPC response: {}
...

I have loaded rbd module on the nomad client nodes.

# load rbd module on the nomad client node.
[opc@nomad-client-1 ~]$ sudo modprobe rbd
[opc@nomad-client-1 ~]$ sudo lsmod |grep rbd
rbd                    83733  0
libceph               306750  1 rbd

But After resubmitting mysql job, I got another errors.


# errors.
2021-04-28T05:09:05Z  Driver Failure   Failed to start container 5fcc2ec625ffb151c2883eaace915fdcf709eada1f6d0d57787e8e72cceb72ae: API error (400): OCI runtime create failed: container_linux.go:367: starting container process caused: process_linux.go:495: container init caused: rootfs_linux.go:60: mounting "/export/nomad-data/client/csi/node/ceph-csi/per-alloc/c1b2c081-1c04-6081-9053-b2a264c68d42/ceph-mysql/rw-block-device-single-node-writer" to rootfs at "/var/lib/docker/overlay2/832138acf233436fe55bbfbd1723d6c916a8ecaf5c535cc441ff885526437f1e/merged/srv" caused: not a directory: unknown: Are you trying to mount a directory onto a file (or vice-versa)? Check if the specified host path exists and is the expected type
2021-04-28T05:07:22Z  Driver           Downloading image
2021-04-28T05:07:22Z  Task Setup       Building Task Directory
2021-04-28T05:07:22Z  Received         Task received by client

I have changed block-device of attachment_mode to file-system with mount_options in the volume like this.

## create volume.
cat <<EOF > ceph-volume.hcl
type = "csi"
id   = "ceph-mysql"
name = "ceph-mysql"
external_id     = "0001-0024-c628ebf1-d03f-4806-9941-8b5840338b14-0000000000000009-00000000-1111-2222-bbbb-cacacacacaca"                          
access_mode     = "single-node-writer"
attachment_mode = "file-system"
mount_options {
  fs_type = "ext4"
}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBemoFg1vvcBBAAX81SSAlWEtoOWLubhnIyVA=="  
}
context {  
  clusterID = "c628ebf1-d03f-4806-9941-8b5840338b14"
  pool      = "myPool" 
  imageFeatures = "layering"
}
EOF

Now, I finally got what I want, it works!