Mounting cephfs using csi plugin

Hello,

I was following your demo to mount ceph filesystem using csi plugin.

I am able to run node and controller ceph csi plugins and register volume. However when I try to run task with mounted volume I got:

E1023 17:40:33.414525       1 utils.go:163] ID: 101 Req-ID: 53245bc5-e638-4b24-b692-505feaef8f2f GRPC error: rpc error: code = Internal desc = missing required field monitors
E1023 17:40:33.564481       1 utils.go:163] ID: 102 Req-ID: 53245bc5-e638-4b24-b692-505feaef8f2f GRPC error: rpc error: code = Internal desc = an error occurred while running (20) umount [/csi/per-alloc/99a505df-7c92-4119-b2ec-23c3659ff1bb/cephfs/rw-file-system-single-node-writer]: exit status 32: umount: /csi/per-alloc/99a505df-7c92-4119-b2ec-23c3659ff1bb/cephfs/rw-file-system-single-node-writer: mountpoint not found

I have checked and plugin tasks have properly mounted /etc/ceph-csi-config/config.json with monitors field defined. So I am stuck trying to figure out what is wrong with my config. Below I post my plugin tasks, my volume definition and example task I use to test. I use nomad 0.12.4 with docker 19.03.13-ce. In ceph-volume.hcl userID and userKey are empty because so far I have not set up keyring on my ceph cluster and secrets can not be empty (I guess I should get authorization error but nevertheless it should not be connected with currently given error).

plugin-ceph-nodes.nomad:

job "plugin-ceph-nodes" {
  datacenters = ["dc1"]

  type = "system"

  group "nodes" {
    task "ceph-node" {
      driver = "docker"

      template {
        data        = <<EOF
[{
  "clusterID": "53245bc5-e638-4b24-b692-505feaef8f2f",
  "monitors": [
      {{range $index, $service := service "ceph-mon"}}{{if gt $index 0}}, {{end}}"{{.Address}}"{{end}}
  ]
}]
EOF
        destination = "local/config.json"
        change_mode = "restart"
      }

      config {
        image = "https://quay.io/cephcsi/cephcsi:v2.1.2"

        args = [
          "--endpoint=unix://csi/csi.sock",
          "--type=cephfs",
          "--drivername=cephfs.csi.ceph.com",
          "--nodeid=${node.unique.id}",
          "--logtostderr",
          "--nodeserver=true"
        ]

        privileged = true

        volumes = [
          "./local/config.json:/etc/ceph-csi-config/config.json"
        ]
      }

      resources {
        cpu    = 500
        memory = 256
        network {
          mbits = 1
        }
      }

      csi_plugin {
        id        = "ceph-csi"
        type      = "node"
        mount_dir = "/csi"
      }
    }

    task "ceph-controller" {
      driver = "docker"

      template {
        data        = <<EOF
[{
  "clusterID": "53245bc5-e638-4b24-b692-505feaef8f2f",
  "monitors": [
      {{range $index, $service := service "ceph-mon"}}{{if gt $index 0}}, {{end}}"{{.Address}}"{{end}}
  ]
}]
EOF
        destination = "local/config.json"
        change_mode = "restart"
      }

      config {
        image = "quay.io/cephcsi/cephcsi:v2.1.2"

        args = [
          "--type=cephfs",
          "--controllerserver=true",
          "--drivername=cephfs.csi.ceph.com",
          "--logtostderr",
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=${node.unique.id}"
        ]

        volumes = [
          "./local/config.json:/etc/ceph-csi-config/config.json"
        ]
      }

      resources {
        cpu    = 200
        memory = 500
        network {
          mbits = 1
        }
      }

      // service {
      //   name = "prometheus"
      //   port = "prometheus"
      //   tags = ["ceph-csi"]
      // }
      csi_plugin {
        id        = "ceph-csi"
        type      = "controller"
        mount_dir = "/csi"
      }
    }
  }
}

ceph-volume.hcl

id = "cephfs"
name = "CephFS"
external_id = "53245bc5-e638-4b24-b692-505feaef8f2f"
access_mode = "single-node-writer"
attachment_mode = "file-system"
plugin_id = "ceph-csi"

// mount_options {
//    fs_type = "ext4"
//    mount_flags = ["ro"]
// }

parameters {
}

secrets {
  userID  = ""
  userKey = ""
}

context {
  pool = "cephfs_data"
  clusterID = "53245bc5-e638-4b24-b692-505feaef8f2f",
}

test.nomad

job "mysql-server" {
  datacenters = ["dc1"]
  type        = "service"

  group "mysql-server" {
    count = 1

    volume "mysql" {
      type      = "csi"
      read_only = false
      source    = "cephfs"
    }

    restart {
      attempts = 10
      interval = "5m"
      delay    = "25s"
      mode     = "delay"
    }

    task "mysql-server" {
      env {
        CHANGE = 2
      }

      driver = "docker"

      volume_mount {
        volume      = "mysql"
        destination = "/srv"
        read_only   = false
      }

      env = {
        "MYSQL_ROOT_PASSWORD" = "password"
        CHANGE = 2
      }

      config {
        image = "hashicorp/mysql-portworx-demo:latest"
        args = ["--datadir", "/srv/mysql"]

        port_map {
          db = 3306
        }
      }

      resources {
        cpu    = 500
        memory = 1024

        network {
          port "db" {
            static = 3306
          }
        }
      }

      service {
        name = "mysql-server"
        port = "db"

        check {
          type     = "tcp"
          interval = "10s"
          timeout  = "2s"
        }
      }
    }
  }
}