Workspace Deployments affecting other deployments

Waypoint
1

Use Case:
Currently, we support 2 “tenants” in our system that deploy the exact same apps, only difference being they get dedicated resources separate database, ect.

We would like to deploy these with the same Waypoint.hcl, and nomad.jobspec. The solution that seemed to make the most sense here is to use waypoint workspaces, and nomad datacenters to manage this.

Our Setup:

  • One Waypoint Server
  • One Nomad Cluster, 2 Datacenters
  • One Consul Cluster
  • One Consul Cluster
  • Application has a waypoint.hcl that deploys with use “nomad-jobspec”
use "nomad-jobspec" {
      jobspec = templatefile("${path.app}/.jobspec.nomad", {
        kv_path      = var.ENV,
        image        = "${artifact.image}:${artifact.tag}",
        datacenter   = var.nomad_datacenter,
        job_name     = "${var.nomad_jobname}",
      })
    }

We want are trying to deploy 2 tenants. So we would run these following 2 commands

waypoint up -workspace=tenant1 -var nomad_datacenter=datacenter-1 -var job_name=tenant1-app-name -var env=tenant1-stage

waypoint up -workspace=tenant2 -var nomad_datacenter=datacenter-2 -var job_name=tenant2-app-name -var env=tenant2-stage

For the most part, this works fine, but the issue I am seeing is that when deploying one, the other in nomad is also restarted even though they are registered with different job names, different service names in consul ect, and when I look at the container I can see the env is set to whatever the most recent deployed app was.

Looking in nomad, I do not see a deployment logged for the other tenant, but new tasks are created regardless.

Any ideas what is going on here?

2

Hey @rlandingham ! Would you mind sharing your jobspec template with the sensitive info redacted?

3

Here you go!

Jobspec:

job "${job_name}" {
  datacenters = ["${datacenter}"]

  group "app" {
    count = "${web_task_count}"

    network {
      mode = "bridge"
      port "http" {}
    }

    vault {
      policies  = ["kv_user"]
    }

    service {
      name = "${job_name}"
      port = "http"

      tags = [
        "traefik.enable=true",
        "traefik.http.routers.${job_name}.rule=Host(`app.${public_dns_domain}`)",
        "traefik.consulcatalog.connect=true",
      ]

      connect {
        sidecar_service {
          proxy {
            local_service_port = 3000
          }
        }
      }
    }

    task "app" {
      driver = "docker"

      resources {
        cpu    = "${web_cpu}"
        memory = "${web_memory}"
        ##Keep same as memory for now
        memory_max = "${web_memory_max}"
      }

      config {
        image = "${image}"
        entrypoint = [
          "/waypoint-entrypoint",
          "/app/entrypoint_web.sh"
        ]
        ports = ["http"]
      }

      env {
        %{ for k, v in entrypoint.env ~}
        ${k} = "${v}"
        %{ endfor ~}
      }
    }
  }
  group "celery" {
    count = "${celery_task_count}"

    network {
      mode = "bridge"
      port "http" {}
    }

    vault {
      policies  = ["kv_user"]
    }

    service {
      name = "${job_name}"
      port = "http"

      connect {
        sidecar_service {
          proxy {
            local_service_port = 3000
          }
        }
      }
    }

    task "worker" {
      driver = "docker"

      resources {
        cpu    = "${celery_cpu}"
        memory = "${celery_memory}"
        memory_max = "${celery_memory_max}"
      }

      config {
        image = "${image}"
        entrypoint = [
          "/waypoint-entrypoint",
          "/app/entrypoint_worker.sh"
        ]
      }

      env {
        %{ for k, v in entrypoint.env ~}
        ${k} = "${v}"
        %{ endfor ~}
      }
    }
  }
}

Waypoint HCL


project = "app-name"

runner {
  enabled = true
}

app "app-name {

  config {
    env = {
      ENV = var.ENV

      SQLALCHEMY_ECHO = dynamic("vault", {
              path = "kv/data/app-name/${var.ENV}",
              key  = "data/SQLALCHEMY_ECHO"
            })

    }
  }

  build {
    use "docker" {
      dockerfile = "docker/Dockerfile"
    }

    registry {
      use "aws-ecr" {
        tag        = var.image_tag
        repository = "app-name"
      }
    }
  }

  deploy {
    use "nomad-jobspec" {
      jobspec = templatefile("${path.app}/.jobspec.nomad", {
        kv_path      = var.ENV,
        image        = "${artifact.image}:${artifact.tag}",
        datacenter   = var.nomad_datacenter,
        job_name     = "${var.nomad_jobname}",
        public_dns_domain = "${var.public_dns_domain}",
        private_dns_domain = "${var.private_dns_domain}",
        web_cpu          = var.web_cpu,
        web_memory       = var.web_memory_soft_limit,
        web_memory_max   = var.web_memory_hard_limit,
        web_task_count   = var.web_task_count,
        celery_cpu          = var.celery_cpu,
        celery_memory       = var.celery_memory_soft_limit,
        celery_memory_max   = var.celery_memory_hard_limit,
        celery_task_count   = var.celery_task_count
      })
    }
  }
}

# Variables
variable "image_tag" {
  type    = string
  default = "latest"
}

variable "ENV" {
  type        = string
  default     = null
  description = "The ENV of the server. Pass it as -var or use a waypoint config file"
}

variable "nomad_datacenter" {
  type        = string
  description = "The Nomad datacenter we are deploying to. Set the ENV WP_VAR_nomad_datacenter."
}

variable "nomad_jobname" {
  type        = string
  description = "The name of the job in Nomad."
  default     = "app-name"
}

variable "public_dns_domain" {
  type        = string
  description = "The DNS domain hosts are on for public access"
}

variable "private_dns_domain" {
  type        = string
  description = "The DNS domain hosts are on for private access"
}

variable "web_cpu" {
  type    = number
  default = 1000
}

variable "web_memory_soft_limit" {
  type    = number
  default = 600
}

variable "web_memory_hard_limit" {
  type    = number
  default = 600
}

variable "web_task_count" {
  type    = number
  default = 1
}

variable "celery_cpu" {
  type    = number
  default = 500
}

variable "celery_memory_soft_limit" {
  type    = number
  default = 600
}

variable "celery_memory_hard_limit" {
  type    = number
  default = 600
}

variable "celery_task_count" {
  type    = number
  default = 1
}