Use Case:
Currently, we support 2 “tenants” in our system that deploy the exact same apps, only difference being they get dedicated resources separate database, ect.
We would like to deploy these with the same Waypoint.hcl, and nomad.jobspec. The solution that seemed to make the most sense here is to use waypoint workspaces, and nomad datacenters to manage this.
Our Setup:
- One Waypoint Server
- One Nomad Cluster, 2 Datacenters
- One Consul Cluster
- One Consul Cluster
- Application has a waypoint.hcl that deploys with use “nomad-jobspec”
use "nomad-jobspec" {
jobspec = templatefile("${path.app}/.jobspec.nomad", {
kv_path = var.ENV,
image = "${artifact.image}:${artifact.tag}",
datacenter = var.nomad_datacenter,
job_name = "${var.nomad_jobname}",
})
}
We want are trying to deploy 2 tenants. So we would run these following 2 commands
waypoint up -workspace=tenant1 -var nomad_datacenter=datacenter-1 -var job_name=tenant1-app-name -var env=tenant1-stage
waypoint up -workspace=tenant2 -var nomad_datacenter=datacenter-2 -var job_name=tenant2-app-name -var env=tenant2-stage
For the most part, this works fine, but the issue I am seeing is that when deploying one, the other in nomad is also restarted even though they are registered with different job names, different service names in consul ect, and when I look at the container I can see the env is set to whatever the most recent deployed app was.
Looking in nomad, I do not see a deployment logged for the other tenant, but new tasks are created regardless.
Any ideas what is going on here?
Hey @rlandingham ! Would you mind sharing your jobspec template with the sensitive info redacted?
Here you go!
Jobspec:
job "${job_name}" {
datacenters = ["${datacenter}"]
group "app" {
count = "${web_task_count}"
network {
mode = "bridge"
port "http" {}
}
vault {
policies = ["kv_user"]
}
service {
name = "${job_name}"
port = "http"
tags = [
"traefik.enable=true",
"traefik.http.routers.${job_name}.rule=Host(`app.${public_dns_domain}`)",
"traefik.consulcatalog.connect=true",
]
connect {
sidecar_service {
proxy {
local_service_port = 3000
}
}
}
}
task "app" {
driver = "docker"
resources {
cpu = "${web_cpu}"
memory = "${web_memory}"
##Keep same as memory for now
memory_max = "${web_memory_max}"
}
config {
image = "${image}"
entrypoint = [
"/waypoint-entrypoint",
"/app/entrypoint_web.sh"
]
ports = ["http"]
}
env {
%{ for k, v in entrypoint.env ~}
${k} = "${v}"
%{ endfor ~}
}
}
}
group "celery" {
count = "${celery_task_count}"
network {
mode = "bridge"
port "http" {}
}
vault {
policies = ["kv_user"]
}
service {
name = "${job_name}"
port = "http"
connect {
sidecar_service {
proxy {
local_service_port = 3000
}
}
}
}
task "worker" {
driver = "docker"
resources {
cpu = "${celery_cpu}"
memory = "${celery_memory}"
memory_max = "${celery_memory_max}"
}
config {
image = "${image}"
entrypoint = [
"/waypoint-entrypoint",
"/app/entrypoint_worker.sh"
]
}
env {
%{ for k, v in entrypoint.env ~}
${k} = "${v}"
%{ endfor ~}
}
}
}
}
Waypoint HCL
project = "app-name"
runner {
enabled = true
}
app "app-name {
config {
env = {
ENV = var.ENV
SQLALCHEMY_ECHO = dynamic("vault", {
path = "kv/data/app-name/${var.ENV}",
key = "data/SQLALCHEMY_ECHO"
})
}
}
build {
use "docker" {
dockerfile = "docker/Dockerfile"
}
registry {
use "aws-ecr" {
tag = var.image_tag
repository = "app-name"
}
}
}
deploy {
use "nomad-jobspec" {
jobspec = templatefile("${path.app}/.jobspec.nomad", {
kv_path = var.ENV,
image = "${artifact.image}:${artifact.tag}",
datacenter = var.nomad_datacenter,
job_name = "${var.nomad_jobname}",
public_dns_domain = "${var.public_dns_domain}",
private_dns_domain = "${var.private_dns_domain}",
web_cpu = var.web_cpu,
web_memory = var.web_memory_soft_limit,
web_memory_max = var.web_memory_hard_limit,
web_task_count = var.web_task_count,
celery_cpu = var.celery_cpu,
celery_memory = var.celery_memory_soft_limit,
celery_memory_max = var.celery_memory_hard_limit,
celery_task_count = var.celery_task_count
})
}
}
}
# Variables
variable "image_tag" {
type = string
default = "latest"
}
variable "ENV" {
type = string
default = null
description = "The ENV of the server. Pass it as -var or use a waypoint config file"
}
variable "nomad_datacenter" {
type = string
description = "The Nomad datacenter we are deploying to. Set the ENV WP_VAR_nomad_datacenter."
}
variable "nomad_jobname" {
type = string
description = "The name of the job in Nomad."
default = "app-name"
}
variable "public_dns_domain" {
type = string
description = "The DNS domain hosts are on for public access"
}
variable "private_dns_domain" {
type = string
description = "The DNS domain hosts are on for private access"
}
variable "web_cpu" {
type = number
default = 1000
}
variable "web_memory_soft_limit" {
type = number
default = 600
}
variable "web_memory_hard_limit" {
type = number
default = 600
}
variable "web_task_count" {
type = number
default = 1
}
variable "celery_cpu" {
type = number
default = 500
}
variable "celery_memory_soft_limit" {
type = number
default = 600
}
variable "celery_memory_hard_limit" {
type = number
default = 600
}
variable "celery_task_count" {
type = number
default = 1
}