Datadog: Inconsistent datadog_logs_custom_pipeline API error 400 Bad Request

Hello, I’m having an issue when creating Datadog log pipelines using Terraform.

I posted on the github as well, but haven’t received a response.

I have 18 pipelines I’m trying to launch, but I’m getting inconsistent errors when attempting to do so.

Key Points:

  • The first time I apply I get the following error (it gets thrown for 2 random pipelines)

    • Error: failed to create logs pipeline using Datadog API: API error 400 Bad Request: {“error”:{“code”:“InvalidArgument”,“message”:“This pipeline does not exist or is outdated”}}
  • It will still create the majority of the pipelines (usually 2/18 remaining)

  • If I terraform apply again it usually throws the error again, but creates 1 out of the 2 remaining

  • If I run terraform apply one more time it creates the remaining resource without error

  • The pipelines that throw the error are different each time. So if I re-do the whole process of creating the pipelines the next time I run terraform apply (the first time) the 2 pipelines that throw the errors will be different

  • When I terraform destroy, it says it destroys all of the resources, but if I go to Datadog there are also 2-3 pipelines that I have to manually remove. These are also different each time.

Any ideas/advice would be great. I’ll post the config below.

Cantelope Pipeline

cantelope_pipeline_filter_query = “source:cantelope”
cantelope_pipeline_name = “cantelope”

cantelope_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{data}\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} — %%{notSpace:thread} %%{notSpace:logger} %%{data}\n\nrule3 %%{data} %%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status} %%{data}”
name = “Parsing Cantelope Logs”
is_enabled = true
}
]

cantelope_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

walnut Pipeline

walnut_pipeline_filter_query = “source:walnut”
walnut_pipeline_name = “walnut”

walnut_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{data}\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} — %%{notSpace:thread} %%{notSpace:logger} %%{data}\n\nrule3 %%{data} %%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status} %%{data}”
name = “Parsing Walnut Logs”
is_enabled = true
}
]

walnut_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Prune Pipeline

prune_pipeline_filter_query = “source:prune”
prune_pipeline_name = “prune”

prune_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{data}\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} — %%{notSpace:thread} %%{notSpace:logger} %%{data}\n\nrule3 %%{data} %%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status} %%{data}”
name = “Parsing Prune Logs”
is_enabled = true
}
]

prune_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

coconut Pipeline

coconut_pipeline_filter_query = “source:coconut”
coconut_pipeline_name = “coconut”

coconut_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n”
name = “Parsing coconut Logs”
is_enabled = true
}
]

coconut_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

coconut_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

jabanero Pipeline

jabanero_pipeline_filter_query = “source:jabanero”
jabanero_pipeline_name = “jabanero”

jabanero_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{regex(“OFF:|ERROR:|WARNING:|INFO:|DEBUG:|TRACE:|ALL:”):status} .*\n\nrule %%{data} %%{regex(“off|error|warning|warn|inf|debug|trace”):status}”
name = “Parsing jabanero Logs”
is_enabled = true
}
]

jabanero_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

jabanero_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Pineapple Pipeline

pineapple_pipeline_filter_query = “source:pineapple”
pineapple_pipeline_name = “pineapple”

pineapple_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule3 %%{data} %%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status} %%{data}\n\nrule4 %%{data} %%{regex(“ERROR|WARN|INFO|DEBUG|TRACE”):status} %%{data}\n\nrule5 %%{data} %%{regex(“error|warn|info|debug|trace”):status}”
name = “Parsing pineapple Logs”
is_enabled = true
}
]

pineapple_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

pineapple_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Pineapple Slave Pipeline

pineapple_slave_pipeline_filter_query = “source:pineapple-slave”
pineapple_slave_pipeline_name = “pineapple-slave”

pineapple_slave_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule3 %%{data} %%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status} %%{data}\n\nrule4 %%{data} %%{regex(“ERROR|WARN|INFO|DEBUG|TRACE”):status} %%{data}\n\nrule5 %%{data} %%{regex(“error|warn|info|debug|trace”):status}”
name = “Parsing pineapple-slave Logs”
is_enabled = true
}
]

pineapple_slave_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

pineapple_slave_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Strawberry Pipeline

strawberry_pipeline_filter_query = “source:strawberry”
strawberry_pipeline_name = “strawberry”

strawberry_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n”
name = “Parsing strawberry Logs”
is_enabled = true
}
]

strawberry_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

strawberry_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Grapefruit Pipeline

grapefruit_pipeline_filter_query = “source:grapefruit”
grapefruit_pipeline_name = “grapefruit”

grapefruit_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n”
name = “Parsing grapefruit Logs”
is_enabled = true
}
]

grapefruit_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

grapefruit_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

lychee Pipeline

lychee_pipeline_filter_query = “source:lychee”
lychee_pipeline_name = “lychee”

lychee_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = "rule %%{date(“yyyy-MM-dd HH:MM:SS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL “):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule3 %%{data} %%{regex(“ERR|ERROR|WARN|INFO|DEBUG|TRACE”):status} %%{data}\n\nrule4 %%{regex(“ERR|ERROR|WARN|INFO|DEBUG|TRACE”):status}:%%{data}\n\nrule5 %%{data} %%{regex(“Err|Error|Warn|Info|Debug|Trace”):status} %%{data}\n\nrule6 %%{regex(“Error”):status} %%{data}”
name = “Parsing lychee Logs”
is_enabled = true
}
]

lychee_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

lychee_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

mango Pipeline

mango_pipeline_filter_query = “source:mango”
mango_pipeline_name = “mango”

mango_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} \[%%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status}\].*\n\n”
name = “Parsing mango Logs”
is_enabled = true
}
]

mango_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

passionfruit Pipeline

passionfruit_pipeline_filter_query = “source:passionfruit”
passionfruit_pipeline_name = “passionfruit”

passionfruit_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“error|warn|info|debug|trace”):status} %%{data}\n\nrule2 %%{regex(“Error”):status}: %%{data}\n\nrule3 %%{date(“yyyy-MM-dd HH:MM:ss.SSS”):timestamp} \[%%{regex(“error”):status}\] %%{data}”
name = “Parsing passionfruit Logs”
is_enabled = true
}
]

passionfruit_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Grape Pipeline

grape_pipeline_filter_query = “source:grape”
grape_pipeline_name = “grape”

grape_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“ERROR|WARNING|INFO|DEBUG|TRACE”):status} \[%%{notSpace:thread}\] \[%%{notSpace:logger}\] \[dd.trace_id= %%{notSpace:dd.trace_id}. dd.span_id=%%{notSpace:dd.span_id}].*”
name = “Parsing grape Logs”
is_enabled = true
}
]

grape_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

grape_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

OrangePipeline

orange_pipeline_filter_query = “source:orange”
orange_pipeline_name = “orange”

orange_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“ERROR|WARNING|INFO|DEBUG|TRACE”):status} \[%%{notSpace:thread}\] \[%%{notSpace:logger}\] \[dd.trace_id= %%{notSpace:dd.trace_id}. dd.span_id=%%{notSpace:dd.span_id}].*”
name = “Parsing orange Logs”
is_enabled = true
}
]

orange_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

orange_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

Banana Pipeline

banana_pipeline_filter_query = “source:banana”
banana_pipeline_name = “banana”

banana_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“ERROR|WARNING|INFO|DEBUG|TRACE”):status} \[%%{notSpace:thread}\] \[%%{notSpace:logger}\] \[dd.trace_id= %%{notSpace:dd.trace_id}. dd.span_id=%%{notSpace:dd.span_id}].*”
name = “Parsing banana Logs”
is_enabled = true
}
]

banana_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]

banana_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

cherry Pipeline

cherry_pipeline_filter_query = “source:cherry”
cherry_pipeline_name = “cherry”

cherry_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“ERR|WARN|INFO|DEBUG|TRACE”):status}.*\n”
name = “Parsing cherry Logs”
is_enabled = true
}
]

cherry_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

kiwi Pipeline

kiwi_pipeline_filter_query = “source:kiwi”
kiwi_pipeline_name = “kiwi”

kiwi_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“ERROR|WARN|INFO|DEBUG|TRACE”):status}.*\n”
name = “Parsing kiwi Logs”
is_enabled = true
}
]

kiwi_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]

starberry Pipeline

starberry_pipeline_filter_query = “source:starberry”
starberry_pipeline_name = “starberry”

starberry_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex(“ERROR|WARN|INFO|DEBUG|TRACE”):status}.*\n”
name = “Parsing starberry Logs”
is_enabled = true
}
]

starberry_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]