Hello, I’m having an issue when creating Datadog log pipelines using Terraform.
I posted on the github as well, but haven’t received a response.
https://github.com/terraform-providers/terraform-provider-datadog/issues/388
I have 18 pipelines I’m trying to launch, but I’m getting inconsistent errors when attempting to do so.
Key Points:
-
The first time I apply I get the following error (it gets thrown for 2 random pipelines)
- Error: failed to create logs pipeline using Datadog API: API error 400 Bad Request: {“error”:{“code”:“InvalidArgument”,“message”:“This pipeline does not exist or is outdated”}}
-
It will still create the majority of the pipelines (usually 2/18 remaining)
-
If I terraform apply again it usually throws the error again, but creates 1 out of the 2 remaining
-
If I run terraform apply one more time it creates the remaining resource without error
-
The pipelines that throw the error are different each time. So if I re-do the whole process of creating the pipelines the next time I run terraform apply (the first time) the 2 pipelines that throw the errors will be different
-
When I terraform destroy, it says it destroys all of the resources, but if I go to Datadog there are also 2-3 pipelines that I have to manually remove. These are also different each time.
Any ideas/advice would be great. I’ll post the config below.
Cantelope Pipeline
cantelope_pipeline_filter_query = “source:cantelope”
cantelope_pipeline_name = “cantelope”cantelope_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{data}\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{data}\n\nrule3 %%{data} %%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status} %%{data}”
name = “Parsing Cantelope Logs”
is_enabled = true
}
]cantelope_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]walnut Pipeline
walnut_pipeline_filter_query = “source:walnut”
walnut_pipeline_name = “walnut”walnut_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{data}\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{data}\n\nrule3 %%{data} %%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status} %%{data}”
name = “Parsing Walnut Logs”
is_enabled = true
}
]walnut_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Prune Pipeline
prune_pipeline_filter_query = “source:prune”
prune_pipeline_name = “prune”prune_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{data}\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{data}\n\nrule3 %%{data} %%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status} %%{data}”
name = “Parsing Prune Logs”
is_enabled = true
}
]prune_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]coconut Pipeline
coconut_pipeline_filter_query = “source:coconut”
coconut_pipeline_name = “coconut”coconut_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n”
name = “Parsing coconut Logs”
is_enabled = true
}
]coconut_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]coconut_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]jabanero Pipeline
jabanero_pipeline_filter_query = “source:jabanero”
jabanero_pipeline_name = “jabanero”jabanero_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{regex("OFF:|ERROR:|WARNING:|INFO:|DEBUG:|TRACE:|ALL:"):status} .*\n\nrule %%{data} %%{regex("off|error|warning|warn|inf|debug|trace"):status}”
name = “Parsing jabanero Logs”
is_enabled = true
}
]jabanero_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]jabanero_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Pineapple Pipeline
pineapple_pipeline_filter_query = “source:pineapple”
pineapple_pipeline_name = “pineapple”pineapple_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule3 %%{data} %%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status} %%{data}\n\nrule4 %%{data} %%{regex("ERROR|WARN|INFO|DEBUG|TRACE"):status} %%{data}\n\nrule5 %%{data} %%{regex("error|warn|info|debug|trace"):status}”
name = “Parsing pineapple Logs”
is_enabled = true
}
]pineapple_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]pineapple_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Pineapple Slave Pipeline
pineapple_slave_pipeline_filter_query = “source:pineapple-slave”
pineapple_slave_pipeline_name = “pineapple-slave”pineapple_slave_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule3 %%{data} %%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status} %%{data}\n\nrule4 %%{data} %%{regex("ERROR|WARN|INFO|DEBUG|TRACE"):status} %%{data}\n\nrule5 %%{data} %%{regex("error|warn|info|debug|trace"):status}”
name = “Parsing pineapple-slave Logs”
is_enabled = true
}
]pineapple_slave_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]pineapple_slave_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Strawberry Pipeline
strawberry_pipeline_filter_query = “source:strawberry”
strawberry_pipeline_name = “strawberry”strawberry_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n”
name = “Parsing strawberry Logs”
is_enabled = true
}
]strawberry_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]strawberry_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Grapefruit Pipeline
grapefruit_pipeline_filter_query = “source:grapefruit”
grapefruit_pipeline_name = “grapefruit”grapefruit_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n”
name = “Parsing grapefruit Logs”
is_enabled = true
}
]grapefruit_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]grapefruit_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]lychee Pipeline
lychee_pipeline_filter_query = “source:lychee”
lychee_pipeline_name = “lychee”lychee_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{date("yyyy-MM-dd HH:MM:SS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} %%{notSpace:function_name} - %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule2 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):event_date} %%{regex("OFF |ERROR|WARN |INFO |DEBUG|TRACE|ALL "):status} — %%{notSpace:thread} %%{notSpace:logger} %%{notSpace:dd.trace_id} %%{notSpace:dd.span_id}.\n\nrule3 %%{data} %%{regex("ERR|ERROR|WARN|INFO|DEBUG|TRACE"):status} %%{data}\n\nrule4 %%{regex("ERR|ERROR|WARN|INFO|DEBUG|TRACE"):status}:%%{data}\n\nrule5 %%{data} %%{regex("Err|Error|Warn|Info|Debug|Trace"):status} %%{data}\n\nrule6 %%{regex("Error"):status} %%{data}”
name = “Parsing lychee Logs”
is_enabled = true
}
]lychee_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]lychee_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]mango Pipeline
mango_pipeline_filter_query = “source:mango”
mango_pipeline_name = “mango”mango_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} \[%%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status}\].*\n\n”
name = “Parsing mango Logs”
is_enabled = true
}
]mango_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]passionfruit Pipeline
passionfruit_pipeline_filter_query = “source:passionfruit”
passionfruit_pipeline_name = “passionfruit”passionfruit_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("error|warn|info|debug|trace"):status} %%{data}\n\nrule2 %%{regex("Error"):status}: %%{data}\n\nrule3 %%{date("yyyy-MM-dd HH:MM:ss.SSS"):timestamp} \[%%{regex("error"):status}\] %%{data}”
name = “Parsing passionfruit Logs”
is_enabled = true
}
]passionfruit_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Grape Pipeline
grape_pipeline_filter_query = “source:grape”
grape_pipeline_name = “grape”grape_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("ERROR|WARNING|INFO|DEBUG|TRACE"):status} \[%%{notSpace:thread}\] \[%%{notSpace:logger}\] \[dd.trace_id= %%{notSpace:dd.trace_id}. dd.span_id=%%{notSpace:dd.span_id}].*”
name = “Parsing grape Logs”
is_enabled = true
}
]grape_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]grape_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]OrangePipeline
orange_pipeline_filter_query = “source:orange”
orange_pipeline_name = “orange”orange_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("ERROR|WARNING|INFO|DEBUG|TRACE"):status} \[%%{notSpace:thread}\] \[%%{notSpace:logger}\] \[dd.trace_id= %%{notSpace:dd.trace_id}. dd.span_id=%%{notSpace:dd.span_id}].*”
name = “Parsing orange Logs”
is_enabled = true
}
]orange_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]orange_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]Banana Pipeline
banana_pipeline_filter_query = “source:banana”
banana_pipeline_name = “banana”banana_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("ERROR|WARNING|INFO|DEBUG|TRACE"):status} \[%%{notSpace:thread}\] \[%%{notSpace:logger}\] \[dd.trace_id= %%{notSpace:dd.trace_id}. dd.span_id=%%{notSpace:dd.span_id}].*”
name = “Parsing banana Logs”
is_enabled = true
}
]banana_trace_id_remapper = [
{
sources = [“dd.trace_id”]
name = “Remapping trace id from the dd.trace_id attribute”
is_enabled = true
}
]banana_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]cherry Pipeline
cherry_pipeline_filter_query = “source:cherry”
cherry_pipeline_name = “cherry”cherry_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("ERR|WARN|INFO|DEBUG|TRACE"):status}.*\n”
name = “Parsing cherry Logs”
is_enabled = true
}
]cherry_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]kiwi Pipeline
kiwi_pipeline_filter_query = “source:kiwi”
kiwi_pipeline_name = “kiwi”kiwi_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("ERROR|WARN|INFO|DEBUG|TRACE"):status}.*\n”
name = “Parsing kiwi Logs”
is_enabled = true
}
]kiwi_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]starberry Pipeline
starberry_pipeline_filter_query = “source:starberry”
starberry_pipeline_name = “starberry”starberry_grok_parser = [
{
source = “message”
support_rules = “”
match_rules = “rule %%{data} %%{regex("ERROR|WARN|INFO|DEBUG|TRACE"):status}.*\n”
name = “Parsing starberry Logs”
is_enabled = true
}
]starberry_status_remapper = [
{
sources = [“status”]
name = “Remapping status from the status attribute”
is_enabled = true
}
]