IAM role chaining with kinesis firehosedelivery stream is failing

Terraform Providers AWS
1

Team,

We are trying to create Kinesis firehose delivery stream with the source as Kinesis stream and destination as Opensearch cluster (Which created in a different account). We have two IAM roles (Opensearch role and firehose role) with the dedicated policy. Opensearch role is with openseach get and put permissions and the Firehose role has the permission for S3, cloudwatch, lambda, kinesis firehose and more.

Am trying to configure a workflow so that “Opensearch_role” will assume the role “firehose_role.” With this in mind, we modified the trust policy of the target role, “firehose_role” so that it trusted “open_search_role.” However which fails during the terraform apply with the below mentioned error. Code details are below.

Terraform version:

terraform version
Terraform v1.3.2
on darwin_amd64
+ provider registry.terraform.io/carlpett/sops v0.7.2
+ provider registry.terraform.io/cyrilgdn/postgresql v1.15.0
+ provider registry.terraform.io/hashicorp/aws v5.0.1
+ provider registry.terraform.io/hashicorp/external v2.3.2
+ provider registry.terraform.io/hashicorp/local v2.4.1
+ provider registry.terraform.io/hashicorp/null v3.2.2

Your version of Terraform is out of date! The latest version
is 1.7.0. You can update by downloading from https://www.terraform.io/downloads.ht

IAM roles:

Open search IAM role:

data "aws_iam_policy_document" "os_access_policy" {
  statement {
    sid = "openSearchPut"
    actions = [
      "es:ESHttpPost",
      "es:ESHttpPut",
      "es:ESHttpGet",
      "es:DescribeDomain",
      "es:DescribeDomains",
      "es:DescribeDomainConfig"
    ]
    effect = "Allow"
    resources = [
      local.os_domain_arn,
      "${local.os_domain_arn}/*"
    ]
  }

  statement {
    sid = "openSearchGet"
    actions = [
      "es:ESHttpGet"
    ]
    effect = "Allow"
    resources = [
      "${local.os_domain_arn}/_all/_settings",
      "${local.os_domain_arn}/_cluster/stats",
      "${local.os_domain_arn}/index-name*/_mapping/superstore",
      "${local.os_domain_arn}/_nodes",
      "${local.os_domain_arn}/_nodes/stats",
      "${local.os_domain_arn}/_nodes/*/stats",
      "${local.os_domain_arn}/_stats",
      "${local.os_domain_arn}/index-name*/_stats",
    ]
  }

  statement {
    actions = [
      "sts:AssumeRole"
    ]
    effect = "Allow"
    resources = [
      "aws_iam_role.firehose_iam_role.arn"
    ]
  }
}

resource "aws_iam_policy" "os_policy" {
  name   = "${local.aws_account}_opensearch_policy"
  path   = "/"
  policy = data.aws_iam_policy_document.os_access_policy.json
}

data "aws_iam_policy_document" "os_assume_role_policy" {
  statement {
    sid     = "OpenSearchAssumeRole"
    actions = ["sts:AssumeRole"]

    principals {
      type        = "Service"
      identifiers = ["firehose.amazonaws.com"]
    }
  }

}

resource "aws_iam_role" "os_iam_role" {
  name = "${local.aws_account}_opensearch_role"

  assume_role_policy = data.aws_iam_policy_document.os_assume_role_policy.json
  managed_policy_arns = [
    aws_iam_policy.os_policy.arn
  ]
  tags = local.tags
}

firehose role

data "aws_iam_policy_document" "firehose_delivery_policy" {
  statement {
    sid = "firehoseS3Access"
    actions = [
      "kms:Decrypt",
      "kms:GenerateDataKey"
    ]
    effect = "Allow"
    resources = [
      data.aws_kms_key.s3_key.arn
    ]
    condition {
      test     = "StringEquals"
      variable = "kms:ViaService"
      values = [
        format(
          "s3.%s.amazonaws.com",
          var.region
        )
      ]
    }
    condition {
      test     = "StringLike"
      variable = "kms:EncryptionContext:aws:s3:arn"
      values = [
        "${module.kinesis_fail_bucket.s3_bucket_arn}/*"
      ]
    }
  }

  statement {
    sid = "firehoseS3Put"
    actions = [
      "s3:AbortMultipartUpload",
      "s3:GetBucketLocation",
      "s3:GetObject",
      "s3:ListBucket",
      "s3:ListBucketMultipartUploads",
      "s3:PutObject"
    ]
    effect = "Allow"
    resources = [
      module.kinesis_fail_bucket.s3_bucket_arn,
      "${module.kinesis_fail_bucket.s3_bucket_arn}/*"
    ]
  }

  statement {
    sid = "firehoseLogEvents"
    actions = [
      "logs:PutLogEvents"
    ]
    effect = "Allow"
    resources = [
      "${aws_cloudwatch_log_group.kinesis_opensearch.arn}:logstream:*"
    ]
  }

  statement {
    sid = "kinesisDescribe"
    actions = [
      "kinesis:DescribeStream",
      "kinesis:GetShardIterator",
      "kinesis:GetRecords",
      "kinesis:ListShards"
    ]
    effect = "Allow"
    resources = [
      local.firehose_arn,
      aws_kinesis_stream.cf_stream.arn
    ]
  }

  statement {
    sid = "cloudWatchPut"
    actions = [
      "logs:PutLogEvents"
    ]
    effect = "Allow"
    resources = [
      aws_cloudwatch_log_stream.kinesis_opensearch.arn
    ]
  }

  statement {
    sid = "lambdaTransform"
    actions = [
      "lambda:InvokeFunction"
    ]
    effect    = "Allow"
    resources = [module.cf_logs_transform.lambda_function_arn]
  }
}

resource "aws_iam_policy" "firehose_delivery_policy" {
  name   = "${terraform.workspace}_firehose_policy"
  path   = "/"
  policy = data.aws_iam_policy_document.firehose_delivery_policy.json
}

resource "aws_iam_role" "firehose_iam_role" {
  name = "${terraform.workspace}_firehose_role"
  path = var.iam_path

  assume_role_policy = <<EOF
  {
    "Version":  "2012-10-17",
    "Statement": [
      {
        "Sid":     "firehoseAssumeRole",
        "Effect": "Allow",
        "Principal": {
          "Service": "firehose.amazonaws.com"
        },
        "Action": "sts:AssumeRole"
      },
      {
        "Sid": "OpensearchAssumeRole",
        "Effect": "Allow",
        "Principal": {
          "AWS": "aws_iam_role.os_iam_role.arn"
        },
        "Action": "sts:AssumeRole"
      }
    ]
  }
  EOF
  managed_policy_arns = [
    aws_iam_policy.firehose_delivery_policy.arn
  ]
}

Kinesis_Firehose_delivery_stream :

resource "aws_kinesis_firehose_delivery_stream" "kinesis_opensearch" {
  name        = local.firehose_name
  destination = "elasticsearch"

  kinesis_source_configuration {
    kinesis_stream_arn = aws_kinesis_stream.cf_stream.arn
    role_arn           = aws_iam_role.os_iam_role.arn
  }

  elasticsearch_configuration {
    buffering_interval = 60
    cluster_endpoint = <cluster_endpoint>
    role_arn           = aws_iam_role.os_iam_role.arn
    index_name       = "cloudfront"
   
    cloudwatch_logging_options {
      enabled         = true
      log_group_name  = aws_cloudwatch_log_group.kinesis_opensearch.name
      log_stream_name = aws_cloudwatch_log_stream.kinesis_opensearch.name
    }

    s3_configuration {
      role_arn           = aws_iam_role.os_iam_role.arn
      bucket_arn         = module.kinesis_fail_bucket.s3_bucket_arn
      buffering_size     = 10
      buffering_interval = 400
      kms_key_arn        = data.aws_kms_key.s3_key.arn
      compression_format = "GZIP"

      prefix = "${aws_cloudfront_distribution.ui_cloudfront.domain_name}/"

      cloudwatch_logging_options {
        enabled         = true
        log_group_name  = aws_cloudwatch_log_group.kinesis_opensearch.name
        log_stream_name = aws_cloudwatch_log_stream.kinesis_opensearch.name
      }
    }
    processing_configuration {
      enabled = "true"

      processors {
        type = "Lambda"
        parameters {
          parameter_name  = "LambdaArn"
          parameter_value = module.cf_logs_transform.lambda_function_arn
        }
        parameters {
          parameter_name  = "BufferSizeInMBs"
          parameter_value = "1"
        }

        parameters {
          parameter_name  = "BufferIntervalInSeconds"
          parameter_value = "60"
        }
      }
    }
  }

  tags = local.tags
}

While applying this code getting the following error:

│ Error: creating Kinesis Firehose Delivery Stream (qa_tbv3_opensearch_stream): InvalidArgumentException: Role arn:aws:iam::<>role/test_opensearch_role is not authorized to perform: kinesis:DescribeStream on resource arn:aws:kinesis:us-east-1:<>:stream/kinesis_cloudfront_qa.
│
│   with aws_kinesis_firehose_delivery_stream.kinesis_opensearch,
│   on cloudfront.tf line 227, in resource "aws_kinesis_firehose_delivery_stream" "kinesis_opensearch":
│  227: resource "aws_kinesis_firehose_delivery_stream" "kinesis_opensearch" {
│
╵