How to exclude allowlist from WAF rule?

I implemented the following custom WAF rule in Terraform and wondered if I missed something in the statement section!

resource "aws_wafv2_ip_set" "allow" {
  addresses          = var.allowlist
  description        = "Allowlist IP set "
  ip_address_version = "IPV4"
  name               = "ip-set"
  scope              = var.scope
}

rule {
    name     = "rule-test"
    priority = 10

    action {
      block {}
    }

    statement {
      rate_based_statement {
        limit              = 2000 
        aggregate_key_type = "IP"

        scope_down_statement {
          not_statement {
            statement {
              ip_set_reference_statement {
                arn = aws_wafv2_ip_set.allow.arn
              }
            }
          }
        }
      }
    }

    visibility_config {
      cloudwatch_metrics_enabled = true
      metric_name                = "rule-rate-limit-per-ip"
      sampled_requests_enabled   = true
    }
  }

What exactly is the problem?