How to make the provider configuration optional and based on the condition

Here is my provider configuration:

provider "azurerm" {

  features {}

        subscription_id = "${var.azure.AzureSubscriptionId}"

        client_id       = "${var.azure.AzureClientId}"

        client_secret   = "${var.azure.AzureClientSecret}"

        tenant_id       = "${var.azure.AzureTenantId}"

        skip_provider_registration = true

}

provider "aws" {

        region     = "${var.aws.AwsRegion}"

        access_key = "${var.aws.AwsAccessKey}"

        secret_key = "${var.aws.AwsSecretKey}"

}

and this is how my root module is configured:

module "aws_bootstrap_vm" {

      count = var.CloudProvider == "aws" ? 1 : 0
      source = "./modules/aws/bootstrap_vm"
}

module "azure_bootstrap_vm" {

      count = var.CloudProvider == "azure" ? 1 : 0
      source = "./modules/azure/bootstrap_vm"
}

Currently I need to provide the credentials for both aws & azure to get this to work or else I get this error when I only want to use the AWS module:

 Error: building AzureRM Client: 1 error occurred:
│       * A Subscription ID must be configured when authenticating as a Service Principal using a Client Secret.
│