Terraform多云Kafka实例部署方案详解

一、Terraform 基础知识

1.1 什么是 Terraform?

Terraform 是 HashiCorp 公司开发的开源基础设施即代码(Infrastructure as Code)工具,允许用户通过声明式配置文件定义和管理云基础设施。它支持多种云服务提供商,包括阿里云、腾讯云、AWS、Azure 等。

1.2 核心概念

1.3 工作流程

  1. 编写配置:创建 .tf 文件定义基础设施
  2. 初始化:运行 terraform init 下载依赖和初始化 backend
  3. 计划:运行 terraform plan 预览变更
  4. 应用:运行 terraform apply 应用变更
  5. 销毁:运行 terraform destroy 销毁资源

二、项目结构设计

2.1 整体结构

├── aliyun/                          # 阿里云实例文件
│   └── aliyun_alikafka_demo/
│       └── aliyun_china_pt_7111/    # 阿里云账号
│           └── alikafka_demo_202502061910/  # 实例
│               ├── backend.tf
│               └── main.tf
├── tcloud/                          # 腾讯云实例文件
│   └── tcloud_ckafka_demo/
│       └── tcloud_china_game_x6/    # 腾讯云账号
│           └── ckafka_demo_202501221738/    # 实例
│               ├── backend.tf
│               └── main.tf
└── modules/                         # 模块文件
    ├── aliyun/
    │   └── aliyun_alikafka_mod_demo/
    │       ├── main.tf
    │       └── variables.tf
    └── tcloud/
        └── tcloud_ckafka_mod_demo/
            ├── main.tf
            ├── outputs.tf
            └── variables.tf

2.2 结构说明

三、模块定义详解

3.1 阿里云 Kafka 模块

主配置文件(main.tf):

terraform {
  required_providers {
    alicloud = {
      source  = "aliyun/alicloud"
      version = "1.225.0"
    }
  }
}

provider "alicloud" {
  region = var.region
}

resource "alicloud_alikafka_instance" "this" {
  name          = var.instance_name
  deploy_type   = var.deploy_type
  disk_size     = var.disk_size
  disk_type     = var.disk_type
  vswitch_id    = var.vswitch_id
  partition_num = var.partition_num
  io_max        = var.io_max
}

变量定义(variables.tf):

variable "region" {
  description = "地域"
  type        = string
}

variable "instance_name" {
  description = "kafka实例名"
  type        = string
}

variable "deploy_type" {
  // 4: eip/vpc instance; 5: vpc instance
  description = "部署类型"
  type        = number
  default     = 5
  validation {
    condition     = contains([4, 5], var.deploy_type)
    error_message = "The deploy_type must be one of 4, 5"
  }
}

variable "disk_size" {
  description = "kafka实例磁盘规格"
  type        = number
  validation {
    condition     = contains([500, 1000], var.disk_size)
    error_message = "The disk_size must be one of 500, 1000"
  }
}

variable "disk_type" {
  # 0: efficient cloud disk, 1: SSD
  description = "磁盘类型"
  type        = number
  default     = 1
  validation {
    condition     = contains([0, 1], var.disk_type)
    error_message = "The disk_type must be one of 0, 1"
  }
}

variable "vswitch_id" {
  description = "绑定虚拟交换机id"
  type        = string
}

variable "partition_num" {
  description = "分区数量"
  type        = number
}

variable "io_max" {
  description = "io的最大值"
  type        = number
}

3.2 腾讯云 Kafka 模块

主配置文件(main.tf):

terraform {
  required_providers {
    tencentcloud = {
      source = "tencentcloudstack/tencentcloud"
    }
  }
}

provider "tencentcloud" {
  region = var.region
}

data "tencentcloud_availability_zones_by_product" "zone" {
  name    = var.availability_zone
  product = "ckafka"
}

resource "tencentcloud_ckafka_instance" "this" {
  instance_name      = var.instance_name
  zone_id            = data.tencentcloud_availability_zones_by_product.zone.zones[0].id
  vpc_id             = var.vpc_id
  subnet_id          = var.vswitch_id
  msg_retention_time = var.msg_retention_time
  kafka_version      = var.kafka_version
  disk_size          = var.disk_size
  band_width         = var.band_width
  disk_type          = var.disk_type
  partition          = var.partition
  charge_type        = var.charge_type

  config {
    auto_create_topic_enable   = var.auto_create_topic_enable
    default_num_partitions     = var.num_partitions
    default_replication_factor = var.replication_factor
  }

  dynamic_retention_config {
    enable = var.dynamic_retention_config_enable
  }
}

变量定义(variables.tf):

variable "region" {
  description = "地域"
  type        = string
  default     = "ap-shanghai"
}

variable "instance_name" {
  description = "kafka实例名"
  type        = string
}

variable "availability_zone" {
  description = "可用区"
  type        = string
}

variable "charge_type" {
  # PREPAID(预付费), POSTPAID_BY_HOUR(按量付费)
  description = "kafka实例计费方式"
  type        = string
  validation {
    condition     = contains(["PREPAID", "POSTPAID_BY_HOUR"], var.charge_type)
    error_message = "The charge_type must be one of PREPAID, POSTPAID_BY_HOUR"
  }
}

variable "kafka_version" {
  description = "kafka实例版本"
  type        = string
  validation {
    condition     = contains(["0.10.2", "1.1.1", "2.4.1", "2.4.2", "2.8.1"], var.kafka_version)
    error_message = "The kafka_version must be one of 0.10.2, 1.1.1, 2.4.1, 2.4.2, 2.8.1"
  }
}

variable "vswitch_id" {
  description = "绑定子网id"
  type        = string
}

variable "vpc_id" {
  description = "绑定vpc_id"
  type        = string
}

variable "disk_size" {
  description = "kafka实例磁盘规格"
  type        = number
  validation {
    condition     = contains([200, 400], var.disk_size)
    error_message = "The disk_size must be one of 200, 400"
  }
}

variable "disk_type" {
  # CLOUD_SSD(SSD云硬盘), CLOUD_BASIC(高性能云硬盘)
  description = "kafka专业版实例磁盘类型"
  type        = string
  default     = ""
  validation {
    condition     = contains(["", "CLOUD_SSD", "CLOUD_BASIC"], var.disk_type)
    error_message = "The disk_type must be empty, or one of CLOUD_SSD, CLOUD_BASIC"
  }
}

variable "band_width" {
  # 单位为MBps
  description = "kafka实例带宽"
  type        = number
}

variable "auto_create_topic_enable" {
  description = "是否自动创建topic"
  type        = bool
  default     = true
}

variable "num_partitions" {
  description = "kafka实例默认分区数"
  type        = number
  default     = 3
}

variable "replication_factor" {
  description = "kafka实例默认副本数"
  type        = number
  default     = 2
}

variable "dynamic_retention_config_enable" {
  description = "是否启用动态消息保留时间配置"
  type        = number
  default     = 0
  validation {
    condition     = contains([0, 1], var.dynamic_retention_config_enable)
    error_message = "The dynamic_retention_config_enable must be one of 0, 1"
  }
}

variable "msg_retention_time" {
  # 以分钟为单位
  description = "kafka实例日志的最大保留时间"
  type        = number
  default     = 10080
}

variable "partition" {
  description = "kafka实例分区大小"
  type        = number
  default     = 3
}

输出定义(outputs.tf):

output "ckafka_instance_id" {
  value = tencentcloud_ckafka_instance.this.id
}

四、实例使用方法

4.1 阿里云 Kafka 实例

后端配置(backend.tf):

terraform {
  backend "oss" {
    endpoint = "oss-cn-hangzhou.aliyuncs.com"
    bucket   = "your-bucket-name"
    prefix   = "terraform_state/aliyun/aliyun_alikafka_demo/aliyun_china_pt_7111/alikafka_demo_202502061910"
    key      = "terraform.tfstate"
    region   = "cn-hangzhou"
  }
}

主配置(main.tf):

module "alikafka_instance" {
  source        = "../../../../modules/aliyun/aliyun_alikafka_mod_demo"
  region        = "cn-hangzhou"
  disk_size     = 500
  instance_name = "alikafka-testV2"
  deploy_type   = 5
  vswitch_id    = "vsw-xxxxxxxxxxxx"
  partition_num = 50
  io_max        = 20
}

4.2 腾讯云 Kafka 实例

后端配置(backend.tf):

terraform {
  backend "oss" {
    endpoint = "oss-cn-hangzhou.aliyuncs.com"
    bucket   = "your-bucket-name"
    prefix   = "terraform_state/tcloud/tcloud_ckafka_demo/tcloud_china_game_x6/ckafka_demo_202501221738"
    key      = "terraform.tfstate"
    region   = "cn-hangzhou"
  }
}

主配置(main.tf):

module "ckafka_instance" {
  source                         = "../../../../modules/tcloud/tcloud_ckafka_mod_demo"
  region                         = "ap-shanghai"
  instance_name                  = "ckafka-test"
  availability_zone              = "ap-shanghai-2"
  charge_type                    = "POSTPAID_BY_HOUR"
  kafka_version                  = "2.4.2"
  vpc_id                         = "vpc-xxxxxxxxxxxx"
  vswitch_id                     = "subnet-xxxxxxxxxxxx"
  disk_size                      = 200
  band_width                     = 20
  auto_create_topic_enable       = true
  num_partitions                 = 3
  replication_factor             = 3
  dynamic_retention_config_enable = 1
  msg_retention_time             = 1300
  partition                      = 400
}

五、运行测试步骤

5.1 准备工作

获取云服务提供商的 API 密钥:

设置环境变量:

# 阿里云
export ALICLOUD_ACCESS_KEY=your_access_key
export ALICLOUD_SECRET_KEY=your_secret_key

# 腾讯云
export TENCENTCLOUD_SECRET_ID=your_secret_id
export TENCENTCLOUD_SECRET_KEY=your_secret_key

5.2 运行 Terraform 命令

# 进入实例目录
cd ./aliyun/aliyun_alikafka_demo/aliyun_china_pt_7111/alikafka_demo_202502061910

# 初始化项目
terraform init -backend-config="access_key=xxx" -backend-config="secret_key=yyy"

# 预览变更
terraform plan

# 应用变更
terraform apply

# 销毁资源
terraform destroy

5.3 验证结果

六、总结

方案优势

  1. 标准化:使用 Terraform 定义基础设施,实现配置标准化
  2. 模块化:封装可重用的模块,提高代码复用性和可维护性
  3. 多云支持:支持阿里云和腾讯云,实现多云管理统一
  4. 自动化:自动化基础设施部署和管理,减少人为错误
  5. 可追溯:使用 Git 管理配置,实现基础设施变更的可追溯性

适用场景