Terraform多云Kafka实例部署方案详解
一、Terraform 基础知识
1.1 什么是 Terraform?
Terraform 是 HashiCorp 公司开发的开源基础设施即代码(Infrastructure as Code)工具,允许用户通过声明式配置文件定义和管理云基础设施。它支持多种云服务提供商,包括阿里云、腾讯云、AWS、Azure 等。
1.2 核心概念
- Provider:云服务提供商的插件,负责与云 API 交互
- Resource:基础设施的组件,如 VM、网络、存储等
- Module:可重用的配置块,封装了一组相关资源
- State:存储基础设施当前状态的文件
- Backend:存储状态文件的位置,可以是本地或远程存储
1.3 工作流程
- 编写配置:创建
.tf文件定义基础设施 - 初始化:运行
terraform init下载依赖和初始化 backend - 计划:运行
terraform plan预览变更 - 应用:运行
terraform apply应用变更 - 销毁:运行
terraform destroy销毁资源
二、项目结构设计
2.1 整体结构
├── aliyun/ # 阿里云实例文件
│ └── aliyun_alikafka_demo/
│ └── aliyun_china_pt_7111/ # 阿里云账号
│ └── alikafka_demo_202502061910/ # 实例
│ ├── backend.tf
│ └── main.tf
├── tcloud/ # 腾讯云实例文件
│ └── tcloud_ckafka_demo/
│ └── tcloud_china_game_x6/ # 腾讯云账号
│ └── ckafka_demo_202501221738/ # 实例
│ ├── backend.tf
│ └── main.tf
└── modules/ # 模块文件
├── aliyun/
│ └── aliyun_alikafka_mod_demo/
│ ├── main.tf
│ └── variables.tf
└── tcloud/
└── tcloud_ckafka_mod_demo/
├── main.tf
├── outputs.tf
└── variables.tf
2.2 结构说明
- 云厂商目录:按云厂商分类,如 aliyun、tcloud
- 项目目录:按项目或用途分类
- 账号目录:按云账号分类
- 实例目录:按具体实例分类,包含时间戳便于区分版本
- 模块目录:按云厂商和资源类型分类,封装可重用的配置
三、模块定义详解
3.1 阿里云 Kafka 模块
主配置文件(main.tf):
terraform {
required_providers {
alicloud = {
source = "aliyun/alicloud"
version = "1.225.0"
}
}
}
provider "alicloud" {
region = var.region
}
resource "alicloud_alikafka_instance" "this" {
name = var.instance_name
deploy_type = var.deploy_type
disk_size = var.disk_size
disk_type = var.disk_type
vswitch_id = var.vswitch_id
partition_num = var.partition_num
io_max = var.io_max
}
变量定义(variables.tf):
variable "region" {
description = "地域"
type = string
}
variable "instance_name" {
description = "kafka实例名"
type = string
}
variable "deploy_type" {
// 4: eip/vpc instance; 5: vpc instance
description = "部署类型"
type = number
default = 5
validation {
condition = contains([4, 5], var.deploy_type)
error_message = "The deploy_type must be one of 4, 5"
}
}
variable "disk_size" {
description = "kafka实例磁盘规格"
type = number
validation {
condition = contains([500, 1000], var.disk_size)
error_message = "The disk_size must be one of 500, 1000"
}
}
variable "disk_type" {
# 0: efficient cloud disk, 1: SSD
description = "磁盘类型"
type = number
default = 1
validation {
condition = contains([0, 1], var.disk_type)
error_message = "The disk_type must be one of 0, 1"
}
}
variable "vswitch_id" {
description = "绑定虚拟交换机id"
type = string
}
variable "partition_num" {
description = "分区数量"
type = number
}
variable "io_max" {
description = "io的最大值"
type = number
}
3.2 腾讯云 Kafka 模块
主配置文件(main.tf):
terraform {
required_providers {
tencentcloud = {
source = "tencentcloudstack/tencentcloud"
}
}
}
provider "tencentcloud" {
region = var.region
}
data "tencentcloud_availability_zones_by_product" "zone" {
name = var.availability_zone
product = "ckafka"
}
resource "tencentcloud_ckafka_instance" "this" {
instance_name = var.instance_name
zone_id = data.tencentcloud_availability_zones_by_product.zone.zones[0].id
vpc_id = var.vpc_id
subnet_id = var.vswitch_id
msg_retention_time = var.msg_retention_time
kafka_version = var.kafka_version
disk_size = var.disk_size
band_width = var.band_width
disk_type = var.disk_type
partition = var.partition
charge_type = var.charge_type
config {
auto_create_topic_enable = var.auto_create_topic_enable
default_num_partitions = var.num_partitions
default_replication_factor = var.replication_factor
}
dynamic_retention_config {
enable = var.dynamic_retention_config_enable
}
}
变量定义(variables.tf):
variable "region" {
description = "地域"
type = string
default = "ap-shanghai"
}
variable "instance_name" {
description = "kafka实例名"
type = string
}
variable "availability_zone" {
description = "可用区"
type = string
}
variable "charge_type" {
# PREPAID(预付费), POSTPAID_BY_HOUR(按量付费)
description = "kafka实例计费方式"
type = string
validation {
condition = contains(["PREPAID", "POSTPAID_BY_HOUR"], var.charge_type)
error_message = "The charge_type must be one of PREPAID, POSTPAID_BY_HOUR"
}
}
variable "kafka_version" {
description = "kafka实例版本"
type = string
validation {
condition = contains(["0.10.2", "1.1.1", "2.4.1", "2.4.2", "2.8.1"], var.kafka_version)
error_message = "The kafka_version must be one of 0.10.2, 1.1.1, 2.4.1, 2.4.2, 2.8.1"
}
}
variable "vswitch_id" {
description = "绑定子网id"
type = string
}
variable "vpc_id" {
description = "绑定vpc_id"
type = string
}
variable "disk_size" {
description = "kafka实例磁盘规格"
type = number
validation {
condition = contains([200, 400], var.disk_size)
error_message = "The disk_size must be one of 200, 400"
}
}
variable "disk_type" {
# CLOUD_SSD(SSD云硬盘), CLOUD_BASIC(高性能云硬盘)
description = "kafka专业版实例磁盘类型"
type = string
default = ""
validation {
condition = contains(["", "CLOUD_SSD", "CLOUD_BASIC"], var.disk_type)
error_message = "The disk_type must be empty, or one of CLOUD_SSD, CLOUD_BASIC"
}
}
variable "band_width" {
# 单位为MBps
description = "kafka实例带宽"
type = number
}
variable "auto_create_topic_enable" {
description = "是否自动创建topic"
type = bool
default = true
}
variable "num_partitions" {
description = "kafka实例默认分区数"
type = number
default = 3
}
variable "replication_factor" {
description = "kafka实例默认副本数"
type = number
default = 2
}
variable "dynamic_retention_config_enable" {
description = "是否启用动态消息保留时间配置"
type = number
default = 0
validation {
condition = contains([0, 1], var.dynamic_retention_config_enable)
error_message = "The dynamic_retention_config_enable must be one of 0, 1"
}
}
variable "msg_retention_time" {
# 以分钟为单位
description = "kafka实例日志的最大保留时间"
type = number
default = 10080
}
variable "partition" {
description = "kafka实例分区大小"
type = number
default = 3
}
输出定义(outputs.tf):
output "ckafka_instance_id" {
value = tencentcloud_ckafka_instance.this.id
}
四、实例使用方法
4.1 阿里云 Kafka 实例
后端配置(backend.tf):
terraform {
backend "oss" {
endpoint = "oss-cn-hangzhou.aliyuncs.com"
bucket = "your-bucket-name"
prefix = "terraform_state/aliyun/aliyun_alikafka_demo/aliyun_china_pt_7111/alikafka_demo_202502061910"
key = "terraform.tfstate"
region = "cn-hangzhou"
}
}
主配置(main.tf):
module "alikafka_instance" {
source = "../../../../modules/aliyun/aliyun_alikafka_mod_demo"
region = "cn-hangzhou"
disk_size = 500
instance_name = "alikafka-testV2"
deploy_type = 5
vswitch_id = "vsw-xxxxxxxxxxxx"
partition_num = 50
io_max = 20
}
4.2 腾讯云 Kafka 实例
后端配置(backend.tf):
terraform {
backend "oss" {
endpoint = "oss-cn-hangzhou.aliyuncs.com"
bucket = "your-bucket-name"
prefix = "terraform_state/tcloud/tcloud_ckafka_demo/tcloud_china_game_x6/ckafka_demo_202501221738"
key = "terraform.tfstate"
region = "cn-hangzhou"
}
}
主配置(main.tf):
module "ckafka_instance" {
source = "../../../../modules/tcloud/tcloud_ckafka_mod_demo"
region = "ap-shanghai"
instance_name = "ckafka-test"
availability_zone = "ap-shanghai-2"
charge_type = "POSTPAID_BY_HOUR"
kafka_version = "2.4.2"
vpc_id = "vpc-xxxxxxxxxxxx"
vswitch_id = "subnet-xxxxxxxxxxxx"
disk_size = 200
band_width = 20
auto_create_topic_enable = true
num_partitions = 3
replication_factor = 3
dynamic_retention_config_enable = 1
msg_retention_time = 1300
partition = 400
}
五、运行测试步骤
5.1 准备工作
获取云服务提供商的 API 密钥:
- 阿里云:登录阿里云控制台,在「访问控制」中创建 AccessKey
- 腾讯云:登录腾讯云控制台,在「访问管理」中创建 SecretId 和 SecretKey
设置环境变量:
# 阿里云
export ALICLOUD_ACCESS_KEY=your_access_key
export ALICLOUD_SECRET_KEY=your_secret_key
# 腾讯云
export TENCENTCLOUD_SECRET_ID=your_secret_id
export TENCENTCLOUD_SECRET_KEY=your_secret_key
5.2 运行 Terraform 命令
# 进入实例目录
cd ./aliyun/aliyun_alikafka_demo/aliyun_china_pt_7111/alikafka_demo_202502061910
# 初始化项目
terraform init -backend-config="access_key=xxx" -backend-config="secret_key=yyy"
# 预览变更
terraform plan
# 应用变更
terraform apply
# 销毁资源
terraform destroy
5.3 验证结果
- 阿里云:在阿里云控制台「消息服务 Kafka」中查看实例
- 腾讯云:在腾讯云控制台「消息队列 CKafka」中查看实例
六、总结
方案优势
- 标准化:使用 Terraform 定义基础设施,实现配置标准化
- 模块化:封装可重用的模块,提高代码复用性和可维护性
- 多云支持:支持阿里云和腾讯云,实现多云管理统一
- 自动化:自动化基础设施部署和管理,减少人为错误
- 可追溯:使用 Git 管理配置,实现基础设施变更的可追溯性
适用场景
- 多环境部署:开发、测试、生产环境的标准化部署
- 多云管理:同时管理多个云服务提供商的资源
- 大规模部署:需要批量创建和管理大量资源的场景
- DevOps 实践:将基础设施纳入 CI/CD 流程,实现持续部署