Feat: Support distribute auth key to the service

cli/command/deploy.go

@@ -44,6 +44,7 @@ const (
 	SYNC_CONFIG                = playbook.SYNC_CONFIG
 	START_ETCD                 = playbook.START_ETCD
 	START_MDS                  = playbook.START_MDS
+	DIST_AUTH_KEY              = playbook.DIST_AUTH_KEY
 	CREATE_PHYSICAL_POOL       = playbook.CREATE_PHYSICAL_POOL
 	START_CHUNKSERVER          = playbook.START_CHUNKSERVER
 	CREATE_LOGICAL_POOL        = playbook.CREATE_LOGICAL_POOL
@@ -66,6 +67,7 @@ var (
 		SYNC_CONFIG,
 		START_ETCD,
 		START_MDS,
+		DIST_AUTH_KEY,
 		CREATE_PHYSICAL_POOL,
 		START_CHUNKSERVER,
 		CREATE_LOGICAL_POOL,
@@ -93,12 +95,14 @@ var (
 		CREATE_PHYSICAL_POOL: ROLE_MDS,
 		CREATE_LOGICAL_POOL:  ROLE_MDS,
 		BALANCE_LEADER:       ROLE_MDS,
+		DIST_AUTH_KEY:        ROLE_MDS,
 	}
 
 	DEPLOY_LIMIT_SERVICE = map[int]int{
 		CREATE_PHYSICAL_POOL: 1,
 		CREATE_LOGICAL_POOL:  1,
 		BALANCE_LEADER:       1,
+		DIST_AUTH_KEY:        1,
 	}
 
 	CAN_SKIP_ROLES = []string{
@@ -160,13 +164,15 @@ func skipServiceRole(deployConfigs []*topology.DeployConfig, options deployOptio
 	return dcs
 }
 
-func skipDeploySteps(deploySteps []int, options deployOptions) []int {
+func skipDeploySteps(dcs []*topology.DeployConfig, deploySteps []int, options deployOptions) []int {
 	steps := []int{}
 	skipped := utils.Slice2Map(options.skip)
 	for _, step := range deploySteps {
-		if step == START_SNAPSHOTCLONE && skipped[ROLE_SNAPSHOTCLONE] {
+		if (step == START_SNAPSHOTCLONE && skipped[ROLE_SNAPSHOTCLONE]) ||
+			(step == DIST_AUTH_KEY && !dcs[0].GetAuthEnable()) {
 			continue
 		}
+
 		steps = append(steps, step)
 	}
 	return steps
@@ -211,10 +217,38 @@ func genDeployPlaybook(curveadm *cli.CurveAdm,
 	if kind == topology.KIND_CURVEBS {
 		steps = CURVEBS_DEPLOY_STEPS
 	}
-	steps = skipDeploySteps(steps, options)
+	steps = skipDeploySteps(dcs, steps, options)
 	poolset := options.poolset
 	diskType := options.poolsetDiskType
 
+	// record all auth key info
+	var authServerKey string
+	stepDistAuthKeyOptions := make(map[string]comm.RoleAuthInfo)
+	if kind == topology.KIND_CURVEBS && dcs[0].GetAuthEnable() {
+		for _, dc := range dcs {
+			role := dc.GetRole()
+			if role == ROLE_ETCD {
+				continue
+			}
+			if _, ok := stepDistAuthKeyOptions[role]; ok {
+				continue
+			}
+
+			stepDistAuthKeyOptions[role] = comm.RoleAuthInfo{
+				AuthEnable:       dc.GetAuthEnable(),
+				AuthClientEnable: dc.GetAuthClientEnable(),
+				AuthServerKey:    dc.GetAuthServerKey(),
+				AuthKeyCurrent:   dc.GetAuthKeyCurrent(),
+				AuthClientKey:    dc.GetAuthClientKey(),
+				AuthClientId:     dc.GetAuthClientId(),
+			}
+
+			if role == topology.ROLE_MDS {
+				authServerKey = dc.GetAuthServerKey()
+			}
+		}
+	}
+
 	pb := playbook.NewPlaybook(curveadm)
 	for _, step := range steps {
 		// configs
@@ -237,6 +271,9 @@ func genDeployPlaybook(curveadm *cli.CurveAdm,
 			options[comm.POOLSET_DISK_TYPE] = diskType
 		} else if step == CREATE_LOGICAL_POOL {
 			options[comm.KEY_CREATE_POOL_TYPE] = comm.POOL_TYPE_LOGICAL
+		} else if step == DIST_AUTH_KEY {
+			options[comm.AUTH_SERVER_KEY] = authServerKey
+			options[comm.ROLES_AUTH_INFO] = stepDistAuthKeyOptions
 		}
 
 		pb.AddStep(&playbook.PlaybookStep{

client.yaml

@@ -0,0 +1,7 @@
+kind: curvebs
+container_image: harbor.cloud.netease.com/curve/curvebs:wanghai-auth
+mds.listen.addr: 59.111.93.76:6700,59.111.93.76:6701,59.111.93.76:6702
+log_dir: /root/logs/client
+auth.client.enable: true
+auth.client.key: 60a114c91c419831
+auth.client.id: mds_tools

internal/common/common.go

@@ -47,6 +47,8 @@ const (
 	POOL_TYPE_PHYSICAL     = "physicalpool"
 	POOLSET                = "poolset"
 	POOLSET_DISK_TYPE      = "poolset-disktype"
+	AUTH_SERVER_KEY        = "auth-server-key"
+	ROLES_AUTH_INFO        = "roles-auth-info"
 
 	// disk
 	DISK_DEFAULT_NULL_SIZE            = "-"
@@ -148,3 +150,12 @@ const (
 	POLICY_NEVER_RESTART  = "no"
 	POLICY_UNLESS_STOPPED = "unless-stopped"
 )
+
+type RoleAuthInfo struct {
+	AuthEnable       bool
+	AuthClientEnable bool
+	AuthServerKey    string // mds
+	AuthKeyCurrent   string // mds, chunkserver, snapshotclone
+	AuthClientKey    string // mds, chunkserver, snapshotclone
+	AuthClientId     string
+}

internal/configure/client.go

@@ -52,6 +52,10 @@ const (
 	KEY_CLIENT_S3_ADDRESS     = "s3.endpoint"
 	KEY_CLIENT_S3_BUCKET_NAME = "s3.bucket_name"
 
+	KEY_AUTH_CLIENT_ENABLE = "auth.client.enable"
+	KEY_AUTH_CLIENT_KEY    = "auth.client.key"
+	KEY_AUTH_CLIENT_ID     = "auth.client.id"
+
 	DEFAULT_CORE_LOCATE_DIR = "/core"
 )
 
@@ -183,6 +187,9 @@ func (cc *ClientConfig) GetS3AccessKey() string              { return cc.getStri
 func (cc *ClientConfig) GetS3SecretKey() string              { return cc.getString(KEY_CLIENT_S3_SECRET_KEY) }
 func (cc *ClientConfig) GetS3Address() string                { return cc.getString(KEY_CLIENT_S3_ADDRESS) }
 func (cc *ClientConfig) GetS3BucketName() string             { return cc.getString(KEY_CLIENT_S3_BUCKET_NAME) }
+func (cc *ClientConfig) GetAuthClientEnable() bool           { return cc.getBool(KEY_AUTH_CLIENT_ENABLE) }
+func (cc *ClientConfig) GetAuthClientKey() string            { return cc.getString(KEY_AUTH_CLIENT_KEY) }
+func (cc *ClientConfig) GetAuthClientId() string             { return cc.getString(KEY_AUTH_CLIENT_ID) }
 func (cc *ClientConfig) GetContainerPid() string             { return cc.getString(KEY_CONTAINER_PID) }
 func (cc *ClientConfig) GetEnvironments() string             { return cc.getString(KEY_ENVIRONMENT) }
 func (cc *ClientConfig) GetCoreLocateDir() string            { return DEFAULT_CORE_LOCATE_DIR }

internal/configure/topology/dc.go

@@ -44,6 +44,7 @@ const (
 	ROLE_CHUNKSERVER   = "chunkserver"
 	ROLE_SNAPSHOTCLONE = "snapshotclone"
 	ROLE_METASERVER    = "metaserver"
+	ROLE_TOOLS         = "tools"
 )
 
 type (
@@ -123,6 +124,33 @@ func NewDeployConfig(ctx *Context, kind, role, host, name string, replicas int,
 	}
 	delete(config, CONFIG_VARIABLE.key)
 
+	// auth.enable is optional and default is false
+	if config[CONFIG_ENABLE_AUTH.key] == nil {
+		config[CONFIG_ENABLE_AUTH.key] = CONFIG_ENABLE_AUTH.defaultValue
+	}
+
+	// and user only configure auth.key.current is ok.
+	authEnable := config[CONFIG_ENABLE_AUTH.key].(bool)
+	if authEnable && role != ROLE_ETCD {
+		// autn.client.enable is equal to auth.enable
+		if config[CONFIG_ENABLE_CLIENT_AUTH.key] == nil {
+			config[CONFIG_ENABLE_CLIENT_AUTH.key] = config[CONFIG_ENABLE_AUTH.key]
+		}
+		// auth.client.key is equal to auth.key.current
+		if config[CONFIG_AUTH_CLIENT_KEY.key] == nil {
+			config[CONFIG_AUTH_CLIENT_KEY.key] = config[CONFIG_AUTH_KEY_CURRENT.key]
+		}
+		// auth.key.last
+		if config[CONFIG_AUTH_KEY_LAST.key] != nil &&
+			config[CONFIG_AUTH_CLIENT_LASTKEY.key] == nil {
+			config[CONFIG_AUTH_CLIENT_LASTKEY.key] = config[CONFIG_AUTH_KEY_LAST.key]
+		}
+		// auth.client.id
+		if config[CONFIG_AUTH_CLIENT_ID.key] == nil {
+			config[CONFIG_AUTH_CLIENT_ID.key] = fmt.Sprintf("%s_%s", role, ROLE_TOOLS)
+		}
+	}
+
 	// We should convert all value to string for rendering variable,
 	// after that we will convert the value to specified type according to
 	// the its require type

internal/configure/topology/dc_get.go

@@ -145,6 +145,16 @@ func (dc *DeployConfig) GetS3Address() string        { return dc.getString(CONFI
 func (dc *DeployConfig) GetS3BucketName() string     { return dc.getString(CONFIG_S3_BUCKET_NAME) }
 func (dc *DeployConfig) GetEnableRDMA() bool         { return dc.getBool(CONFIG_ENABLE_RDMA) }
 func (dc *DeployConfig) GetEnableRenameAt2() bool    { return dc.getBool(CONFIG_ENABLE_RENAMEAT2) }
+func (dc *DeployConfig) GetAuthEnable() bool         { return dc.getBool(CONFIG_ENABLE_AUTH) }
+func (dc *DeployConfig) GetAuthClientEnable() bool   { return dc.getBool(CONFIG_ENABLE_CLIENT_AUTH) }
+func (dc *DeployConfig) GetAuthClientKey() string    { return dc.getString(CONFIG_AUTH_CLIENT_KEY) }
+func (dc *DeployConfig) GetAuthKeyCurrent() string   { return dc.getString(CONFIG_AUTH_KEY_CURRENT) }
+func (dc *DeployConfig) GetAuthServerKey() string    { return dc.getString(CONFIG_AUTH_SERVER_KEY) }
+func (dc *DeployConfig) GetAuthKeyLast() string      { return dc.getString(CONFIG_AUTH_KEY_LAST) }
+func (dc *DeployConfig) GetAuthClientId() string     { return dc.getString(CONFIG_AUTH_CLIENT_ID) }
+func (dc *DeployConfig) GetAuthClientLastkey() string {
+	return dc.getString(CONFIG_AUTH_CLIENT_LASTKEY)
+}
 func (dc *DeployConfig) GetEnableChunkfilePool() bool {
 	return dc.getBool(CONFIG_ENABLE_CHUNKFILE_POOL)
 }

internal/configure/topology/dc_item.go

@@ -24,7 +24,10 @@
 
 package topology
 
-import "path"
+import (
+	"fmt"
+	"path"
+)
 
 const (
 	REQUIRE_ANY = iota
@@ -272,6 +275,64 @@ var (
 		true,
 	)
 
+	CONFIG_ENABLE_AUTH = itemset.insert(
+		"auth.enable",
+		REQUIRE_BOOL,
+		false,
+		false,
+	)
+
+	CONFIG_ENABLE_CLIENT_AUTH = itemset.insert(
+		"auth.client.enable",
+		REQUIRE_BOOL,
+		false,
+		false,
+	)
+
+	CONFIG_AUTH_KEY_CURRENT = itemset.insert(
+		"auth.key.current",
+		REQUIRE_STRING,
+		false,
+		nil,
+	)
+
+	CONFIG_AUTH_SERVER_KEY = itemset.insert(
+		"auth.server.key",
+		REQUIRE_STRING,
+		false,
+		nil,
+	)
+
+	CONFIG_AUTH_CLIENT_KEY = itemset.insert(
+		"auth.client.key",
+		REQUIRE_STRING,
+		false,
+		nil,
+	)
+
+	CONFIG_AUTH_KEY_LAST = itemset.insert(
+		"auth.key.last",
+		REQUIRE_STRING,
+		false,
+		nil,
+	)
+
+	CONFIG_AUTH_CLIENT_LASTKEY = itemset.insert(
+		"auth.client.lastkey",
+		REQUIRE_STRING,
+		false,
+		nil,
+	)
+
+	CONFIG_AUTH_CLIENT_ID = itemset.insert(
+		"auth.client.id",
+		REQUIRE_STRING,
+		false,
+		func(dc *DeployConfig) interface{} {
+			return fmt.Sprintf("%s_%s", dc.GetRole(), "tool")
+		},
+	)
+
 	CONFIG_ENABLE_CHUNKFILE_POOL = itemset.insert(
 		"chunkfilepool.enable_get_chunk_from_pool",
 		REQUIRE_BOOL,

internal/errno/errno.go

@@ -413,21 +413,23 @@ var (
 	ERR_DECODE_CLUSTER_POOL_JSON_FAILED      = EC(410016, "decode cluster pool json to string failed")
 	ERR_WAIT_MDS_ELECTION_SUCCESS_TIMEOUT    = EC(410017, "wait mds election success timeout")
 	ERR_WAIT_ALL_CHUNKSERVERS_ONLINE_TIMEOUT = EC(410018, "wait all chunkservers online timeout")
-	ERR_CREATE_LOGICAL_POOL_FAILED           = EC(410019, "create logical pool failed")
+	ERR_CREATE_LOGICAL_POOL_FAILED           = EC(410019, "create physical/logical pool failed")
 	ERR_INVALID_DEVICE_USAGE                 = EC(410020, "invalid device usage")
 	ERR_ENCRYPT_FILE_FAILED                  = EC(410021, "encrypt file failed")
 	ERR_CLIENT_ID_NOT_FOUND                  = EC(410022, "client id not found")
 
 	// 420: common (curvebs client)
-	ERR_VOLUME_ALREADY_MAPPED             = EC(420000, "volume already mapped")
-	ERR_VOLUME_CONTAINER_LOSED            = EC(420001, "volume container is losed")
-	ERR_VOLUME_CONTAINER_ABNORMAL         = EC(420002, "volume container is abnormal")
-	ERR_CREATE_VOLUME_FAILED              = EC(420003, "create volume failed")
-	ERR_MAP_VOLUME_FAILED                 = EC(420004, "map volume to NBD device failed")
-	ERR_ENCODE_VOLUME_INFO_TO_JSON_FAILED = EC(420005, "encode volume info to json failed")
-	ERR_UNMAP_VOLUME_FAILED               = EC(420006, "unmap volume failed")
-	ERR_OLD_TARGET_DAEMON_IS_ABNORMAL     = EC(420007, "old target daemon is abnormal")
-	ERR_TARGET_DAEMON_IS_ABNORMAL         = EC(420008, "target daemon is abnormal")
+	ERR_VOLUME_ALREADY_MAPPED                   = EC(420000, "volume already mapped")
+	ERR_VOLUME_CONTAINER_LOSED                  = EC(420001, "volume container is losed")
+	ERR_VOLUME_CONTAINER_ABNORMAL               = EC(420002, "volume container is abnormal")
+	ERR_CREATE_VOLUME_FAILED                    = EC(420003, "create volume failed")
+	ERR_MAP_VOLUME_FAILED                       = EC(420004, "map volume to NBD device failed")
+	ERR_ENCODE_VOLUME_INFO_TO_JSON_FAILED       = EC(420005, "encode volume info to json failed")
+	ERR_UNMAP_VOLUME_FAILED                     = EC(420006, "unmap volume failed")
+	ERR_OLD_TARGET_DAEMON_IS_ABNORMAL           = EC(420007, "old target daemon is abnormal")
+	ERR_TARGET_DAEMON_IS_ABNORMAL               = EC(420008, "target daemon is abnormal")
+	ERR_CREATE_VOLUME_FAILED_AUTH_FAILED        = EC(420009, "create volume failed with errCode: kAuthFailed")
+	ERR_CREATE_VOLUME_FAILED_AUTH_KEY_NOT_EXIST = EC(420010, "create volume failed because auth key not exist")
 
 	// 430: common (curvefs client)
 	ERR_FS_PATH_ALREADY_MOUNTED  = EC(430000, "path already mounted")
@@ -464,6 +466,11 @@ var (
 	ERR_CHUNKSERVER_REQUIRES_3_HOSTS      = EC(503007, "chunkserver requires at least 3 hosts to distrubute zones")
 	ERR_SNAPSHOTCLONE_REQUIRES_3_HOSTS    = EC(503008, "snapshotclone requires at least 3 hosts for deploy")
 	ERR_METASERVER_REQUIRES_3_HOSTS       = EC(503009, "metaserver requires at least 3 hosts to distrubute zones")
+	// 504: checker (topology/auth)
+	ERR_AUTH_SERVER_KEY_REQUIRE_SET           = EC(504000, "auth.server.key requires to be set")
+	ERR_AUTH_CURRENT_KEY_REQUIRE_SET          = EC(504001, "auth.key.current requires to be set")
+	ERR_AUTH_SERVER_KEY_REQUIRE_16_CHARACTER  = EC(504002, "auth.server.key requires 16 characters")
+	ERR_AUTH_CURRENT_KEY_REQUIRE_16_CHARACTER = EC(504003, "auth.key.current requires 16 characters")
 
 	// 510: checker (ssh)
 	ERR_SSH_CONNECT_FAILED = EC(510000, "SSH connect failed")
@@ -545,6 +552,7 @@ var (
 	ERR_SECURE_COPY_FILE_TO_REMOTE_FAILED          = EC(620026, "secure copy file to remote failed (scp)")
 	ERR_RUN_SCRIPT_FAILED                          = EC(620998, "run script failed (bash script.sh)")
 	ERR_RUN_A_BASH_COMMAND_FAILED                  = EC(620999, "run a bash command failed (bash -c)")
+	ERR_DIST_SERVICE_KEY_FAILED                    = EC(621000, "distribute service auth key failed")
 
 	// 630: execute task (docker command)
 	ERR_GET_DOCKER_INFO_FAILED          = EC(630000, "get docker info failed (docker info)")

internal/playbook/factory.go

@@ -90,6 +90,7 @@ const (
 	CREATE_VOLUME
 	MAP_IMAGE
 	UNMAP_IMAGE
+	DIST_AUTH_KEY
 
 	// monitor
 	PULL_MONITOR_IMAGE
@@ -232,6 +233,8 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
 		case CREATE_PHYSICAL_POOL,
 			CREATE_LOGICAL_POOL:
 			t, err = comm.NewCreateTopologyTask(curveadm, config.GetDC(i))
+		case DIST_AUTH_KEY:
+			t, err = comm.NewDiskAuthKeyTask(curveadm, config.GetDC(i))
 		case UPDATE_TOPOLOGY:
 			t, err = comm.NewUpdateTopologyTask(curveadm, nil)
 		case INIT_SERVIE_STATUS:

internal/task/scripts/create_volume.go

@@ -34,10 +34,14 @@ g_volume=$2
 g_size=$3
 g_poolset=$4
 
-output=$(curve_ops_tool create -userName=$g_user -fileName=$g_volume -fileLength=$g_size -poolset=$g_poolset)
+output=$(curve_ops_tool create -userName=$g_user -fileName=$g_volume -fileLength=$g_size -poolset=$g_poolset 2>dev/null)
 if [ $? -ne 0 ]; then
   if [ "$output" = "CreateFile fail with errCode: 101" ]; then
      echo "EXIST"
+  elif echo ${output} | grep -q "kAuthFailed"; then
+     echo "AuthFailed"
+  elif echo ${output} | grep -q "auth info fail"; then
+     echo "AUTH_KEY_NOT_EXIST"
   else
      echo "FAILED"
   fi