func ScaleUp(context *context.AutoscalingContext, processors *ca_processors.AutoscalingProcessors, clusterStateRegistry *clusterstate.ClusterStateRegistry, unschedulablePods []*apiv1.Pod, nodes []*apiv1.Node, daemonSets []*appsv1.DaemonSet, nodeInfos map[string]*schedulernodeinfo.NodeInfo, ignoredTaints taints.TaintKeySet) (*status.ScaleUpStatus, errors.AutoscalerError) {
    
    ......
    // 驗證當前集群中所有 ready node 是否來自于 nodeGroups，取得所有非組內(nèi)的 node
    nodesFromNotAutoscaledGroups, err := utils.FilterOutNodesFromNotAutoscaledGroups(nodes, context.CloudProvider)
    if err != nil {
        return &status.ScaleUpStatus{Result: status.ScaleUpError}, err.AddPrefix("failed to filter out nodes which are from not autoscaled groups: ")
    }

    nodeGroups := context.CloudProvider.NodeGroups()
    gpuLabel := context.CloudProvider.GPULabel()
    availableGPUTypes := context.CloudProvider.GetAvailableGPUTypes()

    // 資源限制對象，會在 build cloud provider 時傳入
    // 如果有需要可在 CloudProvider 中自行更改，但不建議改動，會對用戶造成迷惑
    resourceLimiter, errCP := context.CloudProvider.GetResourceLimiter()
    if errCP != nil {
        return &status.ScaleUpStatus{Result: status.ScaleUpError}, errors.ToAutoscalerError(
            errors.CloudProviderError,
            errCP)
    }

    // 計算資源限制
    // nodeInfos 是所有擁有節(jié)點組的節(jié)點與示例節(jié)點的映射
    // 示例節(jié)點會優(yōu)先考慮真實節(jié)點的數(shù)據(jù)，如果 NodeGroup 中還沒有真實節(jié)點的部署，則使用 Template 的節(jié)點數(shù)據(jù)
    scaleUpResourcesLeft, errLimits := computeScaleUpResourcesLeftLimits(context.CloudProvider, nodeGroups, nodeInfos, nodesFromNotAutoscaledGroups, resourceLimiter)
    if errLimits != nil {
        return &status.ScaleUpStatus{Result: status.ScaleUpError}, errLimits.AddPrefix("Could not compute total resources: ")
    }

    // 根據(jù)當前節(jié)點與 NodeGroups 中的節(jié)點來計算會有多少節(jié)點即將加入集群中
    // 由于云服務商的伸縮組 increase size 操作并不是同步加入 node，所以將其統(tǒng)計，以便于后面計算節(jié)點資源
    upcomingNodes := make([]*schedulernodeinfo.NodeInfo, 0)
    for nodeGroup, numberOfNodes := range clusterStateRegistry.GetUpcomingNodes() {
        ......
    }
    klog.V(4).Infof("Upcoming %d nodes", len(upcomingNodes))

    // 最終會進入選擇的節(jié)點組
    expansionOptions := make(map[string]expander.Option, 0)
    ......
    // 出于某些限制或錯誤導致不能加入新節(jié)點的節(jié)點組，例如節(jié)點組已達到 MaxSize
    skippedNodeGroups := map[string]status.Reasons{}
    // 綜合各種情況，篩選出節(jié)點組
    for _, nodeGroup := range nodeGroups {
    ......
    }
    iflen(expansionOptions) == 0 {
        klog.V(1).Info("No expansion options")
        return &status.ScaleUpStatus{
            Result:                 status.ScaleUpNoOptionsAvailable,
            PodsRemainUnschedulable: getRemainingPods(podEquivalenceGroups, skippedNodeGroups),
            ConsideredNodeGroups:   nodeGroups,
        }, nil
    }

    ......
    // 選擇一個最佳的節(jié)點組進行擴容，expander 用于選擇一個合適的節(jié)點組進行擴容，默認為 RandomExpander，flag: expander
    // random 隨機選一個，適合只有一個節(jié)點組
    // most-pods 選擇能夠調(diào)度最多 pod 的節(jié)點組，比如有 noSchedulerPods 是有 nodeSelector 的，它會優(yōu)先選擇此類節(jié)點組以滿足大多數(shù) pod 的需求
    // least-waste 優(yōu)先選擇能滿足 pod 需求資源的最小資源類型的節(jié)點組
    // price 根據(jù)價格模型，選擇最省錢的
    // priority 根據(jù)優(yōu)先級選擇
    bestOption := context.ExpanderStrategy.BestOption(options, nodeInfos)
    if bestOption != nil && bestOption.NodeCount > 0 {
    ......
        newNodes := bestOption.NodeCount

        // 考慮到 upcomingNodes, 重新計算本次新加入節(jié)點
        if context.MaxNodesTotal > 0 && len(nodes)+newNodes+len(upcomingNodes) > context.MaxNodesTotal {
            klog.V(1).Infof("Capping size to max cluster total size (%d)", context.MaxNodesTotal)
            newNodes = context.MaxNodesTotal - len(nodes) - len(upcomingNodes)
            if newNodes < 1 {
                return &status.ScaleUpStatus{Result: status.ScaleUpError}, errors.NewAutoscalerError(
                    errors.TransientError,
                    "max node total count already reached")
            }
        }

        createNodeGroupResults := make([]nodegroups.CreateNodeGroupResult, 0)
    
        // 如果節(jié)點組在云服務商端處不存在，會嘗試創(chuàng)建根據(jù)現(xiàn)有信息重新創(chuàng)建一個云端節(jié)點組
        // 但是目前所有的 CloudProvider 實現(xiàn)都沒有允許這種操作，這好像是個多余的方法
        // 云服務商不想，也不應該將云端節(jié)點組的創(chuàng)建權限交給 ClusterAutoscaler
        if !bestOption.NodeGroup.Exist() {
            oldId := bestOption.NodeGroup.Id()
            createNodeGroupResult, err := processors.NodeGroupManager.CreateNodeGroup(context, bestOption.NodeGroup)
        ......
        }

        // 得到最佳節(jié)點組的示例節(jié)點
        nodeInfo, found := nodeInfos[bestOption.NodeGroup.Id()]
        if !found {
            // This should never happen, as we already should have retrieved
            // nodeInfo for any considered nodegroup.
            klog.Errorf("No node info for: %s", bestOption.NodeGroup.Id())
            return &status.ScaleUpStatus{Result: status.ScaleUpError, CreateNodeGroupResults: createNodeGroupResults}, errors.NewAutoscalerError(
                errors.CloudProviderError,
                "No node info for best expansion option!")
        }

        // 根據(jù) CPU、Memory及可能存在的 GPU 資源（hack: we assume anything which is not cpu/memory to be a gpu.），計算出需要多少個 Nodes
        newNodes, err = applyScaleUpResourcesLimits(context.CloudProvider, newNodes, scaleUpResourcesLeft, nodeInfo, bestOption.NodeGroup, resourceLimiter)
        if err != nil {
            return &status.ScaleUpStatus{Result: status.ScaleUpError, CreateNodeGroupResults: createNodeGroupResults}, err
        }

        // 需要平衡的節(jié)點組
        targetNodeGroups := []cloudprovider.NodeGroup{bestOption.NodeGroup}
        // 如果需要平衡節(jié)點組，根據(jù) balance-similar-node-groups flag 設置。
        // 檢測相似的節(jié)點組，并平衡它們之間的節(jié)點數(shù)量
        if context.BalanceSimilarNodeGroups {
        ......
        }
        // 具體平衡策略可以看 (b *BalancingNodeGroupSetProcessor) BalanceScaleUpBetweenGroups 方法
        scaleUpInfos, typedErr := processors.NodeGroupSetProcessor.BalanceScaleUpBetweenGroups(context, targetNodeGroups, newNodes)
        if typedErr != nil {
            return &status.ScaleUpStatus{Result: status.ScaleUpError, CreateNodeGroupResults: createNodeGroupResults}, typedErr
        }
        klog.V(1).Infof("Final scale-up plan: %v", scaleUpInfos)
        // 開始擴容，通過 IncreaseSize 擴容
        for _, info := range scaleUpInfos {
            typedErr := executeScaleUp(context, clusterStateRegistry, info, gpu.GetGpuTypeForMetrics(gpuLabel, availableGPUTypes, nodeInfo.Node(), nil), now)
            if typedErr != nil {
                return &status.ScaleUpStatus{Result: status.ScaleUpError, CreateNodeGroupResults: createNodeGroupResults}, typedErr
            }
        }
        ......
    }
    ......
}

apiVersion: eksctl.io/v1alpha5
kind:ClusterConfig
nodeGroups:
-name:ng-spot
    instanceType:m5.large
    spot:true
    minSize:0
    maxSize:10
    labels:
      node-type:spot
-name:ng-on-demand
    instanceType:m5.xlarge
    minSize:1
    maxSize:5
    labels:
      node-type:on-demand

metadata:
  annotations:
    cluster-autoscaler.kubernetes.io/scale-down-disabled: "true"

type CloudProvider interface {
    // Name returns name of the cloud provider.
    Name() string

    // NodeGroups returns all node groups configured for this cloud provider.
    // 會在一此循環(huán)中多次調(diào)用此方法，所以不適合每次都請求云廠商服務，可以在 Refresh 時存儲狀態(tài)
    NodeGroups() []NodeGroup

    // NodeGroupForNode returns the node group for the given node, nil if the node
    // should not be processed by cluster autoscaler, or non-nil error if such
    // occurred. Must be implemented.
    // 同上
    NodeGroupForNode(*apiv1.Node) (NodeGroup, error)

    // Pricing returns pricing model for this cloud provider or error if not available.
    // Implementation optional.
    // 如果不使用 price expander 就可以不實現(xiàn)此方法
    Pricing() (PricingModel, errors.AutoscalerError)

    // GetAvailableMachineTypes get all machine types that can be requested from the cloud provider.
    // Implementation optional.
    // 沒用，不需要實現(xiàn)
    GetAvailableMachineTypes() ([]string, error)

    // NewNodeGroup builds a theoretical node group based on the node definition provided. The node group is not automatically
    // created on the cloud provider side. The node group is not returned by NodeGroups() until it is created.
    // Implementation optional.
    // 通常情況下，不需要實現(xiàn)此方法，但如果你需要 ClusterAutoscaler 創(chuàng)建一個默認的 NodeGroup 的話，也可以實現(xiàn)。
    // 但其實更好的做法是將默認 NodeGroup 寫入云端的伸縮組
    NewNodeGroup(machineType string, labels map[string]string, systemLabels map[string]string,
        taints []apiv1.Taint, extraResources map[string]resource.Quantity) (NodeGroup, error)

    // GetResourceLimiter returns struct containing limits (max, min) for resources (cores, memory etc.).
    // 資源限制對象，會在 build 時傳入，通常情況下不需要更改，除非在云端有顯示的提示用戶更改的地方，否則使用時會迷惑用戶
    GetResourceLimiter() (*ResourceLimiter, error)

    // GPULabel returns the label added to nodes with GPU resource.
    // GPU 相關，如果集群中有使用 GPU 資源，需要返回對應內(nèi)容。 hack: we assume anything which is not cpu/memory to be a gpu.
    GPULabel() string

    // GetAvailableGPUTypes return all available GPU types cloud provider supports.
    // 同上
    GetAvailableGPUTypes() map[string]struct{}

    // Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc.
    // CloudProvider 只會在啟動時被初始化一次，如果每次循環(huán)后有需要清除的內(nèi)容，在這里處理
    Cleanup() error

    // Refresh is called before every main loop and can be used to dynamically update cloud provider state.
    // In particular the list of node groups returned by NodeGroups can change as a result of CloudProvider.Refresh().
    // 會在 StaticAutoscaler RunOnce 中被調(diào)用
    Refresh() error
}

// NodeGroup contains configuration info and functions to control a set
// of nodes that have the same capacity and set of labels.
type NodeGroup interface {
    // MaxSize returns maximum size of the node group.
    MaxSize() int

    // MinSize returns minimum size of the node group.
    MinSize() int

    // TargetSize returns the current target size of the node group. It is possible that the
    // number of nodes in Kubernetes is different at the moment but should be equal
    // to Size() once everything stabilizes (new nodes finish startup and registration or
    // removed nodes are deleted completely). Implementation required.
    // 響應的是伸縮組的節(jié)點數(shù)，并不一定與 kubernetes 中的節(jié)點數(shù)保持一致
    TargetSize() (int, error)

    // IncreaseSize increases the size of the node group. To delete a node you need
    // to explicitly name it and use DeleteNode. This function should wait until
    // node group size is updated. Implementation required.
    // 擴容的方法，增加伸縮組的節(jié)點數(shù)
    IncreaseSize(delta int) error

    // DeleteNodes deletes nodes from this node group. Error is returned either on
    // failure or if the given node doesn't belong to this node group. This function
    // should wait until node group size is updated. Implementation required.
    // 刪除的節(jié)點一定要在該節(jié)點組中
    DeleteNodes([]*apiv1.Node) error
    // DecreaseTargetSize decreases the target size of the node group. This function
    // doesn't permit to delete any existing node and can be used only to reduce the
    // request for new nodes that have not been yet fulfilled. Delta should be negative.
    // It is assumed that cloud provider will not delete the existing nodes when there
    // is an option to just decrease the target. Implementation required.
    // 當 ClusterAutoscaler 發(fā)現(xiàn) kubernetes 節(jié)點數(shù)與伸縮組的節(jié)點數(shù)長時間不一致，會調(diào)用此方法來調(diào)整
    DecreaseTargetSize(delta int) error

    // Id returns an unique identifier of the node group.
    Id() string

    // Debug returns a string containing all information regarding this node group.
    Debug() string

    // Nodes returns a list of all nodes that belong to this node group.
    // It is required that Instance objects returned by this method have Id field set.
    // Other fields are optional.
    // This list should include also instances that might have not become a kubernetes node yet.
    // 返回伸縮組中的所有節(jié)點，哪怕它還沒有成為 kubernetes 的節(jié)點
    Nodes() ([]Instance, error)

    // TemplateNodeInfo returns a schedulernodeinfo.NodeInfo structure of an empty
    // (as if just started) node. This will be used in scale-up simulations to
    // predict what would a new node look like if a node group was expanded. The returned
    // NodeInfo is expected to have a fully populated Node object, with all of the labels,
    // capacity and allocatable information as well as all pods that are started on
    // the node by default, using manifest (most likely only kube-proxy). Implementation optional.
    // ClusterAutoscaler 會將節(jié)點信息與節(jié)點組對應，來判斷資源條件，如果是一個空的節(jié)點組，那么就會通過此方法來虛擬一個節(jié)點信息。
    TemplateNodeInfo() (*schedulernodeinfo.NodeInfo, error)

    // Exist checks if the node group really exists on the cloud provider side. Allows to tell the
    // theoretical node group from the real one. Implementation required.
    Exist() bool

    // Create creates the node group on the cloud provider side. Implementation optional.
    // 與 CloudProvider.NewNodeGroup 配合使用
    Create() (NodeGroup, error)

    // Delete deletes the node group on the cloud provider side.
    // This will be executed only for autoprovisioned node groups, once their size drops to 0.
    // Implementation optional.
    Delete() error

    // Autoprovisioned returns true if the node group is autoprovisioned. An autoprovisioned group
    // was created by CA and can be deleted when scaled to 0.
    Autoprovisioned() bool
}

kind: DaemonSet
metadata:
  annotations:
    cluster-autoscaler.kubernetes.io/daemonset-taint-eviction: "true"

apiVersion: policy/v1
kind:PodDisruptionBudget
metadata:
name:zk-pdb
spec:
minAvailable:2
selector:
    matchLabels:
      app:zookeeper

# 生產(chǎn)環(huán)境推薦配置（兼顧響應速度與穩(wěn)定性）
command:
-./cluster-autoscaler
---v=4
---stderrthreshold=info
---cloud-provider=aws
---skip-nodes-with-local-storage=false
---expander=least-waste
---scale-down-delay-after-add=20m
---scale-down-unneeded-time=15m
--balance-similar-node-groups=true