【问题排查】一次 pod 调度失败后一直处于 nominated 状态排查

$ kubectl get po -n test-ns test-pod-hgfmk -owide
NAME                      READY    STATUS    RESTARTS    AGE    IP    NODE    NOMINATED NODE    
test-pod-hgfmk             0/1     Pending     0         5m   <none>  <none>   132.10.134.193

// 节点信息
{"log":"Node name: 132.10.134.193\n","stream":"stderr","time":"2021-10-16T08:06:42.929073912Z"}

// 节点requests信息（和上图相符）
{"log":"Requested Resources: {MilliCPU:37500 Memory:56837160960 EphemeralStorage:42949672960 AllowedPodNumber:0 ScalarResources:map[teg.tkex.oa.com/amd-cpu:36000 tke.cloud.tencent.com/eni-ip:1]}\n","stream":"stderr","time":"2021-10-16T08:06:42.929076647Z"}

// 节点 allocatable 信息（总可分配资源）
{"log":"Allocatable Resources:{MilliCPU:98667 Memory:191588200448 EphemeralStorage:482947890401 AllowedPodNumber:110 ScalarResources:map[hugepages-1Gi:0 hugepages-2Mi:0]}\n","stream":"stderr","time":"2021-10-16T08:06:42.929079502Z"}

// 节点 pod 信息
{"log":"Scheduled Pods(number: 3):\n","stream":"stderr","time":"2021-10-16T08:06:42.929085774Z"}
{"log":"name: ip-masq-agent-8v5lm, namespace: kube-system, uid: eee008d9-52ba-4c6d-ba7a-993a9c255718, phase: Running, nominated node: \n","stream":"stderr","time":"2021-10-16T08:06:42.929102926Z"}
{"log":"name: kube-proxy-7sb7d, namespace: kube-system, uid: c53ef512-59f9-4afc-aa32-5f327b1e441d, phase: Running, nominated node: \n","stream":"stderr","time":"2021-10-16T08:06:42.929105692Z"}
{"log":"name: node-exporter-p6rr4, namespace: kube-system, uid: dec7d32d-0652-430e-a013-ac40404a8428, phase: Running, nominated node: \n","stream":"stderr","time":"2021-10-16T08:06:42.929122523Z"}
{"log":"Nominated Pods(number: 1):\n","stream":"stderr","time":"2021-10-16T08:06:42.929125529Z"}

// 该 pod 就是本次出现的 pending pod
{"log":"name: test-pod-hgfmk, namespace: test-ns, uid: ae6fea01-ba91-4a34-9c10-b57f04f2fbf7, phase: Pending, nominated node: 132.10.134.193\n","stream":"stderr","time":"2021-10-16T08:06:42.929128224Z"}

cat << EOF > ./test.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
 labels:
   app: test-schedule
 name: test-schedule
spec:
 replicas: 1
 selector:
   matchLabels:
     app: test-schedule
 template:
   metadata:
     labels:
       app: test-schedule
   spec:
     nodeSelector:
       kubernetes.io/hostname: 132.10.134.193
     containers:
     - image: mirrors.tencent.com/tkestack/nginx:1.7.9
       name: test-schedule
       resources:
          limits:
            cpu: "70"
            memory: 100Gi
          requests:
            cpu: "53"
            memory: 70Gi
     tolerations:
     - effect: NoSchedule
       key: zone/ap-gz-3
       operator: Exists
     - effect: NoSchedule
       key: zone/ap-gz-5
       operator: Exists
     - effect: NoSchedule
       key: zone/ap-gz-6
       operator: Exists
EOF
kubectl create -f test.yaml

// pkg/scheduler/core/generic_scheduler.go
func (g *genericScheduler) Preempt(ctx context.Context, prof *profile.Profile, state *framework.CycleState, pod *v1.Pod, scheduleErr error) (*v1.Node, []*v1.Pod, []*v1.Pod, error) {
   ...
   ...
   // 1、从缓存快照中获取集群所有节点信息   
   allNodes, err := g.nodeInfoSnapshot.NodeInfos().List()
   ...
   // 2、将集群所有健康节点作为备选的 nominated 节点
   potentialNodes := nodesWherePreemptionMightHelp(allNodes, fitError)
   ...
   // 3、从备选 nominated 节点中选出能够真正成为 nominated node 的节点
   // 条件：去掉节点上所有比该 pod 低优先级的 pod 后，再跑一次 filter 调度算法后，集群中有满足条件的节点，则作为 nominated node
   nodeToVictims, err := g.selectNodesForPreemption(ctx, prof, state, pod, potentialNodes, pdbs)
   ...
   // 4、不仅需要走默认调度器的抢占逻辑，还有走扩展调度器的抢占逻辑，再过滤一批 nominated 节点
   nodeToVictims, err = g.processPreemptionWithExtenders(pod, nodeToVictims)
   ...
   // 5、最终剩下的 nominated 节点随机选取一个作为 pod 的nominated node
   candidateNode := pickOneNodeForPreemption(nodeToVictims)
   ...
   return candidateNode, nodeToVictims[candidateNode].Pods, nominatedPods, nil
}

// pkg/schedulerplugin/preempt.go
func (p *FloatingIPPlugin) Preempt(args *schedulerapi.ExtenderPreemptionArgs) map[string]*schedulerapi.MetaVictims {
   fillNodeNameToMetaVictims(args)
   // 1、解析 pod 的 ip 释放策略（永不释放IP、缩容时释放IP、不保留IP）
   policy := parseReleasePolicy(&args.Pod.ObjectMeta)
   // 2、如果 pod 的 ip 释放策略为不保留IP时，则不需要过滤 nominated 节点直接返回
   if policy == constant.ReleasePolicyPodDelete {
      return args.NodeNameToMetaVictims
   }
   // 3、获取业务 Pod 所属子网（因为固定ip策略，pod之前的ip所属子网保存在fip对象中）
   subnetSet, err := p.getSubnet(args.Pod)
   for nodeName := range args.NodeNameToMetaVictims {
      node, err := p.NodeLister.Get(nodeName)
      // 4、获取备选的 nominated node 的子网
      subnet, err := p.getNodeSubnet(node)
      if err != nil {
         glog.Errorf("unable to get node %v subnet: %v", nodeName, err)
         // 5、如果无法调 eni-provider 获取到 node 子网，滤掉这些节点
         delete(args.NodeNameToMetaVictims, nodeName)
         continue
      }
      // 6、如果 node 子网和 pod 原来的子网不匹配，也过滤掉这些节点
      if !subnetSet.Has(subnet.String()) {
         glog.V(4).Infof("remove node %v with subnet %v from victim", node.Name, subnet.String())
         delete(args.NodeNameToMetaVictims, nodeName)
      }
   }
   // 7、返回过滤后的 nominated 节点
   return args.NodeNameToMetaVictims
}

$ cat /etc/kubernetes/schduler-config.json
{
        "apiVersion": "kubescheduler.config.k8s.io/v1alpha2",
        "kind": "KubeSchedulerConfiguration",
        "leaderElection": {
                "leaderElect": true
        },
        ...
        "extenders": [
                {
                        "apiVersion": "v1beta1",
                        "enableHttps": false,
                        "filterVerb": "filter",
                        "BindVerb": "bind",
                        "weight": 1,
                        // 抢占配置
                        //"preemptVerb": "preempt",
                        "nodeCacheCapable": false,
                        "urlPrefix": "http://sheduler-extender:1003/v1",
                }
        ]
}

$ kubectl get po -n test-ns test-pod-bvtjg -owide
NAME                      READY    STATUS    RESTARTS    AGE    IP    NODE    NOMINATED NODE    
test-pod-bvtjg             0/1     Pending     0         5m   <none>  <none>   <none>