// kubeGenericRuntimeManager::runtimeService:  newInstrumentedRuntimeService(runtimeService)

// SyncPod syncs the running pod into the desired pod by executing following steps:
//
//  1. Compute sandbox and container changes.
//  2. Kill pod sandbox if necessary.
//  3. Kill any containers that should not be running.
//  4. Create sandbox if necessary.
//  5. Create init containers.
//  6. Create normal containers.
func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) {
// Step 1: Compute sandbox and container changes.

// Step 2: Kill the pod if the sandbox has changed.
if podContainerChanges.KillPod {
    // ...
} else {
// Step 3: kill any running containers in this pod which are not to keep.
// ...
}

// Keep terminated init containers fairly aggressively controlled
// This is an optimization because container removals are typically handled
// by container garbage collector.
m.pruneInitContainersBeforeStart(pod, podStatus)

// We pass the value of the podIP down to generatePodSandboxConfig and
// generateContainerConfig, which in turn passes it to various other
// functions, in order to facilitate functionality that requires this
// value (hosts file and downward API) and avoid races determining
// the pod IP in cases where a container requires restart but the
// podIP isn't in the status manager yet.
//
// We default to the IP in the passed-in pod status, and overwrite it if the
// sandbox needs to be (re)started.
podIP := ""
if podStatus != nil {
podIP = podStatus.IP
}

// Step 4: Create a sandbox for the pod if necessary.  
  // 创建使用 pause 镜像创建的 sandbox
podSandboxID := podContainerChanges.SandboxID
if podContainerChanges.CreateSandbox {
// ...
podSandboxID, msg, err = m.createPodSandbox(pod, podContainerChanges.Attempt)
    // 底层调用 m.runtimeService.RunPodSandbox(podSandboxConfig, runtimeHandler)
    
glog.V(4).Infof("Created PodSandbox %q for pod %q", podSandboxID, format.Pod(pod))

podSandboxStatus, err := m.runtimeService.PodSandboxStatus(podSandboxID)
}

// Get podSandboxConfig for containers to start.
podSandboxConfig, err := m.generatePodSandboxConfig(pod, podContainerChanges.Attempt)

// Step 5: start the init container.  启动所有的 init container 
if container := podContainerChanges.NextInitContainerToStart; container != nil {
// Start the next init container.
glog.V(4).Infof("Creating init container %+v in pod %v", container, format.Pod(pod))
if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP, kubecontainer.ContainerTypeInit); err != nil {
// ...
}

// Successfully started the container; clear the entry in the failure
glog.V(4).Infof("Completed init container %q for pod %q", container.Name, format.Pod(pod))
}

// Step 6: start containers in podContainerChanges.ContainersToStart. 启动 container 
for _, idx := range podContainerChanges.ContainersToStart {
container := &pod.Spec.Containers[idx]
    // ...

glog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod))
if msg, err := m.startContainer(podSandboxID, podSandboxConfig, container, pod, podStatus, pullSecrets, podIP, kubecontainer.ContainerTypeRegular); err != nil {
    // ...
}
}

return
}

type dockerService struct {
client           libdocker.Interface
os               kubecontainer.OSInterface
podSandboxImage  string
streamingRuntime *streamingRuntime
streamingServer  streaming.Server

network *network.PluginManager   // 实现了网络
// Map of podSandboxID :: network-is-ready
networkReady     map[string]bool
networkReadyLock sync.Mutex

containerManager cm.ContainerManager
// cgroup driver used by Docker runtime.
cgroupDriver      string
checkpointManager checkpointmanager.CheckpointManager
// caches the version of the runtime.
// To be compatible with multiple docker versions, we need to perform
// version checking for some operations. Use this cache to avoid querying
// the docker daemon every time we need to do such checks.
versionCache *cache.ObjectCache
// startLocalStreamingServer indicates whether dockershim should start a
// streaming server on localhost.
startLocalStreamingServer bool
}

// RuntimeService interface should be implemented by a container runtime.
// The methods should be thread-safe.
type RuntimeService interface {
RuntimeVersioner
ContainerManager
PodSandboxManager
ContainerStatsManager

// UpdateRuntimeConfig updates runtime configuration if specified
UpdateRuntimeConfig(runtimeConfig *runtimeapi.RuntimeConfig) error
// Status returns the status of the runtime.
Status() (*runtimeapi.RuntimeStatus, error)
}

// PodSandboxManager contains methods for operating on PodSandboxes. The methods
// are thread-safe.
type PodSandboxManager interface {
// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
// the sandbox is in ready state.
RunPodSandbox(config *runtimeapi.PodSandboxConfig, runtimeHandler string) (string, error)
// StopPodSandbox stops the sandbox. If there are any running containers in the
// sandbox, they should be force terminated.
StopPodSandbox(podSandboxID string) error
// RemovePodSandbox removes the sandbox. If there are running containers in the
// sandbox, they should be forcibly removed.
RemovePodSandbox(podSandboxID string) error
// PodSandboxStatus returns the Status of the PodSandbox.
PodSandboxStatus(podSandboxID string) (*runtimeapi.PodSandboxStatus, error)
// ListPodSandbox returns a list of Sandbox.
ListPodSandbox(filter *runtimeapi.PodSandboxFilter) ([]*runtimeapi.PodSandbox, error)
// PortForward prepares a streaming endpoint to forward ports from a PodSandbox, and returns the address.
PortForward(*runtimeapi.PortForwardRequest) (*runtimeapi.PortForwardResponse, error)
}

// RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
// the sandbox is in ready state.
// For docker, PodSandbox is implemented by a container holding the network
// namespace for the pod.
// Note: docker doesn't use LogDirectory (yet).
func (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPodSandboxRequest) (*runtimeapi.RunPodSandboxResponse, error) {
config := r.GetConfig()

// Step 1: Pull the image for the sandbox.
image := defaultSandboxImage
if err := ensureSandboxImageExists(ds.client, image); err != nil {
}

createConfig, err := ds.makeSandboxDockerConfig(config, image)
createResp, err := ds.client.CreateContainer(*createConfig)

resp := &runtimeapi.RunPodSandboxResponse{PodSandboxId: createResp.ID}

ds.setNetworkReady(createResp.ID, false)

// Step 3: Create Sandbox Checkpoint.
if err = ds.checkpointManager.CreateCheckpoint(createResp.ID, constructPodSandboxCheckpoint(config)); err != nil {
return nil, err
}

// Step 4: Start the sandbox container.
// Assume kubelet's garbage collector would remove the sandbox later, if
// startContainer failed.
err = ds.client.StartContainer(createResp.ID)

  // 设置 DNS config
// Do not invoke network plugins if in hostNetwork mode.
if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetNetwork() == runtimeapi.NamespaceMode_NODE {
return resp, nil
}

// Step 5: Setup networking for the sandbox.
// All pod networking is setup by a CNI plugin discovered at startup time.
// This plugin assigns the pod ip, sets up routes inside the sandbox,
// creates interfaces etc. In theory, its jurisdiction ends with pod
// sandbox networking, but it might insert iptables rules or open ports
// on the host as well, to satisfy parts of the pod spec that aren't
// recognized by the CNI standard yet.
cID := kubecontainer.BuildContainerID(runtimeName, createResp.ID)
err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations)
return resp, nil
}

func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
netnsPath, err := plugin.host.GetNetNS(id.ID)

// Windows doesn't have loNetwork. It comes only with Linux
if plugin.loNetwork != nil {
if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations); err != nil {
}
}
  
_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations)

return err
}

func NewDockerService(...){
  // dockershim currently only supports CNI plugins.
  // 使用传入的 cni 配置目录和bin目录，初始化插件，并供后续选择
pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString)
cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs)
cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs))
  
    // cniPlugins 传入全部的系统 CNI 插件
  plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)

ds.network = network.NewPluginManager(plug)
}

type cniNetworkPlugin struct {
network.NoopNetworkPlugin

loNetwork *cniNetwork        // 指向 lo CNI 的实现，参见后续分析
  defaultNetwork *cniNetwork   // 指向 default CNI 的实现，参见后续分析

sync.RWMutex
defaultNetwork *cniNetwork

host        network.Host
execer      utilexec.Interface
nsenterPath string
confDir     string
binDirs     []string
podCidr     string
}

// Plugin is an interface to network plugins for the kubelet
type NetworkPlugin interface {
// Init initializes the plugin.  This will be called exactly once
// before any other methods are called.
Init(host Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error

// Called on various events like:
// NET_PLUGIN_EVENT_POD_CIDR_CHANGE
Event(name string, details map[string]interface{})

// Name returns the plugin's name. This will be used when searching
// for a plugin by name, e.g.
Name() string

// Returns a set of NET_PLUGIN_CAPABILITY_*
Capabilities() utilsets.Int

// SetUpPod is the method called after the infra container of
// the pod has been created but before the other containers of the
// pod are launched.
SetUpPod(namespace string, name string, podSandboxID kubecontainer.ContainerID, annotations map[string]string) error

// TearDownPod is the method called before a pod's infra container will be deleted
TearDownPod(namespace string, name string, podSandboxID kubecontainer.ContainerID) error

// GetPodNetworkStatus is the method called to obtain the ipv4 or ipv6 addresses of the container
GetPodNetworkStatus(namespace string, name string, podSandboxID kubecontainer.ContainerID) (*PodNetworkStatus, error)

// Status returns error if the network plugin is in error state
Status() error
}

type cniNetworkPlugin struct {
network.NoopNetworkPlugin

loNetwork *cniNetwork        // lo 本地端口
  defaultNetwork *cniNetwork  // 默认网络
}

type cniNetwork struct {
name          string
NetworkConfig *libcni.NetworkConfigList
CNIConfig     libcni.CNI   // 指向 CNI 接口定义
}

type CNI interface {
    AddNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
    DelNetworkList(net *NetworkConfigList, rt *RuntimeConf) error

    AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
    DelNetwork(net *NetworkConfig, rt *RuntimeConf) error
}

func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin {

plugin := &cniNetworkPlugin{
defaultNetwork: nil,
loNetwork:      getLoNetwork(binDirs),  // lo 初始化
execer:         utilexec.New(),
confDir:        confDir,
binDirs:        binDirs,
}

return []network.NetworkPlugin{plugin}
}

func getLoNetwork(binDirs []string) *cniNetwork {
loConfig, err := libcni.ConfListFromBytes([]byte(`{
  "cniVersion": "0.2.0",
  "name": "cni-loopback",
  "plugins":[{
    "type": "loopback"
  }]
}`))
if err != nil {
// The hardcoded config above should always be valid and unit tests will
// catch this
panic(err)
}
loNetwork := &cniNetwork{
name:          "lo",
NetworkConfig: loConfig,
CNIConfig:     &libcni.CNIConfig{Path: binDirs},
}

return loNetwork
}

func (plugin *cniNetworkPlugin) syncNetworkConfig() {
network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs)
plugin.setDefaultNetwork(network)
}

func getDefaulgotCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) {
files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
switch {
case err != nil:
return nil, err
case len(files) == 0:
return nil, fmt.Errorf("No networks found in %s", confDir)
}

sort.Strings(files)
for _, confFile := range files {
var confList *libcni.NetworkConfigList
if strings.HasSuffix(confFile, ".conflist") {
confList, err = libcni.ConfListFromFile(confFile)
} else {
conf, err := libcni.ConfFromFile(confFile)

// Ensure the config has a "type" so we know what plugin to run.
// Also catches the case where somebody put a conflist into a conf file.
if conf.Network.Type == "" {
glog.Warningf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
continue
}

confList, err = libcni.ConfListFromConf(conf)
if err != nil {
glog.Warningf("Error converting CNI config file %s to list: %v", confFile, err)
continue
}
}
if len(confList.Plugins) == 0 {
glog.Warningf("CNI config list %s has no networks, skipping", confFile)
continue
}

glog.V(4).Infof("Using CNI configuration file %s", confFile)

network := &cniNetwork{
name:          confList.Name,
NetworkConfig: confList,
CNIConfig:     &libcni.CNIConfig{Path: binDirs},
}
return network, nil
}
return nil, fmt.Errorf("No valid networks found in %s", confDir)
}

import (
  // ...
  "k8s.io/kubernetes/cmd/kube-scheduler/app"
)
func main() {
// ...

if err := command.Execute(); err != nil {
}
}

// NewSchedulerCommand creates a *cobra.Command object with default parameters
func NewSchedulerCommand() *cobra.Command {
// ...

cmd := &cobra.Command{
Use: "kube-scheduler",
Long: `...`,
Run: func(cmd *cobra.Command, args []string) {
// ...

stopCh := make(chan struct{})
if err := Run(c.Complete(), stopCh); err != nil {
}
},
}
return cmd
}

// Run runs the Scheduler.
func Run(c schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error {

  // 注册算法 参见流程 算法初始化
algorithmprovider.ApplyFeatureGates()  

  // 配置初始化
schedulerConfig, err := NewSchedulerConfig(c)  
  
// Create the scheduler.
sched := scheduler.NewFromConfig(schedulerConfig)

  // ...

// 启动 informer 监听
go c.PodInformer.Informer().Run(stopCh) 
c.InformerFactory.Start(stopCh)

// Wait for all caches to sync before scheduling.
c.InformerFactory.WaitForCacheSync(stopCh)
controller.WaitForCacheSync("scheduler", stopCh, c.PodInformer.Informer().HasSynced)

// Prepare a reusable run function.
run := func(ctx context.Context) {
sched.Run()  // 主入口
<-ctx.Done()
}

  // ...
  
run(ctx)
return fmt.Errorf("finished without leader elect")
}

// Run begins watching and scheduling. It waits for cache to be synced, then starts a goroutine and returns immediately.
func (sched *Scheduler) Run() {
if !sched.config.WaitForCacheSync() {
return
}

  // 主入口函数 scheduleOne 
go wait.Until(sched.scheduleOne, 0, sched.config.StopEverything)
}

// NewSchedulerCommand creates a *cobra.Command object with default parameters
func NewSchedulerCommand() *cobra.Command {
opts, err := options.NewOptions()

cmd := &cobra.Command{
Use: "kube-scheduler",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
c, err := opts.Config()  // 初始化配置

stopCh := make(chan struct{})
Run(c.Complete(), stopCh)  // 完成配置返回  config.CompletedConfig 结构
},
}

return cmd
}

// Config return a scheduler config object
func (o *Options) Config() (*schedulerappconfig.Config, error) {
c := &schedulerappconfig.Config{}
  // ApplyTo 函数完成 opt 参数或者configfile 变量到 schedulerappconfig.Config{} 对象
if err := o.ApplyTo(c); err != nil {
return nil, err
}

// prepare kube clients.
client, leaderElectionClient, eventClient, err := createClients(c.ComponentConfig.ClientConnection, o.Master, c.ComponentConfig.LeaderElection.RenewDeadline.Duration)

// Prepare event clients.
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, corev1.EventSource{Component: c.ComponentConfig.SchedulerName})

// Set up leader election if enabled.
var leaderElectionConfig *leaderelection.LeaderElectionConfig
if c.ComponentConfig.LeaderElection.LeaderElect {
leaderElectionConfig, err = makeLeaderElectionConfig(c.ComponentConfig.LeaderElection, leaderElectionClient, recorder)
}

c.Client = client
c.InformerFactory = informers.NewSharedInformerFactory(client, 0)
c.PodInformer = factory.NewPodInformer(client, 0)
c.EventClient = eventClient
c.Recorder = recorder
c.Broadcaster = eventBroadcaster
c.LeaderElection = leaderElectionConfig

return c, nil
}

 type completedConfig struct {
   *Config
 }
 
 // CompletedConfig same as Config, just to swap private object.
 type CompletedConfig struct {
   // Embed a private pointer that cannot be instantiated outside of this package.
   // 嵌入一个私有的指针，避免在包外初始化
   *completedConfig
 }

// Config has all the context to run a Scheduler
 type Config struct {
   // config is the scheduler server's configuration object.
   ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration
 
 // 认证相关的
   Authentication         apiserver.AuthenticationInfo
   Authorization          apiserver.AuthorizationInfo
 
   Client          clientset.Interface // k8s client
   InformerFactory informers.SharedInformerFactory  // infromer factory
   PodInformer     coreinformers.PodInformer        // pod informer
   EventClient     v1core.EventsGetter
   Recorder        record.EventRecorder
   Broadcaster     record.EventBroadcaster
 
   // LeaderElection is optional.
   LeaderElection *leaderelection.LeaderElectionConfig
 }

// KubeSchedulerConfiguration configures a scheduler
type KubeSchedulerConfiguration struct {
metav1.TypeMeta

// SchedulerName is name of the scheduler, used to select which pods
// will be processed by this scheduler, based on pod's "spec.SchedulerName".
  // 调度器的名字，用于用户选择不同的调度器
SchedulerName string
  
// AlgorithmSource specifies the scheduler algorithm source.
  // 指定算法的来源
AlgorithmSource SchedulerAlgorithmSource
  
// RequiredDuringScheduling affinity is not symmetric, but there is an implicit PreferredDuringScheduling affinity rule
// corresponding to every RequiredDuringScheduling affinity rule.
// HardPodAffinitySymmetricWeight represents the weight of implicit PreferredDuringScheduling affinity rule, in the range 0-100.
HardPodAffinitySymmetricWeight int32

// LeaderElection defines the configuration of leader election client.
LeaderElection KubeSchedulerLeaderElectionConfiguration

// ClientConnection specifies the kubeconfig file and client connection
// settings for the proxy server to use when communicating with the apiserver.
  
ClientConnection apimachineryconfig.ClientConnectionConfiguration
// HealthzBindAddress is the IP address and port for the health check server to serve on,
// defaulting to 0.0.0.0:10251
HealthzBindAddress string
  
// MetricsBindAddress is the IP address and port for the metrics server to
// serve on, defaulting to 0.0.0.0:10251.
MetricsBindAddress string

// DisablePreemption disables the pod preemption feature.
  // 是否禁止强制调度
DisablePreemption bool

  // 每次 Pod 调度的时候获取到整体 Node 的比例，函数 minFeasibleNodesToFind 负责处理，一般使用在
  // 大集群中, 低于 100 的总是返回全部 Node 进行匹配；如果 500 个 Node，设置 30，则搜索 500 * 30/100 = 150，每次搜索 150 个 Node 则停止
PercentageOfNodesToScore int32

BindTimeoutSeconds *int64
}

// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() CompletedConfig {
  cc := completedConfig{c}

  if c.InsecureServing != nil {
    c.InsecureServing.Name = "healthz"
  }
  if c.InsecureMetricsServing != nil {
    c.InsecureMetricsServing.Name = "metrics"
  }

  return CompletedConfig{&cc}
}

// NewSchedulerConfig creates the scheduler configuration. 
func NewSchedulerConfig(s schedulerserverconfig.CompletedConfig) (*scheduler.Config, error) {

// Set up the configurator which can create schedulers from configs.
  // 创建 configurator，用于生成各种需要的 config
  // 返回 configFactory， 文件定义在 k8s.io/kubernetes/pkg/scheduler/factory/factory.go
  //  NewConfigFactory() ->  初始化各种 informer，比较重要的是 PodInformer
  // unscheduled pod queue
  /*
args.PodInformer.Informer().AddEventHandler(
cache.FilteringResourceEventHandler{
FilterFunc: func(obj interface{}) bool { // 用于过滤 pod.Spec.NodeName 为空等条件 Pod
switch t := obj.(type) {
case *v1.Pod:
return unassignedNonTerminatedPod(t) && responsibleForPod(t, args.SchedulerName)
case cache.DeletedFinalStateUnknown:
if pod, ok := t.Obj.(*v1.Pod); ok {
return unassignedNonTerminatedPod(pod) && responsibleForPod(pod, args.SchedulerName)
}
runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c))
return false
default:
runtime.HandleError(fmt.Errorf("unable to handle object in %T: %T", c, obj))
return false
}
},
Handler: cache.ResourceEventHandlerFuncs{
AddFunc:    c.addPodToSchedulingQueue,
UpdateFunc: c.updatePodInSchedulingQueue,
DeleteFunc: c.deletePodFromSchedulingQueue,
},
},
)
*/
configurator := factory.NewConfigFactory(&factory.ConfigFactoryArgs{
SchedulerName:    s.ComponentConfig.SchedulerName,
Client:           s.Client,
NodeInformer:     s.InformerFactory.Core().V1().Nodes(),
PodInformer:      s.PodInformer,
PvInformer:       s.InformerFactory.Core().V1().PersistentVolumes(),
PvcInformer:      s.InformerFactory.Core().V1().PersistentVolumeClaims(),
ReplicationControllerInformer:  s.InformerFactory.Core().V1().ReplicationControllers(),
ReplicaSetInformer:      s.InformerFactory.Apps().V1().ReplicaSets(),
StatefulSetInformer:     s.InformerFactory.Apps().V1().StatefulSets(),
ServiceInformer:         s.InformerFactory.Core().V1().Services(),
PdbInformer:              s.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(),
StorageClassInformer:           storageClassInformer,
HardPodAffinitySymmetricWeight: s.ComponentConfig.HardPodAffinitySymmetricWeight,
EnableEquivalenceClassCache:    utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
DisablePreemption:              s.ComponentConfig.DisablePreemption,
PercentageOfNodesToScore:       s.ComponentConfig.PercentageOfNodesToScore,
BindTimeoutSeconds:             *s.ComponentConfig.BindTimeoutSeconds,
})

  // https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/
source := s.ComponentConfig.AlgorithmSource  // KubeSchedulerConfiguration`
var config *scheduler.Config
switch {
    // 创建调度算法有两种方式  Provider 和 用户指定的 Ploicy 文件（文件或Configmap)
  // --algorithm-provider string
  // DEPRECATED: the scheduling algorithm provider to use, one of: 
  // ClusterAutoscalerProvider | DefaultProvider
case source.Provider != nil:
// Create the config from a named algorithm provider.
    // 使用前面创建的 configurator 创建命名的算法 provider 配置
    // ClusterAutoscalerProvider | DefaultProvider
sc, err := configurator.CreateFromProvider(*source.Provider)
config = sc
  
  // --policy-config-file string
  //DEPRECATED: file with scheduler policy configuration. This file is used if policy ConfigMap is not provided or --use-legacy-policy-config=true

case source.Policy != nil:
// Create the config from a user specified policy source.
policy := &schedulerapi.Policy{}
switch {
case source.Policy.File != nil:
// Use a policy serialized in a file.
policyFile := source.Policy.File.Path
data, err := ioutil.ReadFile(policyFile)
      
err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy)

    // 从 configmap 初始化
case source.Policy.ConfigMap != nil:
   // 处理流程同上
    }

sc, err := configurator.CreateFromConfig(*policy)   
config = sc
default:
return nil, fmt.Errorf("unsupported algorithm source: %v", source)
}
// Additional tweaks to the config produced by the configurator.
config.Recorder = s.Recorder

config.DisablePreemption = s.ComponentConfig.DisablePreemption
return config, nil
}

// Creates a scheduler from a set of registered fit predicate keys and priority keys.
func (c *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String, extenders []algorithm.SchedulerExtender) (*scheduler.Config, error) {
  // ...

predicateFuncs, err := c.GetPredicates(predicateKeys)
priorityConfigs, err := c.GetPriorityFunctionConfigs(priorityKeys)
priorityMetaProducer, err := c.GetPriorityMetadataProducer()
predicateMetaProducer, err := c.GetPredicateMetadataProducer()


// Init equivalence class cache
if c.enableEquivalenceClassCache {
c.equivalencePodCache = equivalence.NewCache()
glog.Info("Created equivalence class cache")
}

  // 完成真正算法的初始化
algo := core.NewGenericScheduler(
c.schedulerCache,
c.equivalencePodCache,
c.podQueue,
predicateFuncs,
predicateMetaProducer,
priorityConfigs,
priorityMetaProducer,
extenders,
c.volumeBinder,
c.pVCLister,
c.alwaysCheckAllPredicates,
c.disablePreemption,
c.percentageOfNodesToScore,
)

podBackoff := util.CreateDefaultPodBackoff()
return &scheduler.Config{
SchedulerCache: c.schedulerCache,
Ecache:         c.equivalencePodCache,
// The scheduler only needs to consider schedulable nodes.
NodeLister:          &nodeLister{c.nodeLister},
Algorithm:           algo, // 设置调度的算法
GetBinder:           c.getBinderFunc(extenders),
PodConditionUpdater: &podConditionUpdater{c.client},
PodPreemptor:        &podPreemptor{c.client},
WaitForCacheSync: func() bool {
return cache.WaitForCacheSync(c.StopEverything, c.scheduledPodsHasSynced)
},
NextPod: func() *v1.Pod { // 设定 NextPod d的
return c.getNextPod()
},
Error:           c.MakeDefaultErrorFunc(podBackoff, c.podQueue),
StopEverything:  c.StopEverything,
VolumeBinder:    c.volumeBinder,
SchedulingQueue: c.podQueue,
}, nil
}

// scheduleOne does the entire scheduling workflow for a single pod.  It is serialized on the scheduling algorithm's host fitting.
func (sched *Scheduler) scheduleOne() {
  // 从队列中取出待调度的 Pod，最终调用的函数为 configFactory.getNextPod()
pod := sched.config.NextPod()
  
  // 检查是否是被删除的，如果是则跳过
  
// Synchronously attempt to find a fit for the pod.
start := time.Now()
  
  // 获取待调度 Pod 匹配的主机名
suggestedHost, err := sched.schedule(pod)  // 见后续分析
if err != nil {
// schedule() 可能因为没有满足调度的 Node 而失败，后续进行 preempt,
if fitError, ok := err.(*core.FitError); ok {
preemptionStartTime := time.Now()
sched.preempt(pod, fitError)
}
return
}
// Tell the cache to assume that a pod now is running on a given node, even though it hasn't been bound yet.
// This allows us to keep scheduling without waiting on binding to occur.
assumedPod := pod.DeepCopy()

  // 判断是否需要VolumeScheduling特性
// Assume volumes first before assuming the pod. If all volumes are completely bound, then allBound is true and binding will be skipped. Otherwise, binding of volumes is started after the pod is assumed, but before pod binding. This function modifies 'assumedPod' if volume binding is required.
allBound, err := sched.assumeVolumes(assumedPod, suggestedHost)
if err != nil {
return
}

// assume modifies `assumedPod` by setting NodeName=suggestedHost
  // Pod 对应的 NodeName 写上主机名，存入缓存
err = sched.assume(assumedPod, suggestedHost)

// bind the pod to its host asynchronously (we can do this b/c of the assumption step above).
  // 请求apiserver，异步处理最终的绑定，写入到etcd
go func() {
// Bind volumes first before Pod
if !allBound {
err = sched.bindVolumes(assumedPod)
}

err := sched.bind(assumedPod, &v1.Binding{
ObjectMeta: metav1.ObjectMeta{Namespace: assumedPod.Namespace, Name: assumedPod.Name, UID: assumedPod.UID},
Target: v1.ObjectReference{
Kind: "Node",
Name: suggestedHost,
},
})
metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
}()
}

// schedule implements the scheduling algorithm and returns the suggested host.
func (sched *Scheduler) schedule(pod *v1.Pod) (string, error) {
  // 调用算法提供的调度器进行调度，为 GenericScheduler
host, err := sched.config.Algorithm.Schedule(pod, sched.config.NodeLister)
  // ...
}

// sched.config.Algorithm = core.NewGenericScheduler()
// k8s.io/kubernetes/pkg/scheduler/factory/factory.go
// 初始化过程参见 func (c *configFactory) CreateFromKeys(){...}
// 
// k8s.io/kubernetes/pkg/scheduler/core/generic_scheduler.go
// NewGenericScheduler creates a genericScheduler object.
func NewGenericScheduler(
cache schedulercache.Cache,
eCache *equivalence.Cache,
podQueue SchedulingQueue,
predicates map[string]algorithm.FitPredicate,
predicateMetaProducer algorithm.PredicateMetadataProducer,
prioritizers []algorithm.PriorityConfig,
priorityMetaProducer algorithm.PriorityMetadataProducer,
extenders []algorithm.SchedulerExtender,
volumeBinder *volumebinder.VolumeBinder,
pvcLister corelisters.PersistentVolumeClaimLister,
alwaysCheckAllPredicates bool,
disablePreemption bool,
percentageOfNodesToScore int32,
) algorithm.ScheduleAlgorithm {
return &genericScheduler{
cache:                    cache,
equivalenceCache:         eCache,
schedulingQueue:          podQueue,
predicates:               predicates,
predicateMetaProducer:    predicateMetaProducer,
prioritizers:             prioritizers,
priorityMetaProducer:     priorityMetaProducer,
extenders:                extenders,
cachedNodeInfoMap:        make(map[string]*schedulercache.NodeInfo),
volumeBinder:             volumeBinder,
pvcLister:                pvcLister,
alwaysCheckAllPredicates: alwaysCheckAllPredicates,
disablePreemption:        disablePreemption,
percentageOfNodesToScore: percentageOfNodesToScore,
}
}

// Schedule tries to schedule the given pod to one of the nodes in the node list.
// If it succeeds, it will return the name of the node.
// If it fails, it will return a FitError error with reasons.
func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister) (string, error) {

nodes, err := nodeLister.List()

// Used for all fit and priority funcs.
err = g.cache.UpdateNodeNameToInfoMap(g.cachedNodeInfoMap)

  // 1. 预选机器列表，获取到 filteredNodes 列表
  // 1.1 如果没有配置预选算法，则直接返回全部 Node 列表
  // 1.2 如果配置了预选算法，则对多个 Node 调用 checkNode 的方法，检查 Pod 是否可以调度到该 Node
  // 1.3 预选筛选之后，如果有扩展算法，则继续匹配筛选，返回最终匹配的 Node 列表
filteredNodes, failedPredicateMap, err := g.findNodesThatFit(pod, nodes)


  // 2. 从预选的机器列表中获取优选机器列表
metaPrioritiesInterface := g.priorityMetaProducer(pod, g.cachedNodeInfoMap)
  
priorityList, err := PrioritizeNodes(pod, g.cachedNodeInfoMap, metaPrioritiesInterface, g.prioritizers, filteredNodes, g.extenders)

  // 选择 Scores 最高的并返回
return g.selectHost(priorityList)
}

// Filters the nodes to find the ones that fit based on the given predicate functions
// Each node is passed through the predicate functions to determine if it is a fit
func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v1.Node, FailedPredicateMap, error) {
var filtered []*v1.Node
failedPredicateMap := FailedPredicateMap{}

  // 如果没有预选算法，则返回全部机器列表
if len(g.predicates) == 0 {
filtered = nodes
} else {
allNodes := int32(g.cache.NodeTree().NumNodes)
    // 保证一次性不用返回过多的Node数量，避免数组过大，当前有个硬编码 100， 100 以内一般全部返回，否则
    // 按照 percentageOfNodesToScore 比例设置每次返回集群中多少个节点返回，以提升调度性能
    // percentageOfNodesToScore 为 0或者大于100，则返回集群全部 Node
numNodesToFind := g.numFeasibleNodesToFind(allNodes)

// Create filtered list with enough space to avoid growing it
// and allow assigning.
filtered = make([]*v1.Node, numNodesToFind)

    // 用于检测 Pod 是否可以调度到 Node 的检查函数
checkNode := func(i int) {
var nodeCache *equivalence.NodeCache
nodeName := g.cache.NodeTree().Next()
      
fits, failedPredicates, err := podFitsOnNode(
pod,
meta,
g.cachedNodeInfoMap[nodeName],
g.predicates,
g.cache,
nodeCache,
g.schedulingQueue,
g.alwaysCheckAllPredicates,
equivClass,
)

if fits {
length := atomic.AddInt32(&filteredLen, 1)
if length > numNodesToFind {
cancel()
atomic.AddInt32(&filteredLen, -1)
} else {
filtered[length-1] = g.cachedNodeInfoMap[nodeName].Node()
}
} 
      
      // ...
}

// Stops searching for more nodes once the configured number of feasible nodes
// are found.
workqueue.ParallelizeUntil(ctx, 16, int(allNodes), checkNode)

    // 如果配置了 extender 算法，则使用扩展算法继续过滤
if len(filtered) > 0 && len(g.extenders) != 0 {
for _, extender := range g.extenders {
filteredList, failedMap, err := extender.Filter(pod, filtered, g.cachedNodeInfoMap)

// ...
filtered = filteredList
}
}
return filtered, failedPredicateMap, nil
}

// podFitsOnNode checks whether a node given by NodeInfo satisfies the given predicate functions.
// For given pod, podFitsOnNode will check if any equivalent pod exists and try to reuse its cached
// predicate results as possible.
// This function is called from two different places: Schedule and Preempt.
// When it is called from Schedule, we want to test whether the pod is schedulable
// on the node with all the existing pods on the node plus higher and equal priority
// pods nominated to run on the node.
// When it is called from Preempt, we should remove the victims of preemption and
// add the nominated pods. Removal of the victims is done by SelectVictimsOnNode().
// It removes victims from meta and NodeInfo before calling this function.
func podFitsOnNode(
pod *v1.Pod,
meta algorithm.PredicateMetadata,
info *schedulercache.NodeInfo,
predicateFuncs map[string]algorithm.FitPredicate,
cache schedulercache.Cache,
nodeCache *equivalence.NodeCache,
queue SchedulingQueue,
alwaysCheckAllPredicates bool,
equivClass *equivalence.Class,
) (bool, []algorithm.PredicateFailureReason, error) {
var (
eCacheAvailable  bool
failedPredicates []algorithm.PredicateFailureReason
)

podsAdded := false
// We run predicates twice in some cases. If the node has greater or equal priority
// nominated pods, we run them when those pods are added to meta and nodeInfo.
// If all predicates succeed in this pass, we run them again when these
// nominated pods are not added. This second pass is necessary because some
// predicates such as inter-pod affinity may not pass without the nominated pods.
// If there are no nominated pods for the node or if the first run of the
// predicates fail, we don't run the second pass.
// We consider only equal or higher priority pods in the first pass, because
// those are the current "pod" must yield to them and not take a space opened
// for running them. It is ok if the current "pod" take resources freed for
// lower priority pods.
// Requiring that the new pod is schedulable in both circumstances ensures that
// we are making a conservative decision: predicates like resources and inter-pod
// anti-affinity are more likely to fail when the nominated pods are treated
// as running, while predicates like pod affinity are more likely to fail when
// the nominated pods are treated as not running. We can't just assume the
// nominated pods are running because they are not running right now and in fact,
// they may end up getting scheduled to a different node.
for i := 0; i < 2; i++ {
metaToUse := meta
nodeInfoToUse := info
    // 第一次调度，根据NominatedPods更新meta和nodeInfo信息，pod根据更新后的信息去预选
// 第二次调度，meta和nodeInfo信息不变，保证pod不完全依赖于NominatedPods（主要考虑到pod亲和性之类的）
if i == 0 {
podsAdded, metaToUse, nodeInfoToUse = addNominatedPods(pod, meta, info, queue)
} else if !podsAdded || len(failedPredicates) != 0 {
break
}
// Bypass eCache if node has any nominated pods.
// TODO(bsalamat): consider using eCache and adding proper eCache invalidations
// when pods are nominated or their nominations change.
eCacheAvailable = equivClass != nil && nodeCache != nil && !podsAdded
for _, predicateKey := range predicates.Ordering() {
var (
fit     bool
reasons []algorithm.PredicateFailureReason
err     error
)

if predicate, exist := predicateFuncs[predicateKey]; exist {
if eCacheAvailable {
fit, reasons, err = nodeCache.RunPredicate(predicate, predicateKey, pod, metaToUse, nodeInfoToUse, equivClass, cache)
} else {
fit, reasons, err = predicate(pod, metaToUse, nodeInfoToUse)
}


if !fit {
// eCache is available and valid, and predicates result is unfit, record the fail reasons
failedPredicates = append(failedPredicates, reasons...)
// if alwaysCheckAllPredicates is false, short circuit all predicates when one predicate fails.

}
}
}
}

return len(failedPredicates) == 0, failedPredicates, nil
}

// PrioritizeNodes prioritizes the nodes by running the individual priority functions in parallel.
// Each priority function is expected to set a score of 0-10
// 0 is the lowest priority score (least preferred node) and 10 is the highest
// Each priority function can also have its own weight
// The node scores returned by the priority function are multiplied by the weights to get weighted scores
// All scores are finally combined (added) to get the total weighted scores of all nodes
func PrioritizeNodes(
pod *v1.Pod,
nodeNameToInfo map[string]*schedulercache.NodeInfo,
meta interface{},
priorityConfigs []algorithm.PriorityConfig,
nodes []*v1.Node,
extenders []algorithm.SchedulerExtender,
) (schedulerapi.HostPriorityList, error) {
// If no priority configs are provided, then the EqualPriority function is applied
// This is required to generate the priority list in the required format
if len(priorityConfigs) == 0 && len(extenders) == 0 {
result := make(schedulerapi.HostPriorityList, 0, len(nodes))
for i := range nodes {
hostPriority, err := EqualPriorityMap(pod, meta, nodeNameToInfo[nodes[i].Name])
if err != nil {
return nil, err
}
result = append(result, hostPriority)
}
return result, nil
}

var (
mu   = sync.Mutex{}
wg   = sync.WaitGroup{}
errs []error
)

results := make([]schedulerapi.HostPriorityList, len(priorityConfigs), len(priorityConfigs))

for i, priorityConfig := range priorityConfigs {
if priorityConfig.Function != nil {
wg.Add(1)
go func(index int, config algorithm.PriorityConfig) {
defer wg.Done()
var err error
results[index], err = config.Function(pod, nodeNameToInfo, nodes)
if err != nil {
appendError(err)
}
}(i, priorityConfig)
} else {
results[i] = make(schedulerapi.HostPriorityList, len(nodes))
}
}

processNode := func(index int) {
nodeInfo := nodeNameToInfo[nodes[index].Name]
var err error
for i := range priorityConfigs {
      // 使用 map 函数计算过程数据
results[i][index], err = priorityConfigs[i].Map(pod, meta, nodeInfo)
}
}
workqueue.Parallelize(16, len(nodes), processNode)
for i, priorityConfig := range priorityConfigs {
wg.Add(1)
go func(index int, config algorithm.PriorityConfig) {
      // 使用 reduce 函数结算结果
if err := config.Reduce(pod, meta, nodeNameToInfo, results[index]); err != nil {
appendError(err)
}
}(i, priorityConfig)
}
// Wait for all computations to be finished.
wg.Wait()
// Summarize all scores.
result := make(schedulerapi.HostPriorityList, 0, len(nodes))

for i := range nodes {
result = append(result, schedulerapi.HostPriority{Host: nodes[i].Name, Score: 0})
for j := range priorityConfigs {
result[i].Score += results[j][i].Score * priorityConfigs[j].Weight
}
}

  // 继续使用 extenders 来进行优选 
if len(extenders) != 0 && nodes != nil {
combinedScores := make(map[string]int, len(nodeNameToInfo))
for _, extender := range extenders {
wg.Add(1)
go func(ext algorithm.SchedulerExtender) {
defer wg.Done()
prioritizedList, weight, err := ext.Prioritize(pod, nodes)
if err != nil {
// Prioritization errors from extender can be ignored, let k8s/other extenders determine the priorities
return
}
mu.Lock()
for i := range *prioritizedList {
host, score := (*prioritizedList)[i].Host, (*prioritizedList)[i].Score
combinedScores[host] += score * weight
}
mu.Unlock()
}(extender)
}
// wait for all go routines to finish
wg.Wait()
for i := range result {
result[i].Score += combinedScores[result[i].Host]
}
}
  
return result, nil
}

package algorithmprovider

import (
"k8s.io/kubernetes/pkg/scheduler/algorithmprovider/defaults"
)

// ApplyFeatureGates applies algorithm by feature gates.
func ApplyFeatureGates() {
defaults.ApplyFeatureGates()
}

func init() {
// Register functions that extract metadata used by predicates and priorities computations.
factory.RegisterPredicateMetadataProducerFactory(
func(args factory.PluginFactoryArgs) algorithm.PredicateMetadataProducer {
return predicates.NewPredicateMetadataFactory(args.PodLister)
})
  
factory.RegisterPriorityMetadataProducerFactory(
func(args factory.PluginFactoryArgs) algorithm.PriorityMetadataProducer {
return priorities.NewPriorityMetadataFactory(args.ServiceLister, args.ControllerLister, args.ReplicaSetLister, args.StatefulSetLister)
})

  // 注册默认的预选算法和优选算法
registerAlgorithmProvider(defaultPredicates(), defaultPriorities())

// IMPORTANT NOTES for predicate developers:
// We are using cached predicate result for pods belonging to the same equivalence class.
// So when implementing a new predicate, you are expected to check whether the result
// of your predicate function can be affected by related API object change (ADD/DELETE/UPDATE).
// If yes, you are expected to invalidate the cached predicate result for related API object change.
// For example:
// https://github.com/kubernetes/kubernetes/blob/36a218e/plugin/pkg/scheduler/factory/factory.go#L422

// Registers predicates and priorities that are not enabled by default, but user can pick when creating their
// own set of priorities/predicates.

// PodFitsPorts has been replaced by PodFitsHostPorts for better user understanding.
// For backwards compatibility with 1.0, PodFitsPorts is registered as well.
factory.RegisterFitPredicate("PodFitsPorts", predicates.PodFitsHostPorts)
// Fit is defined based on the absence of port conflicts.
// This predicate is actually a default predicate, because it is invoked from
// predicates.GeneralPredicates()
factory.RegisterFitPredicate(predicates.PodFitsHostPortsPred, predicates.PodFitsHostPorts)
// Fit is determined by resource availability.
// This predicate is actually a default predicate, because it is invoked from
// predicates.GeneralPredicates()
factory.RegisterFitPredicate(predicates.PodFitsResourcesPred, predicates.PodFitsResources)
// Fit is determined by the presence of the Host parameter and a string match
// This predicate is actually a default predicate, because it is invoked from
// predicates.GeneralPredicates()
factory.RegisterFitPredicate(predicates.HostNamePred, predicates.PodFitsHost)
// Fit is determined by node selector query.
factory.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector)

// ServiceSpreadingPriority is a priority config factory that spreads pods by minimizing
// the number of pods (belonging to the same service) on the same node.
// Register the factory so that it's available, but do not include it as part of the default priorities
// Largely replaced by "SelectorSpreadPriority", but registered for backward compatibility with 1.0
factory.RegisterPriorityConfigFactory(
"ServiceSpreadingPriority",
factory.PriorityConfigFactory{
MapReduceFunction: func(args factory.PluginFactoryArgs) (algorithm.PriorityMapFunction, algorithm.PriorityReduceFunction) {
return priorities.NewSelectorSpreadPriority(args.ServiceLister, algorithm.EmptyControllerLister{}, algorithm.EmptyReplicaSetLister{}, algorithm.EmptyStatefulSetLister{})
},
Weight: 1,
},
)
// EqualPriority is a prioritizer function that gives an equal weight of one to all nodes
// Register the priority function so that its available
// but do not include it as part of the default priorities
factory.RegisterPriorityFunction2("EqualPriority", core.EqualPriorityMap, nil, 1)
// Optional, cluster-autoscaler friendly priority function - give used nodes higher priority.
factory.RegisterPriorityFunction2("MostRequestedPriority", priorities.MostRequestedPriorityMap, nil, 1)
factory.RegisterPriorityFunction2(
"RequestedToCapacityRatioPriority",
priorities.RequestedToCapacityRatioResourceAllocationPriorityDefault().PriorityMap,
nil,
1)
}

// IMPORTANT NOTE: this list contains the ordering of the predicates, if you develop a new predicate
// it is mandatory to add its name to this list.
// Otherwise it won't be processed, see generic_scheduler#podFitsOnNode().
// The order is based on the restrictiveness & complexity of predicates.
// Design doc: https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/predicates-ordering.md

// 预选算法的顺序
var (
predicatesOrdering = []string{CheckNodeConditionPred, CheckNodeUnschedulablePred,
GeneralPred, HostNamePred, PodFitsHostPortsPred,
MatchNodeSelectorPred, PodFitsResourcesPred, NoDiskConflictPred,
PodToleratesNodeTaintsPred, PodToleratesNodeNoExecuteTaintsPred, CheckNodeLabelPresencePred,
CheckServiceAffinityPred, MaxEBSVolumeCountPred, MaxGCEPDVolumeCountPred, MaxCSIVolumeCountPred,
MaxAzureDiskVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred,
CheckNodeMemoryPressurePred, CheckNodePIDPressurePred, CheckNodeDiskPressurePred, MatchInterPodAffinityPred}
)

func defaultPredicates() sets.String {
return sets.NewString(
// ...
    
    // Fit is determined by node conditions: not ready, 
    // network unavailable or out of disk.
factory.RegisterMandatoryFitPredicate(predicates.CheckNodeConditionPred, predicates.CheckNodeConditionPredicate), // 见下文函数分析

// ...
}