Kubernetes 系列 - 5. apiserver（二、创建处理链）5. apiserver（二、创建处理链） 5

5. apiserver（二、创建处理链）

5.1 主流程

// Run runs the specified APIServer.  This should never exit.
func Run(opts options.CompletedOptions, stopCh <-chan struct{}) error {
    // To help debugging, immediately log version
    klog.Infof("Version: %+v", version.Get())

    klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK"))

    config, err := NewConfig(opts)
    if err != nil {
       return err
    }
    completed, err := config.Complete()
    if err != nil {
       return err
    }
    server, err := CreateServerChain(completed)
    if err != nil {
       return err
    }

    prepared, err := server.PrepareRun()
    if err != nil {
       return err
    }

    return prepared.Run(stopCh)
}

apiserver 主要由三个server组成：

aggregatorServer: 处理请求的第一部分，可以接入自定义的apiserver，其他请求转到kubeApiServer；
kubeApiServer: 处理请求的主要部分，通用资源的请求基本都在这里处理，其他请求转到apiExtensionsServer；
apiExtensionsServer: 处理自定义资源的请求，无法处理的请求返回404；

aggregatorServer -> kubeApiServer -> apiExtensionsServer -> notFound
      |
      v
customApiServer

5.2 创建配置

// NewConfig creates all the resources for running kube-apiserver, but runs none of them.
func NewConfig(opts options.CompletedOptions) (*Config, error) {
    c := &Config{
       Options: opts,
    }

    // 1. 创建 apiserverConfig
    controlPlane, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts)
    if err != nil {
       return nil, err
    }
    c.ControlPlane = controlPlane

    // 2. 创建 apiExtensionsConfig
    apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount,
       serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(controlPlane.ExtraConfig.ProxyTransport, controlPlane.GenericConfig.EgressSelector, controlPlane.GenericConfig.LoopbackClientConfig, controlPlane.GenericConfig.TracerProvider))
    if err != nil {
       return nil, err
    }
    c.ApiExtensions = apiExtensions

    // 3. 创建 aggregatorConfig
    aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.CompletedOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, controlPlane.ExtraConfig.PeerProxy, pluginInitializer)
    if err != nil {
       return nil, err
    }
    c.Aggregator = aggregator

    return c, nil
}

这里主要依托GenericConfig创建三个server的config。

5.3 创建处理链

// CreateServerChain creates the apiservers connected via delegation.
func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregator, error) {
    // 1. 最后处理handler，返回404
    notFoundHandler := notfoundhandler.New(config.ControlPlane.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey)
    
    // 2. 创建apiExtensionsServer
    apiExtensionsServer, err := config.ApiExtensions.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler))
    if err != nil {
       return nil, err
    }
    crdAPIEnabled := config.ApiExtensions.GenericConfig.MergedResourceConfig.ResourceEnabled(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions"))

    // 3. 创建kubeApiServer
    kubeAPIServer, err := config.ControlPlane.New(apiExtensionsServer.GenericAPIServer)
    if err != nil {
       return nil, err
    }

    // 4. 创建aggregatorServer
    // aggregator comes last in the chain
    aggregatorServer, err := createAggregatorServer(config.Aggregator, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers, crdAPIEnabled)
    if err != nil {
       // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
       return nil, err
    }

    return aggregatorServer, nil
}

5.3.1 创建apiExtensionsServer

5.3.1.1 创建genericServer

func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
    ...
    
    // 1. 创建handlerChainBuilder，用于创建handlerChain
    handlerChainBuilder := func(handler http.Handler) http.Handler {
       return c.BuildHandlerChainFunc(handler, c.Config)
    }
    
    // 2. 重点：创建handler
    apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())

    // 3. 创建genericServer，大部分属性都是从completeConfig拷贝过来的
    s := &GenericAPIServer{
       discoveryAddresses:             c.DiscoveryAddresses,
       LoopbackClientConfig:           c.LoopbackClientConfig,
       legacyAPIGroupPrefixes:         c.LegacyAPIGroupPrefixes,
       admissionControl:               c.AdmissionControl,
       Serializer:                     c.Serializer,
       AuditBackend:                   c.AuditBackend,
       Authorizer:                     c.Authorization.Authorizer,
       delegationTarget:               delegationTarget,
       EquivalentResourceRegistry:     c.EquivalentResourceRegistry,
       NonLongRunningRequestWaitGroup: c.NonLongRunningRequestWaitGroup,
       WatchRequestWaitGroup:          c.WatchRequestWaitGroup,
       Handler:                        apiServerHandler,
       UnprotectedDebugSocket:         debugSocket,

       listedPathProvider: apiServerHandler,

       minRequestTimeout:                   time.Duration(c.MinRequestTimeout) * time.Second,
       ShutdownTimeout:                     c.RequestTimeout,
       ShutdownDelayDuration:               c.ShutdownDelayDuration,
       ShutdownWatchTerminationGracePeriod: c.ShutdownWatchTerminationGracePeriod,
       SecureServingInfo:                   c.SecureServing,
       ExternalAddress:                     c.ExternalAddress,

       openAPIConfig:           c.OpenAPIConfig,
       openAPIV3Config:         c.OpenAPIV3Config,
       skipOpenAPIInstallation: c.SkipOpenAPIInstallation,

       postStartHooks:         map[string]postStartHookEntry{},
       preShutdownHooks:       map[string]preShutdownHookEntry{},
       disabledPostStartHooks: c.DisabledPostStartHooks,

       healthzRegistry:  healthCheckRegistry{path: "/healthz", checks: c.HealthzChecks},
       livezRegistry:    healthCheckRegistry{path: "/livez", checks: c.LivezChecks, clock: clock.RealClock{}},
       readyzRegistry:   healthCheckRegistry{path: "/readyz", checks: c.ReadyzChecks},
       livezGracePeriod: c.LivezGracePeriod,

       DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),

       maxRequestBodyBytes: c.MaxRequestBodyBytes,

       lifecycleSignals:       c.lifecycleSignals,
       ShutdownSendRetryAfter: c.ShutdownSendRetryAfter,

       APIServerID:           c.APIServerID,
       StorageVersionManager: c.StorageVersionManager,

       Version: c.Version,

       muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
    }

    ...
    
    // 4. 拷贝delegationTarget的hook，在最后启动aggregateServer时可以启动delegationTarget
    // first add poststarthooks from delegated targets
    for k, v := range delegationTarget.PostStartHooks() {
       s.postStartHooks[k] = v
    }
    for k, v := range delegationTarget.PreShutdownHooks() {
       s.preShutdownHooks[k] = v
    }

    // 5. 添加配置的hook，感觉这里会因为拷贝了delegationTarget的hook导致重复注册导致error呢？(其实没有，各个server的config构造的时候虽然使用了apiserver的genericConfig，但清空了postStartHooks，所以并没有重复注册)
    // add poststarthooks that were preconfigured.  Using the add method will give us an error if the same name has already been registered.
    for name, preconfiguredPostStartHook := range c.PostStartHooks {
       if err := s.AddPostStartHook(name, preconfiguredPostStartHook.hook); err != nil {
          return nil, err
       }
    }

   ...

    return s, nil
}

这里重点看一下创建handler的步骤：

func NewAPIServerHandler(name string, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
    // 1. 配置notFoundHandler
    nonGoRestfulMux := mux.NewPathRecorderMux(name)
    if notFoundHandler != nil {
       nonGoRestfulMux.NotFoundHandler(notFoundHandler)
    }

    // 2. 配置gorestfulContainer
    gorestfulContainer := restful.NewContainer()
    gorestfulContainer.ServeMux = http.NewServeMux()
    gorestfulContainer.Router(restful.CurlyRouter{}) // e.g. for proxy/{kind}/{name}/{*}
    gorestfulContainer.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
       logStackOnRecover(s, panicReason, httpWriter)
    })
    gorestfulContainer.ServiceErrorHandler(func(serviceErr restful.ServiceError, request *restful.Request, response *restful.Response) {
       serviceErrorHandler(s, serviceErr, request, response)
    })

    // 3. 配置未经装饰的handler，当前组件无法处理的请求转给下一个组件处理时不需要再进行校验等工作
    director := director{
       name:               name,
       goRestfulContainer: gorestfulContainer,
       nonGoRestfulMux:    nonGoRestfulMux,
    }

    // 4. handlerChainBuilder使用装饰器模式配置handler，并生成apiServerHandler
    return &APIServerHandler{
       FullHandlerChain:   handlerChainBuilder(director),
       GoRestfulContainer: gorestfulContainer,
       NonGoRestfulMux:    nonGoRestfulMux,
       Director:           director,
    }
}

handlerChainBuilder的逻辑如下：

func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
    handler := apiHandler

    handler = filterlatency.TrackCompleted(handler)
    handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
    handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
    
    ...
    
    return handler
}

这里就做了一件事：利用装饰器模式在正式使用director处理请求之前做一些校验等操作。

而director只实现了一个方法：ServeHTTP，将director做成一个handler：

func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    path := req.URL.Path

    // check to see if our webservices want to claim this path
    for _, ws := range d.goRestfulContainer.RegisteredWebServices() {
       switch {
       case ws.RootPath() == "/apis":
          // if we are exactly /apis or /apis/, then we need special handling in loop.
          // normally these are passed to the nonGoRestfulMux, but if discovery is enabled, it will go directly.
          // We can't rely on a prefix match since /apis matches everything (see the big comment on Director above)
          if path == "/apis" || path == "/apis/" {
             klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
             // don't use servemux here because gorestful servemuxes get messed up when removing webservices
             // TODO fix gorestful, remove TPRs, or stop using gorestful
             d.goRestfulContainer.Dispatch(w, req)
             return
          }

       case strings.HasPrefix(path, ws.RootPath()):
          // ensure an exact match or a path boundary match
          if len(path) == len(ws.RootPath()) || path[len(ws.RootPath())] == '/' {
             klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
             // don't use servemux here because gorestful servemuxes get messed up when removing webservices
             // TODO fix gorestful, remove TPRs, or stop using gorestful
             d.goRestfulContainer.Dispatch(w, req)
             return
          }
       }
    }

    // if we didn't find a match, then we just skip gorestful altogether
    klog.V(5).Infof("%v: %v %q satisfied by nonGoRestful", d.name, req.Method, path)
    d.nonGoRestfulMux.ServeHTTP(w, req)
}

// d.nonGoRestfulMux.ServeHTTP(w, req)会转到这儿
// ServeHTTP makes it an http.Handler
func (h *pathHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
    if exactHandler, ok := h.pathToHandler[r.URL.Path]; ok {
       klog.V(5).Infof("%v: %q satisfied by exact match", h.muxName, r.URL.Path)
       exactHandler.ServeHTTP(w, r)
       return
    }

    for _, prefixHandler := range h.prefixHandlers {
       if strings.HasPrefix(r.URL.Path, prefixHandler.prefix) {
          klog.V(5).Infof("%v: %q satisfied by prefix %v", h.muxName, r.URL.Path, prefixHandler.prefix)
          prefixHandler.handler.ServeHTTP(w, r)
          return
       }
    }

    klog.V(5).Infof("%v: %q satisfied by NotFoundHandler", h.muxName, r.URL.Path)
    h.notFoundHandler.ServeHTTP(w, r)
}

director先用goRestfulContainer判断是否处理，再尝试用nonGoRestfulMux处理，最后无法处理的用notFoundHandler进行处理，也就是说director前面做校验等操作的时候并没有判断是否处理，而是在director里面判断是否由当前的组件处理请求，如果处理不了，就转给下一个组件进行处理，为了避免重复做校验这些操作，所以要保留直接处理请求的director，传给上一个组件作为notFoundHandler。

5.3.2 创建kubeApiServer

5.3.3 创建apiExtensionServer

5.3.3.1 创建aggregatorServer

// NewWithDelegate returns a new instance of APIAggregator from the given config.
func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.DelegationTarget) (*APIAggregator, error) {
    // 1. 创建genericServer
    genericServer, err := c.GenericConfig.New("kube-aggregator", delegationTarget)
    if err != nil {
       return nil, err
    }

    ...

    // 2. 创建一个通道 apiServiceRegistrationControllerInitiated，用于通知其他组件 API 服务注册控制器已经完成初始化。
    apiServiceRegistrationControllerInitiated := make(chan struct{})
    if err := genericServer.RegisterMuxAndDiscoveryCompleteSignal("APIServiceRegistrationControllerInitiated", apiServiceRegistrationControllerInitiated); err != nil {
       return nil, err
    }

    ...

    // 3. 创建APIAggregator实例
    s := &APIAggregator{
       GenericAPIServer:           genericServer,
       delegateHandler:            delegationTarget.UnprotectedHandler(),
       proxyTransportDial:         proxyTransportDial,
       proxyHandlers:              map[string]*proxyHandler{},
       handledGroupVersions:       map[string]sets.Set[string]{},
       lister:                     informerFactory.Apiregistration().V1().APIServices().Lister(),
       APIRegistrationInformers:   informerFactory,
       serviceResolver:            c.ExtraConfig.ServiceResolver,
       openAPIConfig:              c.GenericConfig.OpenAPIConfig,
       openAPIV3Config:            c.GenericConfig.OpenAPIV3Config,
       proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil },
       rejectForwardingRedirects:  c.ExtraConfig.RejectForwardingRedirects,
    }

    ...

    // 4. install api
    apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter, resourceExpirationEvaluator.ShouldServeForVersion(1, 22))
    if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
       return nil, err
    }

    ...

    // 5. 设置可用性控制器
    availableController, err := statuscontrollers.NewAvailableConditionController(
       informerFactory.Apiregistration().V1().APIServices(),
       c.GenericConfig.SharedInformerFactory.Core().V1().Services(),
       c.GenericConfig.SharedInformerFactory.Core().V1().Endpoints(),
       apiregistrationClient.ApiregistrationV1(),
       proxyTransportDial,
       (func() ([]byte, []byte))(s.proxyCurrentCertKeyContent),
       s.serviceResolver,
    )
    if err != nil {
       return nil, err
    }

    // 6. 注册postStartHook
    s.GenericAPIServer.AddPostStartHookOrDie("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error {
       ...
    })
    s.GenericAPIServer.AddPostStartHookOrDie("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error {
       ...
    })
    s.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-available-controller", func(context genericapiserver.PostStartHookContext) error {
       ...
    })

    ...

    return s, nil
}