Kubernetes ApiServer三大server許可權與資料儲存解析

// 類似kubectl的原始碼，kube-apiserver的命令列工具也使用了cobra，我們很快就能找到啟動的入口
RunE: func(cmd *cobra.Command, args []string) error {
			// 這裡包含2個引數，前者是引數completedOptions，後者是一個stopCh <-chan struct{}
			return Run(completedOptions, genericapiserver.SetupSignalHandler())
		}
/*
	在這裡，我們可以和kubectl結合起來思考：
	kubectl是一個命令列工具，執行完命令就退出；kube-apiserver是一個常駐的伺服器程序，監聽埠
	這裡引入了一個stopCh <-chan struct{}，可以在啟動後，用一個 <-stopCh 作為阻塞，使程式不退出
	用channel阻塞程序退出，對比傳統的方法 - 用一個永不退出的for迴圈，是一個很優雅的實現
*/
func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error {
        // 這裡進行建立服務鏈
        server, err := CreateServerChain(completeOptions)
	if err != nil {
		return err
	}
	prepared, err := server.PrepareRun()
	if err != nil {
		return err
	}
	return prepared.Run(stopCh)
}

// 在CreateServerChain這個函數下，建立了3個server 
func CreateServerChain(){
  // API擴充套件服務，主要針對CRD
	createAPIExtensionsServer(){} 
  // API核心服務，包括常見的Pod/Deployment/Service，我們今天的重點聚焦在這裡
  // 我會跳過很多非核心的設定引數，一開始就去研究細節，很影響整體程式碼的閱讀效率
	CreateKubeAPIServer(){} 
  // API聚合服務，主要針對metrics
	createAggregatorServer(){} 
  //細節是第二個ApiServer需要第一個server的設定，第三個server會要第二個server的設定，最後返回的是聚合server
  // 這些server的config都是由一個GenericConfig和一個ExtraConfig組成 有自己的特點和鏈上的
      return aggregatorServer, nil
}

// 建立設定的流程
func CreateKubeAPIServerConfig(){
  // 建立通用設定genericConfig
  genericConfig, versionedInformers, insecureServingInfo, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport)
}

// 通用設定的建立
func buildGenericConfig(s *options.ServerRunOptions,proxyTransport *http.Transport){
  // Insecure對應的非安全的通訊，也就是HTTP
  if lastErr = s.InsecureServing...
  // Secure對應的就是HTTPS
  if lastErr = s.SecureServing...
  // OpenAPIConfig是對外提供的API檔案
  genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig()
  // 這一塊是storageFactory的範例化，可以看到採用的是etcd作為儲存方案
  storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
	storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
	completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd)
	storageFactory, lastErr = completedStorageFactoryConfig.New()
  // Authentication 認證相關
  if lastErr = s.Authentication.ApplyTo()...
  // Authorization 授權相關
  genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer()
  // Admission 准入機制
  err = s.Admission.ApplyTo()
}

func (o *BuiltInAuthenticationOptions) ApplyTo(){
  // 前面都是對認證config進行引數設定，這裡才是真正的範例化
  authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New()
}
// New這塊的程式碼，我們要抓住核心變數authenticators和tokenAuthenticators，也就是各種認證方法
func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
  // 核心變數authenticators和tokenAuthenticators
	var authenticators []authenticator.Request
        var tokenAuthenticators []authenticator.Token
	if config.RequestHeaderConfig != nil {
		// 1. 新增requestHeader
		authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, requestHeaderAuthenticator))
	}
	if config.ClientCAContentProvider != nil {
		// 2. 新增ClientCA
    authenticators = append(authenticators, certAuth)
	}
	if len(config.TokenAuthFile) > 0 {
		// 3. token 新增tokenfile
		tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
	}
  // 4. token 新增 service account，分兩種來源
	if len(config.ServiceAccountKeyFiles) > 0 {
		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
	}
	if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) && config.ServiceAccountIssuer != "" {
		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
	}
	if config.BootstrapToken {
		if config.BootstrapTokenAuthenticator != nil {
      // 5. token 新增 bootstrap
			tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, config.BootstrapTokenAuthenticator))
		}
	}
	if len(config.OIDCIssuerURL) > 0 && len(config.OIDCClientID) > 0 {
		// 6. token 新增 oidc
    Authenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, oidcAuth))
	}
	if len(config.WebhookTokenAuthnConfigFile) > 0 {
		// 7. token 新增 webhook
		tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
	}
  // 8. 組合tokenAuthenticators到tokenAuthenticators中
	if len(tokenAuthenticators) > 0 {
		tokenAuth := tokenunion.New(tokenAuthenticators...)
		if config.TokenSuccessCacheTTL > 0 || config.TokenFailureCacheTTL > 0 {
			tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL)
		}
		authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth))
	}
  // 9. 沒有任何認證方式且啟用了Anonymous
	if len(authenticators) == 0 {
		if config.Anonymous {
			return anonymous.NewAuthenticator(), &securityDefinitions, nil
		}
		return nil, &securityDefinitions, nil
	}
  // 10. 組合authenticators
	authenticator := union.New(authenticators...)
	return authenticator, &securityDefinitions, nil
}

func BuildAuthorizer(){
  // 與上面一致，範例化是在這個New中
  return authorizationConfig.New()
}
// 不得不說，Authorizer這塊的閱讀體驗更好
func (config Config) New() (authorizer.Authorizer, authorizer.RuleResolver, error) {
  // 必須傳入一個Authorizer機制
	if len(config.AuthorizationModes) == 0 {
		return nil, nil, fmt.Errorf("at least one authorization mode must be passed")
	}
	var (
		authorizers   []authorizer.Authorizer
		ruleResolvers []authorizer.RuleResolver
	)
	for _, authorizationMode := range config.AuthorizationModes {
		// 具體的mode定義，可以跳轉到對應的連結去看，不細講
		switch authorizationMode {
		case modes.ModeNode:
			authorizers = append(authorizers, nodeAuthorizer)
			ruleResolvers = append(ruleResolvers, nodeAuthorizer)
		case modes.ModeAlwaysAllow:
			authorizers = append(authorizers, alwaysAllowAuthorizer)
			ruleResolvers = append(ruleResolvers, alwaysAllowAuthorizer)
		case modes.ModeAlwaysDeny:
			authorizers = append(authorizers, alwaysDenyAuthorizer)
			ruleResolvers = append(ruleResolvers, alwaysDenyAuthorizer)
		case modes.ModeABAC:
			authorizers = append(authorizers, abacAuthorizer)
			ruleResolvers = append(ruleResolvers, abacAuthorizer)
		case modes.ModeWebhook:
			authorizers = append(authorizers, webhookAuthorizer)
			ruleResolvers = append(ruleResolvers, webhookAuthorizer)
		case modes.ModeRBAC:
			authorizers = append(authorizers, rbacAuthorizer)
			ruleResolvers = append(ruleResolvers, rbacAuthorizer)
		default:
			return nil, nil, fmt.Errorf("unknown authorization mode %s specified", authorizationMode)
		}
	}
	return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil
}
const (
	// ModeAlwaysAllow is the mode to set all requests as authorized
	ModeAlwaysAllow string = "AlwaysAllow"
	// ModeAlwaysDeny is the mode to set no requests as authorized
	ModeAlwaysDeny string = "AlwaysDeny"
	// ModeABAC is the mode to use Attribute Based Access Control to authorize
	ModeABAC string = "ABAC"
	// ModeWebhook is the mode to make an external webhook call to authorize
	ModeWebhook string = "Webhook"
	// ModeRBAC is the mode to use Role Based Access Control to authorize
	ModeRBAC string = "RBAC"
	// ModeNode is an authorization mode that authorizes API requests made by kubelets.
	ModeNode string = "Node"
)

// 檢視定義
err = s.Admission.ApplyTo()
func (a *AdmissionOptions) ApplyTo(){
  return a.GenericAdmission.ApplyTo()
}
func (ps *Plugins) NewFromPlugins(){
  for _, pluginName := range pluginNames {
		// InitPlugin 為初始化的工作
		plugin, err := ps.InitPlugin(pluginName, pluginConfig, pluginInitializer)
		if err != nil {
			return nil, err
		}
	}
}
func (ps *Plugins) InitPlugin(name string, config io.Reader, pluginInitializer PluginInitializer) (Interface, error){
  // 獲取plugin
  plugin, found, err := ps.getPlugin(name, config)
}
// 檢視一下Interface的定義，就是對準入機制的控制 抽象化的外掛化的介面 服務於Admission Control
// Interface is an abstract, pluggable interface for Admission Control decisions.
type Interface interface {
	Handles(operation Operation) bool
}
// 再去看看獲取plugin的地方
func (ps *Plugins) getPlugin(name string, config io.Reader) (Interface, bool, error) {
	ps.lock.Lock()
	defer ps.lock.Unlock()
  // 我們再去研究ps.registry這個引數是在哪裡被初始化的
	f, found := ps.registry[name]
}
// 接下來，我們從kube-apiserver啟動過程，逐步找到Admission被初始化的地方
// 啟動命令
command := app.NewAPIServerCommand()
// server設定
s := options.NewServerRunOptions()
// admission選項
Admission:               kubeoptions.NewAdmissionOptions()
// 註冊准入機制
RegisterAllAdmissionPlugins(options.Plugins)
// 准入機制的所有內容
func RegisterAllAdmissionPlugins(plugins *admission.Plugins){
  // 這裡有很多plugin的註冊
}
// 往上翻，我們能找到所有plugin，也就是准入機制的定義 有三十幾種 已經進行了排序的
var AllOrderedPlugins = []string{
admit.PluginName,                        // AlwaysAdmit
	autoprovision.PluginName,                // NamespaceAutoProvision
	lifecycle.PluginName,                    // NamespaceLifecycle
	exists.PluginName,                       // NamespaceExists
	scdeny.PluginName,                       // SecurityContextDeny
	antiaffinity.PluginName,                 // LimitPodHardAntiAffinityTopology
	limitranger.PluginName,                  // LimitRanger
	serviceaccount.PluginName,               // ServiceAccount
	noderestriction.PluginName,              // NodeRestriction
	nodetaint.PluginName,                    // TaintNodesByCondition
	alwayspullimages.PluginName,             // AlwaysPullImages
	imagepolicy.PluginName,                  // ImagePolicyWebhook
	podsecurity.PluginName,                  // PodSecurity
	podnodeselector.PluginName,              // PodNodeSelector
	podpriority.PluginName,                  // Priority
	defaulttolerationseconds.PluginName,     // DefaultTolerationSeconds
	podtolerationrestriction.PluginName,     // PodTolerationRestriction
	eventratelimit.PluginName,               // EventRateLimit
	extendedresourcetoleration.PluginName,   // ExtendedResourceToleration
	label.PluginName,                        // PersistentVolumeLabel
	setdefault.PluginName,                   // DefaultStorageClass
	storageobjectinuseprotection.PluginName, // StorageObjectInUseProtection
	gc.PluginName,                           // OwnerReferencesPermissionEnforcement
	resize.PluginName,                       // PersistentVolumeClaimResize
	runtimeclass.PluginName,                 // RuntimeClass
	certapproval.PluginName,                 // CertificateApproval
	certsigning.PluginName,                  // CertificateSigning
	certsubjectrestriction.PluginName,       // CertificateSubjectRestriction
	defaultingressclass.PluginName,          // DefaultIngressClass
	denyserviceexternalips.PluginName,       // DenyServiceExternalIPs
	// new admission plugins should generally be inserted above here
	// webhook, resourcequota, and deny plugins must go at the end
	mutatingwebhook.PluginName,   // MutatingAdmissionWebhook
	validatingwebhook.PluginName, // ValidatingAdmissionWebhook
	resourcequota.PluginName,     // ResourceQuota
	deny.PluginName,              // AlwaysDeny
}

// 先對設定進行complete補全再進行new
func CreateKubeAPIServer(kubeAPIServerConfig *controlplane.Config, delegateAPIServer genericapiserver.DelegationTarget) (*controlplane.Instance, error) {
	kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer)
	if err != nil {
		return nil, err
	}
	return kubeAPIServer, nil
}

// 在APIExtensionsServer、KubeAPIServer和AggregatorServer三種Server啟動時，我們都能發現這麼一個函數
// APIExtensionsServer
genericServer, err := c.GenericConfig.New("apiextensions-apiserver", delegationTarget)
// KubeAPIServer
s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
// AggregatorServer
genericServer, err := c.GenericConfig.New("kube-aggregator", delegationTarget)
// 都通過GenericConfig建立了genericServer，我們先大致瀏覽下
func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
	// 新建Handler
	apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
	// 範例化一個Server
	s := &GenericAPIServer{
    ...
  }
	// 處理勾點hook操作
	for k, v := range delegationTarget.PostStartHooks() {
		s.postStartHooks[k] = v
	}
	for k, v := range delegationTarget.PreShutdownHooks() {
		s.preShutdownHooks[k] = v
	}
	// 健康監測
	for _, delegateCheck := range delegationTarget.HealthzChecks() {
		skip := false
		for _, existingCheck := range c.HealthzChecks {
			if existingCheck.Name() == delegateCheck.Name() {
				skip = true
				break
			}
		}
		if skip {
			continue
		}
		s.AddHealthChecks(delegateCheck)
	}
  // 安裝API相關引數，這個是重點
	installAPI(s, c.Config)
	return s, nil
}

func NewAPIServerHandler(name string, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
	// 採用了 github.com/emicklei/go-restful 這個庫作為 RESTful 介面的設計，目前瞭解即可
	gorestfulContainer := restful.NewContainer()
}

func installAPI(s *GenericAPIServer, c *Config) {
  // 新增 /index.html 路由規則
	if c.EnableIndex {
		routes.Index{}.Install(s.listedPathProvider, s.Handler.NonGoRestfulMux)
	}
  // 新增go語言 /pprof 的路由規則，常用於效能分析
	if c.EnableProfiling {
		routes.Profiling{}.Install(s.Handler.NonGoRestfulMux)
		if c.EnableContentionProfiling {
			goruntime.SetBlockProfileRate(1)
		}
		routes.DebugFlags{}.Install(s.Handler.NonGoRestfulMux, "v", routes.StringFlagPutHandler(logs.GlogSetter))
	}
  // 新增監控相關的 /metrics 的指標路由規則
	if c.EnableMetrics {
		if c.EnableProfiling {
			routes.MetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
		} else {
			routes.DefaultMetrics{}.Install(s.Handler.NonGoRestfulMux)
		}
	}
	// 新增版本 /version 的路由規則
	routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer)
	// 開啟服務發現
	if c.EnableDiscovery {
		s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
	}
	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
		c.FlowControl.Install(s.Handler.NonGoRestfulMux)
	}
}

func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Master, error) {
	// genericServer的初始化
	s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
	// 核心KubeAPIServer的範例化
	m := &Master{
		GenericAPIServer:          s,
		ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo,
	}
	// 註冊Legacy API的註冊
	if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {
		legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{}
		if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider); err != nil {
			return nil, err
		}
	}
	// REST介面的儲存定義，可以看到很多k8s上的常見定義，比如node節點/storage儲存/event事件等等
	restStorageProviders := []RESTStorageProvider{
		authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences},
		authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver},
		autoscalingrest.RESTStorageProvider{},
		batchrest.RESTStorageProvider{},
		certificatesrest.RESTStorageProvider{},
		coordinationrest.RESTStorageProvider{},
		discoveryrest.StorageProvider{},
		extensionsrest.RESTStorageProvider{},
		networkingrest.RESTStorageProvider{},
		noderest.RESTStorageProvider{},
		policyrest.RESTStorageProvider{},
		rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer},
		schedulingrest.RESTStorageProvider{},
		settingsrest.RESTStorageProvider{},
		storagerest.RESTStorageProvider{},
		flowcontrolrest.RESTStorageProvider{},
		// keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
		// See https://github.com/kubernetes/kubernetes/issues/42392
		appsrest.StorageProvider{},
		admissionregistrationrest.RESTStorageProvider{},
		eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL},
	}
  // 註冊API
	if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil {
		return nil, err
	}
	// 新增Hook
	m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error {
	})
	return m, nil
}

// 定義了legacy和非legacy資源的路由字首
const (
// DefaultLegacyAPIPrefix is where the legacy APIs will be located.
DefaultLegacyAPIPrefix="/api"
// APTGroupPrefix is where non-legacy API group will be located.
APIGroupPrefix ="/apis"
)
func (m *Master) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) error {
  // RESTStorage的初始化
	legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter)
  // 字首為 /api，註冊上對應的Version和Resource
  // Pod作為核心資源，沒有Group的概念
	if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil {
		return fmt.Errorf("error in registering group versions: %v", err)
	}
	return nil
}
// 我們再細看這個RESTStorage的初始化
func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generic.RESTOptionsGetter) (LegacyRESTStorage, genericapiserver.APIGroupInfo, error) {
	// pod 模板
	podTemplateStorage, err := podtemplatestore.NewREST(restOptionsGetter)
	// event事件
	eventStorage, err := eventstore.NewREST(restOptionsGetter, uint64(c.EventTTL.Seconds()))
	// limitRange資源限制
	limitRangeStorage, err := limitrangestore.NewREST(restOptionsGetter)
	// resourceQuota資源配額
	resourceQuotaStorage, resourceQuotaStatusStorage, err := resourcequotastore.NewREST(restOptionsGetter)
	// secret加密
	secretStorage, err := secretstore.NewREST(restOptionsGetter)
	// PV 儲存
	persistentVolumeStorage, persistentVolumeStatusStorage, err := pvstore.NewREST(restOptionsGetter)
	// PVC 儲存
	persistentVolumeClaimStorage, persistentVolumeClaimStatusStorage, err := pvcstore.NewREST(restOptionsGetter)
	// ConfigMap 設定
	configMapStorage, err := configmapstore.NewREST(restOptionsGetter)
	// 等等核心資源，暫不一一列舉
  // pod模板，我們的範例nginx-pod屬於這個型別的資源
  podStorage, err := podstore.NewStorage()
  // 儲存storage的對應關係
  restStorageMap := map[string]rest.Storage{
		"pods":             podStorage.Pod,
		"pods/attach":      podStorage.Attach,
		"pods/status":      podStorage.Status,
		"pods/log":         podStorage.Log,
		"pods/exec":        podStorage.Exec,
		"pods/portforward": podStorage.PortForward,
		"pods/proxy":       podStorage.Proxy,
		"pods/binding":     podStorage.Binding,
		"bindings":         podStorage.LegacyBinding,
    ...
  }
}

// 檢視Pod初始化 上一步的podStorage
func NewStorage(optsGetter generic.RESTOptionsGetter, k client.ConnectionInfoGetter, proxyTransport http.RoundTripper, podDisruptionBudgetClient policyclient.PodDisruptionBudgetsGetter) (PodStorage, error) {
	store := &genericregistry.Store{
		NewFunc:                  func() runtime.Object { return &api.Pod{} },
		NewListFunc:              func() runtime.Object { return &api.PodList{} },
		PredicateFunc:            registrypod.MatchPod,
		DefaultQualifiedResource: api.Resource("pods"),
		// 增改刪的策略
		CreateStrategy:      registrypod.Strategy,
		UpdateStrategy:      registrypod.Strategy,
		DeleteStrategy:      registrypod.Strategy,
		ReturnDeletedObject: true,
		TableConvertor: printerstorage.TableConvertor{TableGenerator: printers.NewTableGenerator().With(printersinternal.AddHandlers)},
	}
}
// 檢視 Strategy 的初始化 是一個全域性變數  進行範例化 呼叫了Scheme，核心資源的schme，legacyscheme
var Strategy = podStrategy{legacyscheme.Scheme, names.SimpleNameGenerator}
// 又查詢到Scheme的初始化。Schema可以理解為Kubernetes的登入檔，即所有的資源型別必須先註冊進Schema才可使用 註冊裡有資源的增刪改的策略
var Scheme = runtime.NewScheme()

// podStrategy 是封裝了 Pod 的各類動作，這裡我們先關注create這個操作
type podStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}
// podStrategy 的介面
type RESTCreateStrategy interface {
	runtime.ObjectTyper
	names.NameGenerator
  // 是否屬於當前的 namespace
	NamespaceScoped() bool
  // 準備建立前的檢查
	PrepareForCreate(ctx context.Context, obj runtime.Object)
  // 驗證資源物件
	Validate(ctx context.Context, obj runtime.Object) field.ErrorList
  // 規範化
	Canonicalize(obj runtime.Object)
}
// 完成了檢查，我們就要儲存資料了

// PodStorage 是 Pod 儲存的實現，裡面包含了多個儲存的定義
type PodStorage struct {
  // REST implements a RESTStorage for pods
	Pod                 *REST
  // BindingREST implements the REST endpoint for binding pods to nodes when etcd is in use.
	Binding             *BindingREST
  // LegacyBindingREST implements the REST endpoint for binding pods to nodes when etcd is in use.
	LegacyBinding       *LegacyBindingREST
	Eviction            *EvictionREST
  // StatusREST implements the REST endpoint for changing the status of a pod.
	Status              *StatusREST
  // EphemeralContainersREST implements the REST endpoint for adding EphemeralContainers
	EphemeralContainers *EphemeralContainersREST
	Log                 *podrest.LogREST
	Proxy               *podrest.ProxyREST
	Exec                *podrest.ExecREST
	Attach              *podrest.AttachREST
	PortForward         *podrest.PortForwardREST
}
/*
從上一節的map關係中，儲存在REST中
restStorageMap := map[string]rest.Storage{
		"pods":             podStorage.Pod,
}
*/
type REST struct {
	*genericregistry.Store
        // 代理傳輸層 大概率是和網路相關的先不看
	proxyTransport http.RoundTripper
}
// Store是一個通用的資料結構
type Store struct {
	// Storage定義
        ...
	Storage DryRunnableStorage
}
// DryRunnableStorage中的Storage是一個Interface
type DryRunnableStorage struct {
	Storage storage.Interface
        // 和編解碼相關的codec          
	Codec   runtime.Codec
}
func (s *DryRunnableStorage) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64, dryRun bool) error {
	if dryRun {
		if err := s.Storage.Get(ctx, key, storage.GetOptions{}, out); err == nil {
			return storage.NewKeyExistsError(key, 0)
		}
		return s.copyInto(obj, out)
	}
  // 這裡，就是Create的真正呼叫
	return s.Storage.Create(ctx, key, obj, out, ttl)
}

// Storage Interface 的定義，包括基本的增刪改查，以及watch等等進階操作
type Interface interface {
	Versioner() Versioner
	Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error
	Delete(ctx context.Context, key string, out runtime.Object, preconditions *Preconditions, validateDeletion ValidateObjectFunc) error
	Watch(ctx context.Context, key string, opts ListOptions) (watch.Interface, error)
	WatchList(ctx context.Context, key string, opts ListOptions) (watch.Interface, error)
	Get(ctx context.Context, key string, opts GetOptions, objPtr runtime.Object) error
	GetToList(ctx context.Context, key string, opts ListOptions, listObj runtime.Object) error
	List(ctx context.Context, key string, opts ListOptions, listObj runtime.Object) error
	GuaranteedUpdate(
		ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
		precondtions *Preconditions, tryUpdate UpdateFunc, suggestion ...runtime.Object) error
	Count(key string) (int64, error)
}
// 去找Storage的初始化
func NewRawStorage(config *storagebackend.Config) (storage.Interface, factory.DestroyFunc, error) {
	return factory.Create(*config)
}
func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
	switch c.Type {
  // 已經不支援etcd2
	case "etcd2":
		return nil, nil, fmt.Errorf("%v is no longer a supported storage backend", c.Type)
  // 預設為etcd3版本
	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
		return newETCD3Storage(c)
	default:
		return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)
	}
}