k8s 认证机制源码分析
当kube-apiserver收到http请求后，会经过认证，鉴权和准入控制这三个和权限相关的模块的处理，本文主要分析认证模块。
认证模块主要依据请求头携带的token或证书等信息进行认证，支持x509证书，用户名和密码(从1.19已不再支持)，token，webhook和匿名认证等方式，可在kube-apiserver进程启动时配置多种认证，当收到请求时，依次遍历这些认证，有一个认证成功则认为请求通过，并返回user信息，以便鉴权模块根据user信息进行处理，如果认证全部失败，则返回401(Unauthorized)。
如何配置
认证方法比较多，这里选择几种容易理解或常用的进行介绍。
a. 客户端证书认证: 在kube-apiserver进程启动时，通过添加–client-ca-file=SOMEFILE指定客户端证书即可启用。
b. token认证: 在kube-apiserver进程启动时，添加–token-auth-file=SOMEFILE可启动token认证，如果修改了token文件，apiserver进程必须重启才能生效。token文件是一种csv文件，包含最少三列：token,user,uid,“group1,group2,group3”。
c. 匿名认证: 默认是启用的，也可在在kube-apiserver进程启动时通过–anonymous-auth=true/false启用或关闭匿名访问，如果没配置其他任何认证方法时，匿名访问也是默认启用的。未被其他认证方法拒绝的请求将被视为匿名请求，并被赋予system:anonymous的用户名和system:unuthenticated的组名。例如，在配置了token认证和匿名认证情况下，如果提供token无效的请求将收到401 Unauthorized错误，但如果不提供任何token，则将被视为匿名请求。
需要注意的是，如果授权方式为AlwaysAllow，则匿名访问会自动关闭(即使通过参数–anonymous-auth=true启用匿名访问了，也会被自动关闭)，相关代码可参考函数ApplyAuthorization。
认证相关的命令行选项
认证相关命令行选项定义在文件pkg/kubeapiserver/options/authentication.go，初始化流程如下
NewAPIServerCommand
//初始化选项变量
//cmd/kube-apiserver/app/options/options.go
s := options.NewServerRunOptions()
s := ServerRunOptions{
Admission: kubeoptions.NewAdmissionOptions(),
Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
Authorization: kubeoptions.NewBuiltInAuthorizationOptions(),
}
return &s
...
//添加命令行参数
namedFlagSets := s.Flags()
s.Authentication.AddFlags(fss.FlagSet("authentication"))
//将是否启动匿名认证保存到o.Anonymous.Allow
if o.Anonymous != nil {
fs.BoolVar(&o.Anonymous.Allow, "anonymous-auth", o.Anonymous.Allow, ""+
"Enables anonymous requests to the secure port of the API server. "+
"Requests that are not rejected by another authentication method are treated as anonymous requests. "+
"Anonymous requests have a username of system:anonymous, and a group name of system:unauthenticated.")
}
//将客户端证书保存到o.ClientCert.ClientCA
if o.ClientCert != nil {
o.ClientCert.AddFlags(fs)
fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
"If set, any request presenting a client certificate signed by one of "+
"the authorities in the client-ca-file is authenticated with an identity "+
"corresponding to the CommonName of the client certificate.")
}
//将token认证的文件保存到o.TokenFile.TokenFile
if o.TokenFile != nil {
fs.StringVar(&o.TokenFile.TokenFile, "token-auth-file", o.TokenFile.TokenFile, ""+
"If set, the file that will be used to secure the secure port of the API server "+
"via token authentication.")
}
...
kubeoptions.NewBuiltInAuthenticationOptions().WithAll()用来初始化认证相关选项
// NewBuiltInAuthenticationOptions create a new BuiltInAuthenticationOptions, just set default token cache TTL
func NewBuiltInAuthenticationOptions() *BuiltInAuthenticationOptions {
return &BuiltInAuthenticationOptions{
TokenSuccessCacheTTL: 10 * time.Second,
TokenFailureCacheTTL: 0 * time.Second,
}
}
// WithAll set default value for every build-in authentication option
func (o *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
return o.
WithAnonymous().
WithBootstrapToken().
WithClientCert().
WithOIDC().
WithRequestHeader().
WithServiceAccounts().
WithTokenFile().
WithWebHook()
}
// WithAnonymous set default value for anonymous authentication
func (o *BuiltInAuthenticationOptions) WithAnonymous() *BuiltInAuthenticationOptions {
//默认启动匿名认证
o.Anonymous = &AnonymousAuthenticationOptions{Allow: true}
return o
}
// WithClientCert set default value for client cert
func (o *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOptions {
o.ClientCert = &genericoptions.ClientCertAuthenticationOptions{}
return o
}
// WithTokenFile set default value for token file authentication
func (o *BuiltInAuthenticationOptions) WithTokenFile() *BuiltInAuthenticationOptions {
o.TokenFile = &TokenFileAuthenticationOptions{}
return o
}
上面的选项总结如下
构造认证配置
这里是根据前面的命令行选项构造认证配置
type AuthenticationInfo struct {
// APIAudiences is a list of identifier that the API identifies as. This is
// used by some authenticators to validate audience bound credentials.
APIAudiences authenticator.Audiences
// Authenticator determines which subject is making the request
Authenticator authenticator.Request
}
//每种认证方法必须实现AuthenticateRequest接口
// Request attempts to extract authentication information from a request and
// returns a Response or an error if the request could not be checked.
type Request interface {
AuthenticateRequest(req *http.Request) (*Response, bool, error)
}
构造过程如下
buildGenericConfig(...)
s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers)
// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present.
func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error {
//根据BuiltInAuthenticationOptions构造authenticatorConfig
authenticatorConfig, err := o.ToAuthenticationConfig()
...
//认证方法初始化，见下面注释
authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New()
return nil
}
根据配置信息将配置的认证方法进行初始化
// New returns an authenticator.Request or an error that supports the standard
// Kubernetes authentication mechanisms.
func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
var authenticators []authenticator.Request
var tokenAuthenticators []authenticator.Token
securityDefinitions := spec.SecurityDefinitions{}
//初始化证书认证
// X509 methods
if config.ClientCAContentProvider != nil {
certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)
authenticators = append(authenticators, certAuth)
}
//初始化token认证
// Bearer token methods, local first, then remote
if len(config.TokenAuthFile) > 0 {
tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
if err != nil {
return nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
}
...
if len(tokenAuthenticators) > 0 {
// Union the token authenticators
//将配置的可以通过token认证的方法统一放到unionAuthTokenHandler中
tokenAuth := tokenunion.New(tokenAuthenticators...)
// Optionally cache authentication results
if config.TokenSuccessCacheTTL > 0 || config.TokenFailureCacheTTL > 0 {
tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL)
}
authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth))
...
}
//如果没配置任何认证方法，并且配置了匿名认证，则启用匿名认证
if len(authenticators) == 0 {
if config.Anonymous {
return anonymous.NewAuthenticator(), &securityDefinitions, nil
}
return nil, &securityDefinitions, nil
}
//将配置的认证方法统一到unionAuthRequestHandler中
authenticator := union.New(authenticators...)
//又将authenticator保存到AuthenticatedGroupAdder中，如果认证成功后，AuthenticatedGroupAdder会额外增加组system:authenticated
authenticator = group.NewAuthenticatedGroupAdder(authenticator)
if config.Anonymous {
// If the authenticator chain returns an error, return an error (don't consider a bad bearer token
// or invalid username/password combination anonymous).
authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
}
return authenticator, &securityDefinitions, nil
}
经过上面的初始化后，可总结如下图，所有配置的认证方法都保存到AuthenticationInfo->Authenticator中，其为Request接口类型，实现此接口的结构体为AuthenticatedGroupAdder，它的AuthenticateRequest方法没有实现认证功能，其作用是给经过认证的请求添加组system:authenticated，其成员变量Authenticator也是Request接口类型，对应的结构体为unionAuthRequestHandler，其中Handlers为authenticator.Request类型的数组，每个数组元素保存一种认证方法，其中的unionAuthTokenHandler又保存了多种token认证方法。
执行认证
处理http请求流程如下，认证流程是在FullHandlerChain中完成的
aggregatorserver FullHandlerChain -> aggregatorserver director -> apiserver director -> extensionserver director -> NotFound
认证处理函数是在DefaultBuildHandlerChain中注册，其为withAuthentication的返回值http.HandlerFunc
DefaultBuildHandlerChain
failedHandler := genericapifilters.Unauthorized(c.Serializer)
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
//k8s.io/apiserver/pkg/endpoints/filters/authentication.go
// WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then
// stores any such user found onto the provided context for the request. If authentication fails or returns an error
// the failed handler is used. On success, "Authorization" header is removed from the request and handler
// is invoked to serve the request.
func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences) http.Handler {
//返回认证处理函数
return withAuthentication(handler, auth, failed, apiAuds, recordAuthMetrics)
}
func withAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, metrics recordMetrics) http.Handler {
if auth == nil {
klog.Warning("Authentication is disabled")
return handler
}
//认证处理函数
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
authenticationStart := time.Now()
if len(apiAuds) > 0 {
req = req.WithContext(authenticator.WithAudiences(req.Context(), apiAuds))
}
//执行认证处理流程，见后面注释
resp, ok, err := auth.AuthenticateRequest(req)
authenticationFinish := time.Now()
defer func() {
metrics(req.Context(), resp, ok, err, apiAuds, authenticationStart, authenticationFinish)
}()
//如果认证失败，则返回401
if err != nil || !ok {
if err != nil {
klog.ErrorS(err, "Unable to authenticate the request")
}
failed.ServeHTTP(w, req)
return
}
...
//认证成功后，将Authorization字段从请求头里删除
// authorization header is not required anymore in case of a successful authentication.
req.Header.Del("Authorization")
//将认证成功后的user信息保存到请求上下文中，下一阶段会根据user信息进行鉴权
req = req.WithContext(genericapirequest.WithUser(req.Context(), resp.User))
handler.ServeHTTP(w, req)
})
}
前面初始化时，Authenticator指向AuthenticatedGroupAdder，所以auth.AuthenticateRequest(req)为AuthenticatedGroupAdder的AuthenticateRequest方法
//k8s.io/apiserver/pkg/authentication/group/authenticated_group_adder.go
func (g *AuthenticatedGroupAdder) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
//调用unionAuthRequestHandler的AuthenticateRequest
r, ok, err := g.Authenticator.AuthenticateRequest(req)
if err != nil || !ok {
return nil, ok, err
}
...
r.User = &user.DefaultInfo{
Name: r.User.GetName(),
UID: r.User.GetUID(),
//追加组system:authenticated
Groups: append(r.User.GetGroups(), user.AllAuthenticated),
Extra: r.User.GetExtra(),
}
return r, true, nil
}
unionAuthRequestHandler遍历启用的认证方法进行认证
//k8s.io/apiserver/pkg/authentication/request/union/unios.go: AuthenticateRequest
// AuthenticateRequest authenticates the request using a chain of authenticator.Request objects.
func (authHandler *unionAuthRequestHandler) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
var errlist []error
//遍历所有配置的认证方法，执行其AuthenticateRequest函数，有一个成功就返回成功
for _, currAuthRequestHandler := range authHandler.Handlers {
resp, ok, err := currAuthRequestHandler.AuthenticateRequest(req)
//认证失败，继续下一个认证
if err != nil {
if authHandler.FailOnError {
return resp, ok, err
}
errlist = append(errlist, err)
continue
}
if ok {
return resp, ok, err
}
}
return nil, false, utilerrors.NewAggregate(errlist)
}
下面举两个认证方法的例子: 证书认证和token认证。
证书认证处理函数
//k8s.io/apiserver/pkg/Authenticator/request/x509/x509.go
// AuthenticateRequest authenticates the request using presented client certificates
func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
if req.TLS == nil || len(req.TLS.PeerCertificates) == 0 {
return nil, false, nil
}
// Use intermediates, if provided
optsCopy, ok := a.verifyOptionsFn()
// if there are intentionally no verify options, then we cannot authenticate this request
if !ok {
return nil, false, nil
}
if optsCopy.Intermediates == nil && len(req.TLS.PeerCertificates) > 1 {
optsCopy.Intermediates = x509.NewCertPool()
for _, intermediate := range req.TLS.PeerCertificates[1:] {
optsCopy.Intermediates.AddCert(intermediate)
}
}
remaining := req.TLS.PeerCertificates[0].NotAfter.Sub(time.Now())
clientCertificateExpirationHistogram.WithContext(req.Context()).Observe(remaining.Seconds())
chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy)
if err != nil {
return nil, false, fmt.Errorf(
"verifying certificate %s failed: %w",
certificateIdentifier(req.TLS.PeerCertificates[0]),
err,
)
}
var errlist []error
for _, chain := range chains {
user, ok, err := a.user.User(chain)
if err != nil {
errlist = append(errlist, err)
continue
}
if ok {
return user, ok, err
}
}
return nil, false, utilerrors.NewAggregate(errlist)
}
token认证处理函数
//k8s.io/apiserver/pkg/Authenticator/bearertoken/bearertoken.go
func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
//获取请求头里的认证信息，格式为：Authorization: bearer XXXX
//经过处理后(去除最开始和后面的空格)，auth应该为：bearer XXXX
auth := strings.TrimSpace(req.Header.Get("Authorization"))
if auth == "" {
return nil, false, nil
}
//将认证信息按空格分开
parts := strings.SplitN(auth, " ", 3)
//parts不能小于2，parts[0]必须是bearer
if len(parts) < 2 || strings.ToLower(parts[0]) != "bearer" {
return nil, false, nil
}
token := parts[1]
// Empty bearer tokens aren't valid
if len(token) == 0 {
return nil, false, nil
}
//调用 AuthenticateToken，进行token认证
resp, ok, err := a.auth.AuthenticateToken(req.Context(), token)
// if we authenticated successfully, go ahead and remove the bearer token so that no one
// is ever tempted to use it inside of the API server
if ok {
req.Header.Del("Authorization")
}
// If the token authenticator didn't error, provide a default error
if !ok && err == nil {
err = invalidToken
}
return resp, ok, err
}
调用 AuthenticateToken，进行token认证
//k8s.io/apiserver/pkg/Authenticator/token/union/union.go
// AuthenticateToken authenticates the token using a chain of authenticator.Token objects.
func (authHandler *unionAuthTokenHandler) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) {
var errlist []error
//遍历token列表，有一个成功则返回
for _, currAuthRequestHandler := range authHandler.Handlers {
info, ok, err := currAuthRequestHandler.AuthenticateToken(ctx, token)
if err != nil {
if authHandler.FailOnError {
return info, ok, err
}
errlist = append(errlist, err)
continue
}
if ok {
return info, ok, err
}
}
return nil, false, utilerrors.NewAggregate(errlist)
}
举一个token认证的例子
//k8s.io/apiserver/pkg/Authenticator/token/tokenfile/tokenfile.go
func (a *TokenAuthenticator) AuthenticateToken(ctx context.Context, value string) (*authenticator.Response, bool, error) {
//token是否存在，如果不存在说明认证失败
//a.tokens在newAuthenticatorFromTokenFile中通过解析配置文件获取，保存的是token对应的user信息
user, ok := a.tokens[value]
if !ok {
return nil, false, nil
}
//token存在说明认证成功，返回user信息
return &authenticator.Response{User: user}, true, nil
}