# Kubernetes安全

# Kubernetes 安全框架

K8S安全控制框架主要由下面3个阶段进行控制，每一个阶段都支持插件方式，通过API Server配置来启用插件。

Authentication(鉴权)
Authorization(授权)
Admission Control(准入控制)

安全框架

客户端要想访问K8s集群API Server，一般需要证书、Token或者用户名+密码;如果Pod访问，需要ServiceAccount

# 认证，授权，准入控制

# 三种客户端身份认证

HTTPS 证书认证: 基于CA证书签名的数字证书认证
HTTP Token认证: 通过一个Token来识别用户
HTTP Base认证: 用户名+密码的方式认证

# 授权

RBAC(Role-Based Access Control，基于角色的访问控制): 负责完成授权(Authorization)工作。

RBAC根据API请求属性，决定允许还是拒绝。

比较常见的授权维度

user: 用户名
group: 用户分组
资源，例如pod、deployment
资源操作方法: get，list，create，update，patch，watch，delete
命名空间
API组

# 准入控制

Adminssion Control实际上是一个准入控制器插件列表，发送到API Server的请求都需要经过这个列表中的每个准入控制器插件的检查，检查不通过，则拒绝请求。

# 基于角色的权限访问控制:RBAC

RBAC(Role-Based Access Control，基于角色的访问控制)，允许通过Kubernetes API动态配置策略。

基于角色的权限访问控制

# 角色

Role: 授权特定命名空间的访问权限
ClusterRole: 授权所有命名空间的访问权限

# 角色绑定

RoleBinding: 将角色绑定到主体(即subject)
ClusterRoleBinding: 将集群角色绑定到主体

# 主体(subject)

User: 用户
Group: 用户组
ServiceAccount: 服务账号

# 案例:为指定用户授权访问不同命名空间权限

示例: 为用户liuxiaolu(图片上是aliang)授权default命名空间Pod读取权限认证流程

# 用K8SCA签发客户端证书

# 代码准备

# cert.sh


cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

cat > liuxiaolu-csr.json <<EOF
{
  "CN": "liuxiaolu",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=/etc/kubernetes/pki/ca.crt -ca-key=/etc/kubernetes/pki/ca.key -config=ca-config.json -profile=kubernetes liuxiaolu-csr.json | cfssljson -bare liuxiaolu

# 工具包cfssl

点击下载工具包, 密码: 3hd1 (opens new window)

# 生成证书

# 准备完成后，代码清单如下
$ ls
cert.sh  cfssl  cfssl-certinfo  cfssljson

# 将脚本放入二进制目录下，这样就可以执行了
$ mv cfssl* /usr/bin/
$ bash cert.sh
2021/06/06 19:07:44 [INFO] generate received request
2021/06/06 19:07:44 [INFO] received CSR
2021/06/06 19:07:44 [INFO] generating key: rsa-2048
2021/06/06 19:07:45 [INFO] encoded CSR
2021/06/06 19:07:45 [INFO] signed certificate with serial number 173532708244838459846100561504657639090421304385
2021/06/06 19:07:45 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成kubeconfig授权文件

# 代码准备

# kubeconfig.sh

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/pki/ca.crt \
  --embed-certs=true \
  --server=https://10.69.1.160:6443 \
  --kubeconfig=liuxiaolu.kubeconfig
 
# 设置客户端认证
kubectl config set-credentials liuxiaolu \
  --client-key=liuxiaolu-key.pem \
  --client-certificate=liuxiaolu.pem \
  --embed-certs=true \
  --kubeconfig=liuxiaolu.kubeconfig

# 设置默认上下文
kubectl config set-context kubernetes \
  --cluster=kubernetes \
  --user=liuxiaolu \
  --kubeconfig=liuxiaolu.kubeconfig

# 设置当前使用配置
kubectl config use-context kubernetes --kubeconfig=liuxiaolu.kubeconfig

# 执行

$ bash kubeconfig.sh
Cluster "kubernetes" set.
User "liuxiaolu" set.
Context "kubernetes" modified.
Switched to context "kubernetes".

# 创建RBAC权限策略

# 代码准备

# rbac.yaml

创建一个只能读取default命名空间下的pod的角色权限

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: default
  name: pod-reader
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "watch", "list"]

---

kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: read-pods
  namespace: default
subjects:
- kind: User
  name: liuxiaolu
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: pod-reader
  apiGroup: rbac.authorization.k8s.io

# 执行

# node节点在没有配置kubeconfig文件信息的情况下是无法访问集群的
[root@liuxiaolu-node ~]# kubectl get pod
The connection to the server localhost:8080 was refused - did you specify the right host or port?

# 将生成的配置文件拷贝到liuxiaolu-node这个节点
[root@liuxiaolu-master secrity]# scp liuxiaolu.kubeconfig root@10.69.1.161:~
The authenticity of host '10.69.1.161 (10.69.1.161)' can't be established.
root@10.69.1.161's password:
liuxiaolu.kubeconfig                                                                                                                                                      100% 5723     3.5MB/s   00:00

# 使用配置文件访问集群，在没有配置集群访问策略的情况下，是无法访问的
[root@liuxiaolu-node ~]# kubectl get pod --kubeconfig=liuxiaolu.kubeconfig
Error from server (Forbidden): pods is forbidden: User "liuxiaolu" cannot list resource "pods" in API group "" in the namespace "default"

# 创建访问策略
[root@liuxiaolu-master ~]# kubectl apply -f rbac.yaml
role.rbac.authorization.k8s.io/pod-reader created
rolebinding.rbac.authorization.k8s.io/read-pods created

# 创建完成之后再次在node节点进行查询就可以查询到了
[root@liuxiaolu-node ~]# kubectl get pod --kubeconfig=liuxiaolu.kubeconfig
NAME                                     READY   STATUS    RESTARTS   AGE
configmap-demo-pod                       1/1     Running   0          3h48m
......

# 验证是否有删除pod或者查看service的权限，结论是没有
[root@liuxiaolu-node ~]# kubectl delete pod secret-demo-pod  --kubeconfig=liuxiaolu.kubeconfig
Error from server (Forbidden): pods "secret-demo-pod" is forbidden: User "liuxiaolu" cannot delete resource "pods" in API group "" in the namespace "default"
[root@liuxiaolu-node ~]# kubectl get svc  --kubeconfig=liuxiaolu.kubeconfig
Error from server (Forbidden): services is forbidden: User "liuxiaolu" cannot list resource "services" in API group "" in the namespace "default"

# 将配置文件设置为默认使用
[root@liuxiaolu-node ~]# mkdir .kube
mkdir: cannot create directory '.kube': File exists
[root@liuxiaolu-node ~]# cp liuxiaolu.kubeconfig .kube/config
[root@liuxiaolu-node ~]# kubectl get pod
NAME                                     READY   STATUS    RESTARTS   AGE
configmap-demo-pod                       1/1     Running   0          3h55m
deploy-pvc-6994b7ff4b-7p4lz              1/1     Running   0          30h
deploy-pvc-6994b7ff4b-h78j4              1/1     Running   0          30h
......

# 网络策略概述

网络策略(Network Policy)，用于限制Pod出入流量，提供Pod级别和Namespace级别网络访问控制。

# 一些应用场景

应用程序间的访问控制。例如微服务A允许访问微服务B，微服务C不能访问微服务A
开发环境命名空间不能访问测试环境命名空间Pod
当Pod暴露到外部时，需要做Pod白名单
多租户网络环境隔离

# Pod网络入口方向隔离

基于Pod级网络隔离: 只允许特定对象访问Pod(使用标签定义)，允许白名单上的IP地址或者IP段访问Pod
基于Namespace级网络隔离: 多个命名空间，A和B命名空间Pod完全隔离。

# Pod网络出口方向隔离

拒绝某个Namespace上所有Pod访问外部
基于目的IP的网络隔离: 只允许Pod访问白名单上的IP地址或者IP段
基于目标端口的网络隔离: 只允许Pod访问白名单上的端口

# 示例

点击查看官网示例 (opens new window)

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: test-network-policy
  namespace: default
spec:
  podSelector:
    matchLabels:
      # 执行命令kubectl get pod -l app=web查询出来的pod，执行当前策略
      app: web
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - ipBlock:
        # 来自于172.17.0.0/16网段的ip可访问
        cidr: 172.17.0.0/16
        # 但是排除掉来自于172.17.1.0/24网段的ip
        except:
        - 172.17.1.0/24
    # 基于命名空间匹配
    - namespaceSelector:
        matchLabels:
          # 匹配打有ns=web标签的命名空间
          ns: web
    - podSelector:
        matchLabels:
          # 打有role=frontend标签的pod可以访问
          role: frontend
    # 可访问6379这个端口      
    ports:
    - protocol: TCP
      port: 6379
  egress:
  - to:
    - ipBlock:
        # pod的出流量可访问网段10.0.0.0/24
        cidr: 10.0.0.0/24
    ports:
    - protocol: TCP
      # pod的出流量可访问5978端口
      port: 5978

podSelector:目标Pod，根据标签选择
policyTypes:策略类型，指定策略用于入站、出站流量。
Ingress:from是可以访问的白名单，可以来自于IP段、命名空间、 Pod标签等，ports是可以访问的端口。
Egress:这个Pod组可以访问外部的IP段和端口。

# 案例:对项目Pod出入流量访问控制

# 需求1

将default命名空间携带app=web标签的Pod隔离，只允许default命名空间携带run=client1标签的Pod访问80端口。

# 准备测试环境

$ kubectl create deployment web --image=nginx 
$ kubectl run client1 --image=busybox -- sleep 36000
$ kubectl run client2 --image=busybox -- sleep 36000

# 查看app=web的pod的ip地址
$ kubectl get pod -l app=web -o wide
NAME                  READY   STATUS    RESTARTS   AGE     IP              NODE             NOMINATED NODE   READINESS GATES
web-96d5df5c8-cxqjk   1/1     Running   0          2m27s   10.244.17.140   liuxiaolu-node   <none>           <none>

# 验证client1这个pod是可以访问带有标签app=web的pod
$ kubectl exec -it client1 -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
64 bytes from 10.244.17.140: seq=0 ttl=63 time=0.600 ms
^C
--- 10.244.17.140 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.252/0.426/0.600 ms
/ # wget 10.244.17.140
Connecting to 10.244.17.140 (10.244.17.140:80)
saving to 'index.html'
index.html           100% |************************************************************************************************************************************************************|   612  0:00:00 ETA
'index.html' saved

# 验证client2这个pod是可以访问带有标签app=web的pod
$ kubectl exec -it client2 -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
64 bytes from 10.244.17.140: seq=0 ttl=63 time=0.315 ms
^C
--- 10.244.17.140 ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.215/0.254/0.315 ms
/ # wget 10.244.17.140
Connecting to 10.244.17.140 (10.244.17.140:80)
saving to 'index.html'
index.html           100% |************************************************************************************************************************************************************|   612  0:00:00 ETA
'index.html' saved

# 创建网络策略
$ kubectl apply -f np1.yaml
networkpolicy.networking.k8s.io/test-network-policy created

# 确认只有client1这个pod带有标签run=client1
$ kubectl get pod -l run=client1
NAME      READY   STATUS    RESTARTS   AGE
client1   1/1     Running   0          9m23s

# 由于只开放了80端口，所以无法ping通，但可以执行wget访问80端口
$ kubectl exec -it client1 -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
--- 10.244.17.140 ping statistics ---
3 packets transmitted, 0 packets received, 100% packet loss
/ # wget 10.244.17.140
Connecting to 10.244.17.140 (10.244.17.140:80)
saving to 'index.html'
index.html           100% |************************************************************************************************************************************************************|   612  0:00:00 ETA
'index.html' saved

# 由于client2不带有run=client1这个标签，所以不能访问80端口
$ kubectl exec -it client2 -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
--- 10.244.17.140 ping statistics ---
5 packets transmitted, 0 packets received, 100% packet loss
/ # wget 10.244.17.140
Connecting to 10.244.17.140 (10.244.17.140:80)

yaml

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: test-network-policy
  namespace: default
spec:
  podSelector:
    matchLabels:
      app: web
  policyTypes:
    - Ingress
  ingress:
    - from:
        - podSelector:
            matchLabels:
              run: client1
      ports:
        - protocol: TCP
          port: 80

# 需求2

default命名空间下所有pod可以互相访问，也可以访问其他命名空间Pod，但其他命名空间不能访问default命名空间Pod。

# 准备测试环境

# 删除上一步创建的网络策略，避免干扰测试
$ kubectl delete networkpolicy test-network-policy

# 在其他命名空间创建一个pod
$ kubectl run client418 -nkube-system --image=busybox -- sleep 36000

# 测试在没有创建拒绝策略的时候，来自非default命名空间的可以正常访问default命名空间中的pod
$ kubectl exec -it client418 -nkube-system -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
64 bytes from 10.244.17.140: seq=0 ttl=62 time=1.627 ms

# 验证在没有创建拒绝策略的时候，default命名空间中的pod可以相互访问
$ kubectl exec -it client2 -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
64 bytes from 10.244.17.140: seq=0 ttl=63 time=0.222 ms

$ kubectl apply -f deny-from-other-ns-network-policy.yaml
networkpolicy.networking.k8s.io/deny-from-other-ns-network-policy created

# 测试在创建拒绝策略后，来自非default命名空间的不能访问default命名空间中的pod
$ kubectl exec -it client418 -nkube-system -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
--- 10.244.17.140 ping statistics ---
2 packets transmitted, 0 packets received, 100% packet loss

# 测试在创建拒绝策略后，default命名空间中的pod可以相互访问
$ kubectl exec -it client2 -- sh
/ # ping 10.244.17.140
PING 10.244.17.140 (10.244.17.140): 56 data bytes
64 bytes from 10.244.17.140: seq=0 ttl=63 time=0.269 ms
--- 10.244.17.140 ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss

yaml

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: deny-from-other-ns-network-policy
  namespace: default
spec:
  # 未配置的情况下，默认选择所有pod
  podSelector: {}
  policyTypes:
    - Ingress
  ingress:
    - from:
        # 如果未配置，拒绝所有
        - podSelector: {}

← Kubernetes存储 Kubernetes集群维护 →