post image January 6, 2022 | 3 min Read

Audit logging via kube-api server

**Audit log from Mushad’s course''

apiVersion: audit.k8s.io/v1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
rules:
  # Log pod changes at RequestResponse level
  - level: Metadata
    namespace: ["prod"]
    verb: ["delete"]
    resources:
    - group: ""
      # Resource "pods" doesn't match requests to any subresource of pods,
      # which is consistent with the RBAC policy.
      resources: ["secrets"]
  # Log "pods/log", "pods/status" at Metadata level


apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.105.184.10:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=10.105.184.10
    - --allow-privileged=true
    - --authorization-mode=Node,RBAC
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --audit-policy-file=/etc/kubernetes/prod-audit.yaml
    - --audit-log-path=/var/log/prod-secrets.log
    - --audit-log-maxage=30  
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-servers=https://127.0.0.1:2379
    - --insecure-port=0
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --secure-port=6443
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
    - --service-account-key-file=/etc/kubernetes/pki/sa.pub
    - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
    - --service-cluster-ip-range=10.96.0.0/12
    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    image: k8s.gcr.io/kube-apiserver:v1.20.0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 10.105.184.10
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-apiserver
    readinessProbe:
      failureThreshold: 3
      httpGet:
        host: 10.105.184.10
        path: /readyz
        port: 6443
        scheme: HTTPS
      periodSeconds: 1
      timeoutSeconds: 15
    resources:
      requests:
        cpu: 250m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 10.105.184.10
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/kubernetes/prod-audit.yaml
      name: audit
      readOnly: true
    - mountPath: /var/log/prod-secrets.log
      name: audit-log
      readOnly: false
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/ca-certificates
      name: etc-ca-certificates
      readOnly: true
    - mountPath: /etc/kubernetes/pki
      name: k8s-certs
      readOnly: true
    - mountPath: /usr/local/share/ca-certificates
      name: usr-local-share-ca-certificates
      readOnly: true
    - mountPath: /usr/share/ca-certificates
      name: usr-share-ca-certificates
      readOnly: true
  hostNetwork: true
  priorityClassName: system-node-critical
  volumes:
  - name: audit
    hostPath:
      path: /etc/kubernetes/prod-audit.yaml
      type: File
  - name: audit-log
    hostPath:
      path: /var/log/prod-secrets.log
      type: FileOrCreate  
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/ca-certificates
      type: DirectoryOrCreate
    name: etc-ca-certificates
  - hostPath:
      path: /etc/kubernetes/pki
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /usr/local/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-local-share-ca-certificates
  - hostPath:
      path: /usr/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-share-ca-certificates
status: {}



**Policy file: /etc/kubernetes/audit/policy.yaml ''

cat /etc/kubernetes/audit/policy.yaml 
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Metadata

**Advanced Policy file: /etc/kubernetes/audit/policy.yaml ''

 cat  /etc/kubernetes/audit/policy.yaml 
apiVersion: audit.k8s.io/v1 # This is required.
kind: Policy

# Don't generate audit events for all requests in RequestReceived stage.
# 1. Nothing from stage RequestReceived
omitStages:
  - "RequestReceived"

rules:
  - level: None
    # 2. Nothing from "watch", "list", "get"
    verbs: ["watch", "list", "get"]

  - level: Metadata
    resources:
    # 3. From Secrets only metadata level
    - group: "" # core API group
      resources: ["secrets"]
    # 4. log evrything related to "RequestResponse"
  - level: RequestResponse



**Kube-api server file: /etc/kubernetes/manifests/kube-apiserver.yaml''

 cat /etc/kubernetes/manifests/kube-apiserver.yaml                         
apiVersion: v1                                                                               
kind: Pod                                                                                    
metadata:                                                                                    
  annotations:                                                                               
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.156.0.2:6443         
  creationTimestamp: null                                                                    
  labels:                                                                                    
    component: kube-apiserver                                                                
    tier: control-plane                                                                      
  name: kube-apiserver                                                                       
  namespace: kube-system                                                                     
spec:                                                                                        
  containers:                                                                                
  - command:                                                                                 
    - kube-apiserver                                                                         
    - --advertise-address=10.156.0.2                                                         
    - --audit-policy-file=/etc/kubernetes/audit/policy.yaml                                  
    - --audit-log-path=/etc/kubernetes/audit/logs/audit.log                                  
    - --audit-log-maxsize=500                                                                
    - --audit-log-maxbackup=5                                                                
    ... ... ...
    - --allow-privileged=true                                                                
    - --encryption-provider-config=/etc/kubernetes/etcd/ec.yaml                              
    - --anonymous-auth=true                                                                  
    - --authorization-mode=Node,RBAC                                                         
    - --client-ca-fil

...
    - mountPath: /etc/kubernetes/audit
      name: audit

...  
  - hostPath:
      path: /etc/kubernetes/audit
      type: DirectoryOrCreate
    name: audit


author image

Jan Toth

I have been in DevOps related jobs for past 6 years dealing mainly with Kubernetes in AWS and on-premise as well. I spent quite a lot …

comments powered by Disqus