changed 3 years ago  
Linked with GitHub

Detecting direct access to the Kubelet

So looking at whether a user directly hitting the node proxy can be detected by Auditing. Whilst the exact access can't be detected (as the kubelet does not support auditing) it is possible to detect that someone is accessing the node/proxy sub-resource, which provides some indication of a problem.

Audit Policy

The following audit policy (cribbed from this datadog blog) Will show up the information we need






















apiVersion: audit.k8s.io/v1
kind: Policy
rules:

# limit level to Metadata so token is not included in the spec/status
- level: Metadata
  omitStages:
  - RequestReceived
  resources:
  - group: authentication.k8s.io
    resources:
    - tokenreviews

# extended audit of auth delegation
- level: RequestResponse
  omitStages:
  - RequestReceived
  resources:
  - group: authorization.k8s.io
    resources:
    - subjectaccessreviews

Then when a service account called nodeproxy makes a request to the kubelet API directly, it generates the following entries, which (if we're not expecting people to hit the Kubelet) is a reasonable signal.




















































































































































{
  "kind": "Event",
  "apiVersion": "audit.k8s.io/v1",
  "level": "Metadata",
  "auditID": "b429b492-e30f-433f-858d-a524079c430a",
  "stage": "ResponseComplete",
  "requestURI": "/apis/authentication.k8s.io/v1/tokenreviews",
  "verb": "create",
  "user": {
    "username": "system:node:sarauditing-control-plane",
    "groups": [
      "system:nodes",
      "system:authenticated"
    ]
  },
  "sourceIPs": [
    "172.18.0.2"
  ],
  "userAgent": "kubelet/v1.21.1 (linux/amd64) kubernetes/5e58841",
  "objectRef": {
    "resource": "tokenreviews",
    "apiGroup": "authentication.k8s.io",
    "apiVersion": "v1"
  },
  "responseStatus": {
    "metadata": {},
    "code": 201
  },
  "requestReceivedTimestamp": "2022-01-29T11:04:04.073933Z",
  "stageTimestamp": "2022-01-29T11:04:04.077788Z",
  "annotations": {
    "authorization.k8s.io/decision": "allow",
    "authorization.k8s.io/reason": ""
  }
}
{
  "kind": "Event",
  "apiVersion": "audit.k8s.io/v1",
  "level": "RequestResponse",
  "auditID": "c82fc0b9-1c64-4ec4-bdd8-099cb9ac0b12",
  "stage": "ResponseComplete",
  "requestURI": "/apis/authorization.k8s.io/v1/subjectaccessreviews",
  "verb": "create",
  "user": {
    "username": "system:node:sarauditing-control-plane",
    "groups": [
      "system:nodes",
      "system:authenticated"
    ]
  },
  "sourceIPs": [
    "172.18.0.2"
  ],
  "userAgent": "kubelet/v1.21.1 (linux/amd64) kubernetes/5e58841",
  "objectRef": {
    "resource": "subjectaccessreviews",
    "apiGroup": "authorization.k8s.io",
    "apiVersion": "v1"
  },
  "responseStatus": {
    "metadata": {},
    "code": 201
  },
  "requestObject": {
    "kind": "SubjectAccessReview",
    "apiVersion": "authorization.k8s.io/v1",
    "metadata": {
      "creationTimestamp": null
    },
    "spec": {
      "resourceAttributes": {
        "verb": "get",
        "version": "v1",
        "resource": "nodes",
        "subresource": "proxy",
        "name": "sarauditing-control-plane"
      },
      "user": "system:serviceaccount:default:nodeproxy",
      "groups": [
        "system:serviceaccounts",
        "system:serviceaccounts:default",
        "system:authenticated"
      ],
      "uid": "b703429a-4e19-4ade-8329-2f4ed245e77b"
    },
    "status": {
      "allowed": false
    }
  },
  "responseObject": {
    "kind": "SubjectAccessReview",
    "apiVersion": "authorization.k8s.io/v1",
    "metadata": {
      "creationTimestamp": null,
      "managedFields": [
        {
          "manager": "kubelet",
          "operation": "Update",
          "apiVersion": "authorization.k8s.io/v1",
          "time": "2022-01-29T11:04:04Z",
          "fieldsType": "FieldsV1",
          "fieldsV1": {
            "f:spec": {
              "f:groups": {},
              "f:resourceAttributes": {
                ".": {},
                "f:name": {},
                "f:resource": {},
                "f:subresource": {},
                "f:verb": {},
                "f:version": {}
              },
              "f:uid": {},
              "f:user": {}
            }
          }
        }
      ]
    },
    "spec": {
      "resourceAttributes": {
        "verb": "get",
        "version": "v1",
        "resource": "nodes",
        "subresource": "proxy",
        "name": "sarauditing-control-plane"
      },
      "user": "system:serviceaccount:default:nodeproxy",
      "groups": [
        "system:serviceaccounts",
        "system:serviceaccounts:default",
        "system:authenticated"
      ],
      "uid": "b703429a-4e19-4ade-8329-2f4ed245e77b"
    },
    "status": {
      "allowed": true,
      "reason": "RBAC: allowed by ClusterRoleBinding \"nodeproxybinding\" of ClusterRole \"nodeproxy\" to ServiceAccount \"nodeproxy/default\""
    }
  },
  "requestReceivedTimestamp": "2022-01-29T11:04:04.078619Z",
  "stageTimestamp": "2022-01-29T11:04:04.079637Z",
  "annotations": {
    "authorization.k8s.io/decision": "allow",
    "authorization.k8s.io/reason": ""
  }
}
Select a repo