owned this note
owned this note
Published
Linked with GitHub
# Investigate cryptominer warning on cloudbank
GCP emailed us about crypto mining activity, and sure enough i found someone running it on the cloudbank demo hub.
I wanted to look at what systemcalls it is making, and since I can't run `strace` from inside the container, I:
1. Found the node it is on with `kubectl -n demo get pod -o wide`
2. SSH'd into that node with `gcloud compute ssh <name>`
3. Ran a privileged docker container with the host pid namespace mounted with `docker run --privileged --pid=host -it ubuntu:latest`
4. Installed strace in the container with `apt update && apt install -y strace`
5. Found the offending process with a `top`
6. Ran strace with `strace -p <pid> -f -s1000`
Here's what strace output kinda looks like:
```
[pid 33162] epoll_pwait(3, <unfinished ...>
[pid 33171] write(11, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 33162] <... epoll_pwait resumed>[{EPOLLIN, {u32=11, u64=11}}], 1024, 500, NULL, 8) = 1
[pid 33162] read(11, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 33162] read(11, 0x7f4502154450, 8) = -1 EAGAIN (Resource temporarily unavailable)
[pid 33162] futex(0x555557341a74, FUTEX_WAIT_PRIVATE, 2147483664, NULL <unfinished ...>
[pid 33171] futex(0x555557341a74, FUTEX_WAKE_PRIVATE, 1) = 1
[pid 33162] <... futex resumed>) = 0
[pid 33162] write(14, "{\"id\":409,\"jsonrpc\":\"2.0\",\"method\":\"submit\",\"params\":{\"id\":\"633202341\",\"job_id\":\"999693589\",\"nonce\":\"1d922f00\",\"result\":\"9c2f29f31eb225457f72efd04eb7b22619d604dad57d8cefd8d8a28a806d0000\"}}\n", 189) = 189
[pid 33162] epoll_pwait(3, [{EPOLLIN, {u32=14, u64=14}}], 1024, 447, NULL, 8) = 1
[pid 33162] read(14, "{\"id\":409,\"jsonrpc\":\"2.0\",\"error\":null,\"result\":{\"status\":\"OK\"}}\n", 65536) = 65
[pid 33162] writev(1, [{iov_base="", iov_len=0}, {iov_base="[2022-04-16 20:25:48\33[1;30m.962\33[0m] \33[46;1m\33[1;37m cpu \33[0m \33[1;32maccepted\33[0m (393/0) diff \33[1;37m113979\33[0m \33[1;30m(173 ms)\33[0m\33[0m\n", iov_len=140}], 2) = 140
[pid 33162] epoll_pwait(3, [], 1024, 273, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 436, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 63, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 436, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 64, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 435, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 65, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 434, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 66, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 432, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 67, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 430, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 69, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 429, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 61, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 437, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 63, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 436, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 63, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 436, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 1, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 64, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 434, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 65, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 433, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 66, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 432, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 67, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 431, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 68, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 430, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 69, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 429, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 69, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, <unfinished ...>
[pid 33171] write(11, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 33162] <... epoll_pwait resumed>[{EPOLLIN, {u32=11, u64=11}}], 1024, 429, NULL, 8) = 1
[pid 33162] read(11, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 33162] read(11, 0x7f4502154450, 8) = -1 EAGAIN (Resource temporarily unavailable)
[pid 33162] futex(0x555557341a74, FUTEX_WAIT_PRIVATE, 2147483664, NULL <unfinished ...>
[pid 33171] futex(0x555557341a74, FUTEX_WAKE_PRIVATE, 1) = 1
[pid 33162] <... futex resumed>) = 0
[pid 33162] write(14, "{\"id\":410,\"jsonrpc\":\"2.0\",\"method\":\"submit\",\"params\":{\"id\":\"633202341\",\"job_id\":\"999693589\",\"nonce\":\"d5f92f00\",\"result\":\"806e542b08c0bf02384bad3c33789d8e34c5269cd8199aaa0a9bb33142110000\"}}\n", 189) = 189
[pid 33162] epoll_pwait(3, [{EPOLLIN, {u32=14, u64=14}}], 1024, 288, NULL, 8) = 1
[pid 33162] read(14, "{\"id\":410,\"jsonrpc\":\"2.0\",\"error\":null,\"result\":{\"status\":\"OK\"}}\n", 65536) = 65
[pid 33162] writev(1, [{iov_base="", iov_len=0}, {iov_base="[2022-04-16 20:26:05\33[1;30m.578\33[0m] \33[46;1m\33[1;37m cpu \33[0m \33[1;32maccepted\33[0m (394/0) diff \33[1;37m113979\33[0m \33[1;30m(175 ms)\33[0m\33[0m\n", iov_len=140}], 2) = 140
[pid 33162] epoll_pwait(3, [], 1024, 113, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 70, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 428, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 71, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 427, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 72, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 425, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 75, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 423, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 75, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 418, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 81, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 417, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 82, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 416, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 83, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 415, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 84, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 414, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 84, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 414, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 85, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 414, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 85, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 413, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 86, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 412, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 87, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 412, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 87, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 412, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 88, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 411, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 88, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 411, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 88, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [{EPOLLIN, {u32=14, u64=14}}], 1024, 500, NULL, 8) = 1
[pid 33162] read(14, "{\"jsonrpc\":\"2.0\",\"method\":\"job\",\"params\":{\"blob\":\"0c0ceeccec9206600eb9f652c30f632b5f675cf80405ca4d2ae58165072f34df3f8fb2acc7b8a80000000022bfb57e0efdc1b524eb6febd33fa1d96657c0460cbcd9319c48a2d63db4600302\",\"job_id\":\"569093063\",\"target\":\"32930000\",\"id\":\"633202341\",\"diff\":\"113978\"}}\n", 65536) = 280
[pid 33162] writev(1, [{iov_base="", iov_len=0}, {iov_base="[2022-04-16 20:26:22\33[1;30m.835\33[0m] \33[44;1m\33[1;37m net \33[0m \33[1;35mnew job\33[0m from \33[1;37mcryptonight_upx.asia.mine.zergpool.com:4457\33[0m diff \33[1;37m113979\33[0m algo \33[1;37mcn/upx2\33[0m (2 tx)\33[0m\n", iov_len=202}], 2) = 202
[pid 33162] epoll_pwait(3, [], 1024, 464, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 410, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 89, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 410, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 88, NULL, 8) = 0
[pid 33162] epoll_pwait(3, <unfinished ...>
[pid 33169] write(11, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 33162] <... epoll_pwait resumed>[{EPOLLIN, {u32=11, u64=11}}], 1024, 500, NULL, 8) = 1
[pid 33162] read(11, "\1\0\0\0\0\0\0\0", 8) = 8
[pid 33162] read(11, 0x7f4502154450, 8) = -1 EAGAIN (Resource temporarily unavailable)
[pid 33162] futex(0x555557341a74, FUTEX_WAIT_PRIVATE, 2147483664, NULL <unfinished ...>
[pid 33169] futex(0x555557341a74, FUTEX_WAKE_PRIVATE, 1) = 1
[pid 33162] <... futex resumed>) = 0
[pid 33162] write(14, "{\"id\":411,\"jsonrpc\":\"2.0\",\"method\":\"submit\",\"params\":{\"id\":\"633202341\",\"job_id\":\"569093063\",\"nonce\":\"810e0100\",\"result\":\"eaf37805afcd58df45676178d60efdd5734cb1927d864035046dba83384e0000\"}}\n", 189) = 189
[pid 33162] epoll_pwait(3, [], 1024, 129, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [{EPOLLIN, {u32=14, u64=14}}], 1024, 411, NULL, 8) = 1
[pid 33162] read(14, "{\"id\":411,\"jsonrpc\":\"2.0\",\"error\":null,\"result\":{\"status\":\"OK\"}}\n", 65536) = 65
[pid 33162] writev(1, [{iov_base="", iov_len=0}, {iov_base="[2022-04-16 20:26:25\33[1;30m.347\33[0m] \33[46;1m\33[1;37m cpu \33[0m \33[1;32maccepted\33[0m (395/0) diff \33[1;37m113979\33[0m \33[1;30m(175 ms)\33[0m\33[0m\n", iov_len=140}], 2) = 140
[pid 33162] epoll_pwait(3, [], 1024, 365, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 88, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 500, NULL, 8) = 0
[pid 33162] epoll_pwait(3, [], 1024, 411, NULL, 8) = 0
```
This makes sense - it's mostly just doing network shit with syscalls, and CPU events aren't really detectable here.
I decided to try out https://blog.px.dev/detect-monero-miners/#what-can-we-detect now.
Using https://github.com/iovisor/kubectl-trace/ to run the bpftrace program there.
I got it to work! And it is detecting the monero properly. I don't have a way to kill it, but we can fix that.
I installed kubectl-trace, then got [https://raw.githubusercontent.com/pixie-io/pixie-demos/main/detect-monero-demo/detectrandomx.bt](this bpftrace file) from https://github.com/pixie-io/pixie-demos/tree/main/detect-monero-demo (linked to from https://blog.px.dev/detect-monero-miners/).
Then I ran `k trace run --fetch-headers <node-name> -f detectrandomx.bt` to run the bpftrace program. I could then run `k trace logs kubectl-trace-8afd271a-b6de-42c6-ab18-99264b3d45f3` (the name of the trace, which I got from `k trace get`) to see the trace logs. They looked like:
```
time_:12292062462353 pid:33162 comm:christmas fpcr:2
time_:12292063451030 pid:33162 comm:christmas fpcr:2
time_:12292064459886 pid:33162 comm:christmas fpcr:2
time_:12292065452867 pid:33162 comm:christmas fpcr:2
time_:12292066460720 pid:33162 comm:christmas fpcr:2
time_:12292067544413 pid:33162 comm:christmas fpcr:2
time_:12292069451562 pid:33162 comm:christmas fpcr:2
time_:12292070457102 pid:33162 comm:christmas fpcr:2
time_:12292072124629 pid:33162 comm:christmas fpcr:2
time_:12292072453338 pid:33162 comm:christmas fpcr:2
time_:12292072458044 pid:33162 comm:christmas fpcr:2
time_:12292073452149 pid:33162 comm:christmas fpcr:2
time_:12292074456460 pid:33162 comm:christmas fpcr:2
time_:12292077451048 pid:33162 comm:christmas fpcr:2
```
`christmas` was the name of the process that was running the miner, so it successfully detected it. THIS MEANS WE CAN AUTO KILL IT TOO.
Either way, I killed the process and it didn't come back - so this was some kiddie manually doing this.
I'm just killing the demo hub for now, as I'm sure this user will come back.
Their username was saver@sweepz.org - we can figure out which auth provider they used later.
Killed the hub with `k -n demo delete deployment proxy`.9