kprobe: getting random values in arguments

[syogaraj]

Hi, I'm trying to instrument the kill syscall using kprobe. But, I'm getting some random values in place of the pid and signal that is used while execution.

killsnoop.bpf.c

#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include <sys/types.h>

#include "killsnoop.h"
#include "bpf_misc.h"

struct val_t {
    __u32 uid;
    __u32 pid;
    int sig;
    int tpid;
    char comm[TASK_COMM_LEN];
};

struct data_t {
    __u64 ts;
    __u32 uid;
    __u32 pid;
    int tpid;
    int sig;
    int ret;
    char comm[TASK_COMM_LEN];
};


struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __type(key, __u32);
    __type(value, struct val_t);
    __uint(max_entries, 10240);
} info_map SEC(".maps");


struct {
    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
    __uint(key_size, sizeof(__u32));
    __uint(value_size, sizeof(__u32));
    __uint(max_entries, 10240);
} events SEC(".maps");


int probe_entry(struct pt_regs *ctx, pid_t tpid, int sig) {
    __u64 pid_tgid = bpf_get_current_pid_tgid();
    __u32 pid = pid_tgid >> 32;
    __u32 tid = (__u32) pid_tgid;
    __u64 uid_gid = bpf_get_current_uid_gid();
    __u32 uid = (__u32) uid_gid;

    if(sig == 0) {
        /**
            If sig is 0, then no signal is sent, but existence and permission
            checks are still performed; this can be used to check for the
            existence of a process ID or process group ID that the caller is
            permitted to signal.
        */
        return 0;
    }

    char fmt[] = "KILL syscall called for: %d with signal: %d";
    bpf_trace_printk(fmt, sizeof(fmt), tpid, sig);

    struct val_t val = {
            .uid = uid,
            .pid = pid
    };

    if (bpf_get_current_comm(&val.comm, sizeof(val.comm)) == 0) {
        val.tpid = tpid;
        val.sig = sig;
        bpf_map_update_elem(&info_map, &tid, &val, BPF_ANY);
    }
    return 0;
}


int probe_exit(struct pt_regs *ctx) {
    struct data_t data = {};
    struct val_t *valp;
    __u64 pid_tgid = bpf_get_current_pid_tgid();
    __u32 pid = pid_tgid >> 32;
    __u32 tid = (__u32) pid_tgid;

    valp = bpf_map_lookup_elem(&info_map, &tid);
    if (!valp) {
        return 0; // missed entry
    }

    bpf_probe_read_kernel(&data.comm, sizeof(data.comm), valp->comm);
    data.ts = 0;// bpf_ktime_get_ns();
    data.pid = pid;
    data.tpid = valp->tpid;
    data.ret = PT_REGS_RC(ctx);
    data.sig = valp->sig;
    data.uid = valp->uid;

    bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &data, sizeof(data));

    bpf_map_delete_elem(&info_map, &tid);
    return 0;
}

SEC("kprobe/"SYS_PREFIX"sys_kill")
int BPF_KPROBE(entry_probe, pid_t tpid, int sig) {
    return probe_entry(ctx, tpid, sig);
}


SEC("kretprobe/"SYS_PREFIX"sys_kill")
int BPF_KRETPROBE(return_probe) {
    return probe_exit(ctx);
}

char _license[] SEC("license") = "GPL";

killsnoop.c

#include <signal.h>

#include "killsnoop.skel.h"
#include "killsnoop.h"


#define OUTPUT_FORMAT "%llu %lu %d %s %d %d\n"
#define HEADER "TIME        PID      TPID      COMMAND    SIGNAL    UID  "

/* Set the poll timeout when no events occur.*/
#define PERF_POLL_TIMEOUT_MS    1
#define PERF_BUFFER_PAGES 64

static volatile int shutdown = 0;

static void sig_int(int signal) {
    shutdown = 1;
}

void handle_event(void *ctx, int cpu, void *data, u32 data_size) {
    const struct event *e = data;
    printf(OUTPUT_FORMAT, e->ts, (unsigned long) e->pid, e->tpid, e->comm, e->sig, e->uid);
}

void handle_lost_event(void *ctx, int cpu, u64 lost_cnt) {
    fprintf(stderr, "Lost %llu events on CPU #%d!\n", lost_cnt, cpu);
}

int main(int argc, char **argv) {

    int err;

    struct perf_buffer *perfBuffer;
    struct killsnoop_bpf *obj;

    obj = killsnoop_bpf__open_and_load();
    if (!obj) {
        fprintf(stderr, "failed to open/load BPF skeleton!");
        goto cleanup;
    }

    err = killsnoop_bpf__attach(obj);
    if (err) {
        fprintf(stderr, "failed to attach BPF programs\n");
        goto cleanup;
    }

    perfBuffer = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, handle_event, handle_lost_event,
                                  NULL, NULL);

    if (!perfBuffer) {
        err = -errno;
        fprintf(stderr, "failed to open perf buffer: %d\n", err);
        goto cleanup;
    }

    if (signal(SIGINT, sig_int) == SIG_ERR) {
        fprintf(stderr, "can't set signal handler: %s\n", strerror(errno));
        err = 1;
        goto cleanup;
    }

    printf("TIME        PID      TPID      COMMAND    SIGNAL    UID  \n");
    while (!shutdown) {
        err = perf_buffer__poll(perfBuffer, PERF_POLL_TIMEOUT_MS);
        if (err < 0 && err != -EINTR) {
            fprintf(stderr, "error polling perf buffer: %s\n", strerror(-err));
            goto cleanup;
        }
        /* reset err to return 0 if exiting */
        err = 0;
    }

    cleanup:
    perf_buffer__free(perfBuffer);
    killsnoop_bpf__destroy(obj);

    return err != 0;
}

Output

Also, when I execute kill -9 <pid> in another terminal, it's not captured in the output. Can someone please explain on what am I doing wrong?

[JackyYin]

I don't know why _x64_sys_kill didn't work in my environment, either, but I think you could try to use kill_something_info, it gave me a more reasonable output:

SEC("kprobe/kill_something_info")
int BPF_KPROBE(entry_probe, int sig, void *info, pid_t pid) {
    return probe_entry(ctx, pid, sig);
}

Or maybe you could use tp instead of kprobe, because you don't need struct pt_regs in probe_entry function:

struct sys_enter_kill_ctx {
    unsigned short common_type;
    unsigned char common_flags;
    unsigned char common_preempt_count;
    int common_pid;
    int __syscall_nr;
    unsigned long pid;
    int sig;
};

SEC("tp/syscalls/sys_enter_kill")
int handle_sys_enter_kill(struct sys_enter_kill_ctx *ctx) {
    return probe_entry(ctx->pid, ctx->sig);
}

[chenhengqi]

FYI, we have a dedicated tool in BCC repo for this purpose.

Also, when I execute kill -9 in another terminal, it's not captured in the output.

Linux has other syscalls for kill, eg. tkill. You should trace all kill-related syscalls.

[chenhengqi]

I'm getting some random values in place of the pid and signal that is used while execution.

You submit a struct data_t to perf buffer and interpret it as struct event ?

[syogaraj]

I'm getting some random values in place of the pid and signal that is used while execution.

You submit a struct data_t to perf buffer and interpret it as struct event ?

both the struct shares the same data fields in the same order.

[chenhengqi]

They need to be exactly the same for both padding and field size.

[syogaraj]

Linux has other syscalls for kill, eg. tkill. You should trace all kill-related syscalls.

I'm trying to trace the kill syscall alone so that I can get the actual time the parent process quits. Any ideas why the __x64_sys_kill was not being traced?

[anakryiko]

This might be due to CONFIG_ARCH_HAS_SYSCALL_WRAPPER. Latest libbpf supports a special kprobe for syscalls. Try SEC("ksyscall/<syscall-name>") and use BPF_KSYSCALL() macro to define BPF program. See https://github.com/torvalds/linux/blob/master/tools/testing/selftests/bpf/progs/test_attach_probe.c#L33-L38 for one example.

This combination should abstract away a bunch of architecture- and kernel version-specific details. Please give it a try and let us know if it works for you.

[cheneytianx]

Try this, to see if it works on your case.

SEC("kprobe/__x64_sys_kill")
int kprobe_kill(struct pt_regs *kprobe_ctx) {
    int pid, sig;
    struct pt_regs *ctx = (struct pt_regs *)PT_REGS_PARM1(kprobe_ctx);

    bpf_probe_read(&pid, sizeof(pid), &PT_REGS_PARM1(ctx));
    bpf_probe_read(&sig, sizeof(sig), &PT_REGS_PARM2(ctx));

    bpf_printk("pid: %d, sig: %d\n", pid, sig);
}

[anakryiko]

@syogaraj if your issue was fixed with one of the above suggestion, please report back and close the issue. Thank you!

[syogaraj]

@anakryiko I'm yet to try the suggested solution. Will check and report back at the earliest.

[anakryiko]

@syogaraj ping, did you get a chance to try SEC("ksyscall")?