#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "libbpf.h" #include "bpf_load.h" #include "perf-sys.h" #define DEBUGFS "/sys/kernel/debug/tracing/" static char license[128]; static int kern_version; static bool processed_sec[128]; char bpf_log_buf[BPF_LOG_BUF_SIZE]; int map_fd[MAX_MAPS]; int prog_fd[MAX_PROGS]; int event_fd[MAX_PROGS]; int prog_cnt; int prog_array_fd = -1; static int populate_prog_array(const char *event, int prog_fd) { int ind = atoi(event), err; err = bpf_map_update_elem(prog_array_fd, &ind, &prog_fd, BPF_ANY); if (err < 0) { printf("failed to store prog_fd in prog_array\n"); return -1; } return 0; } static int load_and_attach(const char *event, struct bpf_insn *prog, int size) { bool is_socket = strncmp(event, "socket", 6) == 0; bool is_kprobe = strncmp(event, "kprobe/", 7) == 0; bool is_kretprobe = strncmp(event, "kretprobe/", 10) == 0; bool is_tracepoint = strncmp(event, "tracepoint/", 11) == 0; bool is_xdp = strncmp(event, "xdp", 3) == 0; bool is_perf_event = strncmp(event, "perf_event", 10) == 0; bool is_cgroup_skb = strncmp(event, "cgroup/skb", 10) == 0; bool is_cgroup_sk = strncmp(event, "cgroup/sock", 11) == 0; size_t insns_cnt = size / sizeof(struct bpf_insn); enum bpf_prog_type prog_type; char buf[256]; int fd, efd, err, id; struct perf_event_attr attr = {}; attr.type = PERF_TYPE_TRACEPOINT; attr.sample_type = PERF_SAMPLE_RAW; attr.sample_period = 1; attr.wakeup_events = 1; if (is_socket) { prog_type = BPF_PROG_TYPE_SOCKET_FILTER; } else if (is_kprobe || is_kretprobe) { prog_type = BPF_PROG_TYPE_KPROBE; } else if (is_tracepoint) { prog_type = BPF_PROG_TYPE_TRACEPOINT; } else if (is_xdp) { prog_type = BPF_PROG_TYPE_XDP; } else if (is_perf_event) { prog_type = BPF_PROG_TYPE_PERF_EVENT; } else if (is_cgroup_skb) { prog_type = BPF_PROG_TYPE_CGROUP_SKB; } else if (is_cgroup_sk) { prog_type = BPF_PROG_TYPE_CGROUP_SOCK; } else { printf("Unknown event '%s'\n", event); return -1; } fd = bpf_load_program(prog_type, prog, insns_cnt, license, kern_version, bpf_log_buf, BPF_LOG_BUF_SIZE); if (fd < 0) { printf("bpf_load_program() err=%d\n%s", errno, bpf_log_buf); return -1; } prog_fd[prog_cnt++] = fd; if (is_xdp || is_perf_event || is_cgroup_skb || is_cgroup_sk) return 0; if (is_socket) { event += 6; if (*event != '/') return 0; event++; if (!isdigit(*event)) { printf("invalid prog number\n"); return -1; } return populate_prog_array(event, fd); } if (is_kprobe || is_kretprobe) { if (is_kprobe) event += 7; else event += 10; if (*event == 0) { printf("event name cannot be empty\n"); return -1; } if (isdigit(*event)) return populate_prog_array(event, fd); snprintf(buf, sizeof(buf), "echo '%c:%s %s' >> /sys/kernel/debug/tracing/kprobe_events", is_kprobe ? 'p' : 'r', event, event); err = system(buf); if (err < 0) { printf("failed to create kprobe '%s' error '%s'\n", event, strerror(errno)); return -1; } strcpy(buf, DEBUGFS); strcat(buf, "events/kprobes/"); strcat(buf, event); strcat(buf, "/id"); } else if (is_tracepoint) { event += 11; if (*event == 0) { printf("event name cannot be empty\n"); return -1; } strcpy(buf, DEBUGFS); strcat(buf, "events/"); strcat(buf, event); strcat(buf, "/id"); } efd = open(buf, O_RDONLY, 0); if (efd < 0) { printf("failed to open event %s\n", event); return -1; } err = read(efd, buf, sizeof(buf)); if (err < 0 || err >= sizeof(buf)) { printf("read from '%s' failed '%s'\n", event, strerror(errno)); return -1; } close(efd); buf[err] = 0; id = atoi(buf); attr.config = id; efd = sys_perf_event_open(&attr, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0); if (efd < 0) { printf("event %d fd %d err %s\n", id, efd, strerror(errno)); return -1; } event_fd[prog_cnt - 1] = efd; ioctl(efd, PERF_EVENT_IOC_ENABLE, 0); ioctl(efd, PERF_EVENT_IOC_SET_BPF, fd); return 0; } static int load_maps(struct bpf_map_def *maps, int len, const char **map_names, fixup_map_cb fixup_map) { int i; for (i = 0; i < len / sizeof(struct bpf_map_def); i++) { if (fixup_map) fixup_map(&maps[i], map_names[i], i); if (maps[i].type == BPF_MAP_TYPE_ARRAY_OF_MAPS || maps[i].type == BPF_MAP_TYPE_HASH_OF_MAPS) { int inner_map_fd = map_fd[maps[i].inner_map_idx]; map_fd[i] = bpf_create_map_in_map(maps[i].type, maps[i].key_size, inner_map_fd, maps[i].max_entries, maps[i].map_flags); } else { map_fd[i] = bpf_create_map(maps[i].type, maps[i].key_size, maps[i].value_size, maps[i].max_entries, maps[i].map_flags); } if (map_fd[i] < 0) { printf("failed to create a map: %d %s\n", errno, strerror(errno)); return 1; } if (maps[i].type == BPF_MAP_TYPE_PROG_ARRAY) prog_array_fd = map_fd[i]; } return 0; } static int get_sec(Elf *elf, int i, GElf_Ehdr *ehdr, char **shname, GElf_Shdr *shdr, Elf_Data **data) { Elf_Scn *scn; scn = elf_getscn(elf, i); if (!scn) return 1; if (gelf_getshdr(scn, shdr) != shdr) return 2; *shname = elf_strptr(elf, ehdr->e_shstrndx, shdr->sh_name); if (!*shname || !shdr->sh_size) return 3; *data = elf_getdata(scn, 0); if (!*data || elf_getdata(scn, *data) != NULL) return 4; return 0; } static int parse_relo_and_apply(Elf_Data *data, Elf_Data *symbols, GElf_Shdr *shdr, struct bpf_insn *insn) { int i, nrels; nrels = shdr->sh_size / shdr->sh_entsize; for (i = 0; i < nrels; i++) { GElf_Sym sym; GElf_Rel rel; unsigned int insn_idx; gelf_getrel(data, i, &rel); insn_idx = rel.r_offset / sizeof(struct bpf_insn); gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym); if (insn[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { printf("invalid relo for insn[%d].code 0x%x\n", insn_idx, insn[insn_idx].code); return 1; } insn[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; insn[insn_idx].imm = map_fd[sym.st_value / sizeof(struct bpf_map_def)]; } return 0; } static int cmp_symbols(const void *l, const void *r) { const GElf_Sym *lsym = (const GElf_Sym *)l; const GElf_Sym *rsym = (const GElf_Sym *)r; if (lsym->st_value < rsym->st_value) return -1; else if (lsym->st_value > rsym->st_value) return 1; else return 0; } static int get_sorted_map_names(Elf *elf, Elf_Data *symbols, int maps_shndx, int strtabidx, char **map_names) { GElf_Sym map_symbols[MAX_MAPS]; int i, nr_maps = 0; for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { assert(nr_maps < MAX_MAPS); if (!gelf_getsym(symbols, i, &map_symbols[nr_maps])) continue; if (map_symbols[nr_maps].st_shndx != maps_shndx) continue; nr_maps++; } qsort(map_symbols, nr_maps, sizeof(GElf_Sym), cmp_symbols); for (i = 0; i < nr_maps; i++) { char *map_name; map_name = elf_strptr(elf, strtabidx, map_symbols[i].st_name); if (!map_name) { printf("cannot get map symbol\n"); return 1; } map_names[i] = strdup(map_name); if (!map_names[i]) { printf("strdup(%s): %s(%d)\n", map_name, strerror(errno), errno); return 1; } } return 0; } static int do_load_bpf_file(const char *path, fixup_map_cb fixup_map) { int fd, i, ret, maps_shndx = -1, strtabidx = -1; Elf *elf; GElf_Ehdr ehdr; GElf_Shdr shdr, shdr_prog; Elf_Data *data, *data_prog, *data_maps = NULL, *symbols = NULL; char *shname, *shname_prog, *map_names[MAX_MAPS] = { NULL }; /* reset global variables */ kern_version = 0; memset(license, 0, sizeof(license)); memset(processed_sec, 0, sizeof(processed_sec)); if (elf_version(EV_CURRENT) == EV_NONE) return 1; fd = open(path, O_RDONLY, 0); if (fd < 0) return 1; elf = elf_begin(fd, ELF_C_READ, NULL); if (!elf) return 1; if (gelf_getehdr(elf, &ehdr) != &ehdr) return 1; /* clear all kprobes */ i = system("echo \"\" > /sys/kernel/debug/tracing/kprobe_events"); /* scan over all elf sections to get license and map info */ for (i = 1; i < ehdr.e_shnum; i++) { if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) continue; if (0) /* helpful for llvm debugging */ printf("section %d:%s data %p size %zd link %d flags %d\n", i, shname, data->d_buf, data->d_size, shdr.sh_link, (int) shdr.sh_flags); if (strcmp(shname, "license") == 0) { processed_sec[i] = true; memcpy(license, data->d_buf, data->d_size); } else if (strcmp(shname, "version") == 0) { processed_sec[i] = true; if (data->d_size != sizeof(int)) { printf("invalid size of version section %zd\n", data->d_size); return 1; } memcpy(&kern_version, data->d_buf, sizeof(int)); } else if (strcmp(shname, "maps") == 0) { maps_shndx = i; data_maps = data; } else if (shdr.sh_type == SHT_SYMTAB) { strtabidx = shdr.sh_link; symbols = data; } } ret = 1; if (!symbols) { printf("missing SHT_SYMTAB section\n"); goto done; } if (data_maps) { if (get_sorted_map_names(elf, symbols, maps_shndx, strtabidx, map_names)) goto done; if (load_maps(data_maps->d_buf, data_maps->d_size, (const char **)map_names, fixup_map)) goto done; processed_sec[maps_shndx] = true; } /* load programs that need map fixup (relocations) */ for (i = 1; i < ehdr.e_shnum; i++) { if (processed_sec[i]) continue; if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) continue; if (shdr.sh_type == SHT_REL) { struct bpf_insn *insns; if (get_sec(elf, shdr.sh_info, &ehdr, &shname_prog, &shdr_prog, &data_prog)) continue; if (shdr_prog.sh_type != SHT_PROGBITS || !(shdr_prog.sh_flags & SHF_EXECINSTR)) continue; insns = (struct bpf_insn *) data_prog->d_buf; processed_sec[shdr.sh_info] = true; processed_sec[i] = true; if (parse_relo_and_apply(data, symbols, &shdr, insns)) continue; if (memcmp(shname_prog, "kprobe/", 7) == 0 || memcmp(shname_prog, "kretprobe/", 10) == 0 || memcmp(shname_prog, "tracepoint/", 11) == 0 || memcmp(shname_prog, "xdp", 3) == 0 || memcmp(shname_prog, "perf_event", 10) == 0 || memcmp(shname_prog, "socket", 6) == 0 || memcmp(shname_prog, "cgroup/", 7) == 0) load_and_attach(shname_prog, insns, data_prog->d_size); } } /* load programs that don't use maps */ for (i = 1; i < ehdr.e_shnum; i++) { if (processed_sec[i]) continue; if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) continue; if (memcmp(shname, "kprobe/", 7) == 0 || memcmp(shname, "kretprobe/", 10) == 0 || memcmp(shname, "tracepoint/", 11) == 0 || memcmp(shname, "xdp", 3) == 0 || memcmp(shname, "perf_event", 10) == 0 || memcmp(shname, "socket", 6) == 0 || memcmp(shname, "cgroup/", 7) == 0) load_and_attach(shname, data->d_buf, data->d_size); } ret = 0; done: for (i = 0; i < MAX_MAPS; i++) free(map_names[i]); close(fd); return ret; } int load_bpf_file(char *path) { return do_load_bpf_file(path, NULL); } int load_bpf_file_fixup_map(const char *path, fixup_map_cb fixup_map) { return do_load_bpf_file(path, fixup_map); } void read_trace_pipe(void) { int trace_fd; trace_fd = open(DEBUGFS "trace_pipe", O_RDONLY, 0); if (trace_fd < 0) return; while (1) { static char buf[4096]; ssize_t sz; sz = read(trace_fd, buf, sizeof(buf)); if (sz > 0) { buf[sz] = 0; puts(buf); } } } #define MAX_SYMS 300000 static struct ksym syms[MAX_SYMS]; static int sym_cnt; static int ksym_cmp(const void *p1, const void *p2) { return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr; } int load_kallsyms(void) { FILE *f = fopen("/proc/kallsyms", "r"); char func[256], buf[256]; char symbol; void *addr; int i = 0; if (!f) return -ENOENT; while (!feof(f)) { if (!fgets(buf, sizeof(buf), f)) break; if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3) break; if (!addr) continue; syms[i].addr = (long) addr; syms[i].name = strdup(func); i++; } sym_cnt = i; qsort(syms, sym_cnt, sizeof(struct ksym), ksym_cmp); return 0; } struct ksym *ksym_search(long key) { int start = 0, end = sym_cnt; int result; while (start < end) { size_t mid = start + (end - start) / 2; result = key - syms[mid].addr; if (result < 0) end = mid; else if (result > 0) start = mid + 1; else return &syms[mid]; } if (start >= 1 && syms[start - 1].addr < key && key < syms[start].addr) /* valid ksym */ return &syms[start - 1]; /* out of range. return _stext */ return &syms[0]; } int set_link_xdp_fd(int ifindex, int fd) { struct sockaddr_nl sa; int sock, seq = 0, len, ret = -1; char buf[4096]; struct nlattr *nla, *nla_xdp; struct { struct nlmsghdr nh; struct ifinfomsg ifinfo; char attrbuf[64]; } req; struct nlmsghdr *nh; struct nlmsgerr *err; memset(&sa, 0, sizeof(sa)); sa.nl_family = AF_NETLINK; sock = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); if (sock < 0) { printf("open netlink socket: %s\n", strerror(errno)); return -1; } if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) { printf("bind to netlink: %s\n", strerror(errno)); goto cleanup; } memset(&req, 0, sizeof(req)); req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)); req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; req.nh.nlmsg_type = RTM_SETLINK; req.nh.nlmsg_pid = 0; req.nh.nlmsg_seq = ++seq; req.ifinfo.ifi_family = AF_UNSPEC; req.ifinfo.ifi_index = ifindex; nla = (struct nlattr *)(((char *)&req) + NLMSG_ALIGN(req.nh.nlmsg_len)); nla->nla_type = NLA_F_NESTED | 43/*IFLA_XDP*/; nla_xdp = (struct nlattr *)((char *)nla + NLA_HDRLEN); nla_xdp->nla_type = 1/*IFLA_XDP_FD*/; nla_xdp->nla_len = NLA_HDRLEN + sizeof(int); memcpy((char *)nla_xdp + NLA_HDRLEN, &fd, sizeof(fd)); nla->nla_len = NLA_HDRLEN + nla_xdp->nla_len; req.nh.nlmsg_len += NLA_ALIGN(nla->nla_len); if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { printf("send to netlink: %s\n", strerror(errno)); goto cleanup; } len = recv(sock, buf, sizeof(buf), 0); if (len < 0) { printf("recv from netlink: %s\n", strerror(errno)); goto cleanup; } for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, len); nh = NLMSG_NEXT(nh, len)) { if (nh->nlmsg_pid != getpid()) { printf("Wrong pid %d, expected %d\n", nh->nlmsg_pid, getpid()); goto cleanup; } if (nh->nlmsg_seq != seq) { printf("Wrong seq %d, expected %d\n", nh->nlmsg_seq, seq); goto cleanup; } switch (nh->nlmsg_type) { case NLMSG_ERROR: err = (struct nlmsgerr *)NLMSG_DATA(nh); if (!err->error) continue; printf("nlmsg error %s\n", strerror(-err->error)); goto cleanup; case NLMSG_DONE: break; } } ret = 0; cleanup: close(sock); return ret; }