* [PATCH v5 bpf-next 8/9] selftests/bpf: add kprobe/uprobe selftests
From: Andrii Nakryiko @ 2019-07-01 23:59 UTC (permalink / raw)
To: andrii.nakryiko, bpf, netdev, ast, daniel, kernel-team, yhs
Cc: Andrii Nakryiko
In-Reply-To: <20190701235903.660141-1-andriin@fb.com>
Add tests verifying kprobe/kretprobe/uprobe/uretprobe APIs work as
expected.
Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Reviewed-by: Stanislav Fomichev <sdf@google.com>
Acked-by: Song Liu <songliubraving@fb.com>
---
.../selftests/bpf/prog_tests/attach_probe.c | 166 ++++++++++++++++++
.../selftests/bpf/progs/test_attach_probe.c | 55 ++++++
2 files changed, 221 insertions(+)
create mode 100644 tools/testing/selftests/bpf/prog_tests/attach_probe.c
create mode 100644 tools/testing/selftests/bpf/progs/test_attach_probe.c
diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
new file mode 100644
index 000000000000..a4686395522c
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+
+ssize_t get_base_addr() {
+ size_t start;
+ char buf[256];
+ FILE *f;
+
+ f = fopen("/proc/self/maps", "r");
+ if (!f)
+ return -errno;
+
+ while (fscanf(f, "%zx-%*x %s %*s\n", &start, buf) == 2) {
+ if (strcmp(buf, "r-xp") == 0) {
+ fclose(f);
+ return start;
+ }
+ }
+
+ fclose(f);
+ return -EINVAL;
+}
+
+#ifdef __x86_64__
+#define SYS_KPROBE_NAME "__x64_sys_nanosleep"
+#else
+#define SYS_KPROBE_NAME "sys_nanosleep"
+#endif
+
+void test_attach_probe(void)
+{
+ const char *kprobe_name = "kprobe/sys_nanosleep";
+ const char *kretprobe_name = "kretprobe/sys_nanosleep";
+ const char *uprobe_name = "uprobe/trigger_func";
+ const char *uretprobe_name = "uretprobe/trigger_func";
+ const int kprobe_idx = 0, kretprobe_idx = 1;
+ const int uprobe_idx = 2, uretprobe_idx = 3;
+ const char *file = "./test_attach_probe.o";
+ struct bpf_program *kprobe_prog, *kretprobe_prog;
+ struct bpf_program *uprobe_prog, *uretprobe_prog;
+ struct bpf_object *obj;
+ int err, prog_fd, duration = 0, res;
+ struct bpf_link *kprobe_link = NULL;
+ struct bpf_link *kretprobe_link = NULL;
+ struct bpf_link *uprobe_link = NULL;
+ struct bpf_link *uretprobe_link = NULL;
+ int results_map_fd;
+ size_t uprobe_offset;
+ ssize_t base_addr;
+
+ base_addr = get_base_addr();
+ if (CHECK(base_addr < 0, "get_base_addr",
+ "failed to find base addr: %zd", base_addr))
+ return;
+ uprobe_offset = (size_t)&get_base_addr - base_addr;
+
+ /* load programs */
+ err = bpf_prog_load(file, BPF_PROG_TYPE_KPROBE, &obj, &prog_fd);
+ if (CHECK(err, "obj_load", "err %d errno %d\n", err, errno))
+ return;
+
+ kprobe_prog = bpf_object__find_program_by_title(obj, kprobe_name);
+ if (CHECK(!kprobe_prog, "find_probe",
+ "prog '%s' not found\n", kprobe_name))
+ goto cleanup;
+ kretprobe_prog = bpf_object__find_program_by_title(obj, kretprobe_name);
+ if (CHECK(!kretprobe_prog, "find_probe",
+ "prog '%s' not found\n", kretprobe_name))
+ goto cleanup;
+ uprobe_prog = bpf_object__find_program_by_title(obj, uprobe_name);
+ if (CHECK(!uprobe_prog, "find_probe",
+ "prog '%s' not found\n", uprobe_name))
+ goto cleanup;
+ uretprobe_prog = bpf_object__find_program_by_title(obj, uretprobe_name);
+ if (CHECK(!uretprobe_prog, "find_probe",
+ "prog '%s' not found\n", uretprobe_name))
+ goto cleanup;
+
+ /* load maps */
+ results_map_fd = bpf_find_map(__func__, obj, "results_map");
+ if (CHECK(results_map_fd < 0, "find_results_map",
+ "err %d\n", results_map_fd))
+ goto cleanup;
+
+ kprobe_link = bpf_program__attach_kprobe(kprobe_prog,
+ false /* retprobe */,
+ SYS_KPROBE_NAME);
+ if (CHECK(IS_ERR(kprobe_link), "attach_kprobe",
+ "err %ld\n", PTR_ERR(kprobe_link))) {
+ kprobe_link = NULL;
+ goto cleanup;
+ }
+ kretprobe_link = bpf_program__attach_kprobe(kretprobe_prog,
+ true /* retprobe */,
+ SYS_KPROBE_NAME);
+ if (CHECK(IS_ERR(kretprobe_link), "attach_kretprobe",
+ "err %ld\n", PTR_ERR(kretprobe_link))) {
+ kretprobe_link = NULL;
+ goto cleanup;
+ }
+ uprobe_link = bpf_program__attach_uprobe(uprobe_prog,
+ false /* retprobe */,
+ 0 /* self pid */,
+ "/proc/self/exe",
+ uprobe_offset);
+ if (CHECK(IS_ERR(uprobe_link), "attach_uprobe",
+ "err %ld\n", PTR_ERR(uprobe_link))) {
+ uprobe_link = NULL;
+ goto cleanup;
+ }
+ uretprobe_link = bpf_program__attach_uprobe(uretprobe_prog,
+ true /* retprobe */,
+ -1 /* any pid */,
+ "/proc/self/exe",
+ uprobe_offset);
+ if (CHECK(IS_ERR(uretprobe_link), "attach_uretprobe",
+ "err %ld\n", PTR_ERR(uretprobe_link))) {
+ uretprobe_link = NULL;
+ goto cleanup;
+ }
+
+ /* trigger & validate kprobe && kretprobe */
+ usleep(1);
+
+ err = bpf_map_lookup_elem(results_map_fd, &kprobe_idx, &res);
+ if (CHECK(err, "get_kprobe_res",
+ "failed to get kprobe res: %d\n", err))
+ goto cleanup;
+ if (CHECK(res != kprobe_idx + 1, "check_kprobe_res",
+ "wrong kprobe res: %d\n", res))
+ goto cleanup;
+
+ err = bpf_map_lookup_elem(results_map_fd, &kretprobe_idx, &res);
+ if (CHECK(err, "get_kretprobe_res",
+ "failed to get kretprobe res: %d\n", err))
+ goto cleanup;
+ if (CHECK(res != kretprobe_idx + 1, "check_kretprobe_res",
+ "wrong kretprobe res: %d\n", res))
+ goto cleanup;
+
+ /* trigger & validate uprobe & uretprobe */
+ get_base_addr();
+
+ err = bpf_map_lookup_elem(results_map_fd, &uprobe_idx, &res);
+ if (CHECK(err, "get_uprobe_res",
+ "failed to get uprobe res: %d\n", err))
+ goto cleanup;
+ if (CHECK(res != uprobe_idx + 1, "check_uprobe_res",
+ "wrong uprobe res: %d\n", res))
+ goto cleanup;
+
+ err = bpf_map_lookup_elem(results_map_fd, &uretprobe_idx, &res);
+ if (CHECK(err, "get_uretprobe_res",
+ "failed to get uretprobe res: %d\n", err))
+ goto cleanup;
+ if (CHECK(res != uretprobe_idx + 1, "check_uretprobe_res",
+ "wrong uretprobe res: %d\n", res))
+ goto cleanup;
+
+cleanup:
+ bpf_link__destroy(kprobe_link);
+ bpf_link__destroy(kretprobe_link);
+ bpf_link__destroy(uprobe_link);
+ bpf_link__destroy(uretprobe_link);
+ bpf_object__close(obj);
+}
diff --git a/tools/testing/selftests/bpf/progs/test_attach_probe.c b/tools/testing/selftests/bpf/progs/test_attach_probe.c
new file mode 100644
index 000000000000..7a7c5cd728c8
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_attach_probe.c
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017 Facebook
+
+#include <linux/ptrace.h>
+#include <linux/bpf.h>
+#include "bpf_helpers.h"
+
+struct {
+ int type;
+ int max_entries;
+ int *key;
+ int *value;
+} results_map SEC(".maps") = {
+ .type = BPF_MAP_TYPE_ARRAY,
+ .max_entries = 4,
+};
+
+SEC("kprobe/sys_nanosleep")
+int handle_sys_nanosleep_entry(struct pt_regs *ctx)
+{
+ const int key = 0, value = 1;
+
+ bpf_map_update_elem(&results_map, &key, &value, 0);
+ return 0;
+}
+
+SEC("kretprobe/sys_nanosleep")
+int handle_sys_getpid_return(struct pt_regs *ctx)
+{
+ const int key = 1, value = 2;
+
+ bpf_map_update_elem(&results_map, &key, &value, 0);
+ return 0;
+}
+
+SEC("uprobe/trigger_func")
+int handle_uprobe_entry(struct pt_regs *ctx)
+{
+ const int key = 2, value = 3;
+
+ bpf_map_update_elem(&results_map, &key, &value, 0);
+ return 0;
+}
+
+SEC("uretprobe/trigger_func")
+int handle_uprobe_return(struct pt_regs *ctx)
+{
+ const int key = 3, value = 4;
+
+ bpf_map_update_elem(&results_map, &key, &value, 0);
+ return 0;
+}
+
+char _license[] SEC("license") = "GPL";
+__u32 _version SEC("version") = 1;
--
2.17.1
^ permalink raw reply related
* [PATCH v5 bpf-next 7/9] selftests/bpf: switch test to new attach_perf_event API
From: Andrii Nakryiko @ 2019-07-01 23:59 UTC (permalink / raw)
To: andrii.nakryiko, bpf, netdev, ast, daniel, kernel-team, yhs
Cc: Andrii Nakryiko
In-Reply-To: <20190701235903.660141-1-andriin@fb.com>
Use new bpf_program__attach_perf_event() in test previously relying on
direct ioctl manipulations.
Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Reviewed-by: Stanislav Fomichev <sdf@google.com>
Acked-by: Song Liu <songliubraving@fb.com>
---
.../bpf/prog_tests/stacktrace_build_id_nmi.c | 31 +++++++++----------
1 file changed, 15 insertions(+), 16 deletions(-)
diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c
index 1c1a2f75f3d8..9557b7dfb782 100644
--- a/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c
+++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c
@@ -17,6 +17,7 @@ static __u64 read_perf_max_sample_freq(void)
void test_stacktrace_build_id_nmi(void)
{
int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
+ const char *prog_name = "tracepoint/random/urandom_read";
const char *file = "./test_stacktrace_build_id.o";
int err, pmu_fd, prog_fd;
struct perf_event_attr attr = {
@@ -25,7 +26,9 @@ void test_stacktrace_build_id_nmi(void)
.config = PERF_COUNT_HW_CPU_CYCLES,
};
__u32 key, previous_key, val, duration = 0;
+ struct bpf_program *prog;
struct bpf_object *obj;
+ struct bpf_link *link;
char buf[256];
int i, j;
struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
@@ -39,6 +42,10 @@ void test_stacktrace_build_id_nmi(void)
if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
return;
+ prog = bpf_object__find_program_by_title(obj, prog_name);
+ if (CHECK(!prog, "find_prog", "prog '%s' not found\n", prog_name))
+ goto close_prog;
+
pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
0 /* cpu 0 */, -1 /* group id */,
0 /* flags */);
@@ -47,15 +54,12 @@ void test_stacktrace_build_id_nmi(void)
pmu_fd, errno))
goto close_prog;
- err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
- if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
- err, errno))
- goto close_pmu;
-
- err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
- if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
- err, errno))
- goto disable_pmu;
+ link = bpf_program__attach_perf_event(prog, pmu_fd);
+ if (CHECK(IS_ERR(link), "attach_perf_event",
+ "err %ld\n", PTR_ERR(link))) {
+ close(pmu_fd);
+ goto close_prog;
+ }
/* find map fds */
control_map_fd = bpf_find_map(__func__, obj, "control_map");
@@ -134,8 +138,7 @@ void test_stacktrace_build_id_nmi(void)
* try it one more time.
*/
if (build_id_matches < 1 && retry--) {
- ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
- close(pmu_fd);
+ bpf_link__destroy(link);
bpf_object__close(obj);
printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
__func__);
@@ -154,11 +157,7 @@ void test_stacktrace_build_id_nmi(void)
*/
disable_pmu:
- ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
-
-close_pmu:
- close(pmu_fd);
-
+ bpf_link__destroy(link);
close_prog:
bpf_object__close(obj);
}
--
2.17.1
^ permalink raw reply related
* [PATCH v5 bpf-next 2/9] libbpf: introduce concept of bpf_link
From: Andrii Nakryiko @ 2019-07-01 23:58 UTC (permalink / raw)
To: andrii.nakryiko, bpf, netdev, ast, daniel, kernel-team, yhs
Cc: Andrii Nakryiko
In-Reply-To: <20190701235903.660141-1-andriin@fb.com>
bpf_link is an abstraction of an association of a BPF program and one of
many possible BPF attachment points (hooks). This allows to have uniform
interface for detaching BPF programs regardless of the nature of link
and how it was created. Details of creation and setting up of a specific
bpf_link is handled by corresponding attachment methods
(bpf_program__attach_xxx) added in subsequent commits. Once successfully
created, bpf_link has to be eventually destroyed with
bpf_link__destroy(), at which point BPF program is disassociated from
a hook and all the relevant resources are freed.
Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Acked-by: Song Liu <songliubraving@fb.com>
Reviewed-by: Stanislav Fomichev <sdf@google.com>
---
tools/lib/bpf/libbpf.c | 17 +++++++++++++++++
tools/lib/bpf/libbpf.h | 4 ++++
tools/lib/bpf/libbpf.map | 3 ++-
3 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 6e6ebef11ba3..455795e6f8af 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -3941,6 +3941,23 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
return 0;
}
+struct bpf_link {
+ int (*destroy)(struct bpf_link *link);
+};
+
+int bpf_link__destroy(struct bpf_link *link)
+{
+ int err;
+
+ if (!link)
+ return 0;
+
+ err = link->destroy(link);
+ free(link);
+
+ return err;
+}
+
enum bpf_perf_event_ret
bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
void **copy_mem, size_t *copy_size,
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index d639f47e3110..5082a5ebb0c2 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -165,6 +165,10 @@ LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path);
LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path);
LIBBPF_API void bpf_program__unload(struct bpf_program *prog);
+struct bpf_link;
+
+LIBBPF_API int bpf_link__destroy(struct bpf_link *link);
+
struct bpf_insn;
/*
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 2c6d835620d2..3cde850fc8da 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -167,10 +167,11 @@ LIBBPF_0.0.3 {
LIBBPF_0.0.4 {
global:
+ bpf_link__destroy;
+ bpf_object__load_xattr;
btf_dump__dump_type;
btf_dump__free;
btf_dump__new;
btf__parse_elf;
- bpf_object__load_xattr;
libbpf_num_possible_cpus;
} LIBBPF_0.0.3;
--
2.17.1
^ permalink raw reply related
* [PATCH v5 bpf-next 3/9] libbpf: add ability to attach/detach BPF program to perf event
From: Andrii Nakryiko @ 2019-07-01 23:58 UTC (permalink / raw)
To: andrii.nakryiko, bpf, netdev, ast, daniel, kernel-team, yhs
Cc: Andrii Nakryiko
In-Reply-To: <20190701235903.660141-1-andriin@fb.com>
bpf_program__attach_perf_event allows to attach BPF program to existing
perf event hook, providing most generic and most low-level way to attach BPF
programs. It returns struct bpf_link, which should be passed to
bpf_link__destroy to detach and free resources, associated with a link.
Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Reviewed-by: Stanislav Fomichev <sdf@google.com>
---
tools/lib/bpf/libbpf.c | 63 ++++++++++++++++++++++++++++++++++++++++
tools/lib/bpf/libbpf.h | 3 ++
tools/lib/bpf/libbpf.map | 1 +
3 files changed, 67 insertions(+)
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 455795e6f8af..bcaa294f819d 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -32,6 +32,7 @@
#include <linux/limits.h>
#include <linux/perf_event.h>
#include <linux/ring_buffer.h>
+#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
@@ -3958,6 +3959,68 @@ int bpf_link__destroy(struct bpf_link *link)
return err;
}
+struct bpf_link_fd {
+ struct bpf_link link; /* has to be at the top of struct */
+ int fd; /* hook FD */
+};
+
+static int bpf_link__destroy_perf_event(struct bpf_link *link)
+{
+ struct bpf_link_fd *l = (void *)link;
+ int err;
+
+ err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0);
+ if (err)
+ err = -errno;
+
+ close(l->fd);
+ return err;
+}
+
+struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
+ int pfd)
+{
+ char errmsg[STRERR_BUFSIZE];
+ struct bpf_link_fd *link;
+ int prog_fd, err;
+
+ if (pfd < 0) {
+ pr_warning("program '%s': invalid perf event FD %d\n",
+ bpf_program__title(prog, false), pfd);
+ return ERR_PTR(-EINVAL);
+ }
+ prog_fd = bpf_program__fd(prog);
+ if (prog_fd < 0) {
+ pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
+ bpf_program__title(prog, false));
+ return ERR_PTR(-EINVAL);
+ }
+
+ link = malloc(sizeof(*link));
+ if (!link)
+ return ERR_PTR(-ENOMEM);
+ link->link.destroy = &bpf_link__destroy_perf_event;
+ link->fd = pfd;
+
+ if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
+ err = -errno;
+ free(link);
+ pr_warning("program '%s': failed to attach to pfd %d: %s\n",
+ bpf_program__title(prog, false), pfd,
+ libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
+ return ERR_PTR(err);
+ }
+ if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
+ err = -errno;
+ free(link);
+ pr_warning("program '%s': failed to enable pfd %d: %s\n",
+ bpf_program__title(prog, false), pfd,
+ libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
+ return ERR_PTR(err);
+ }
+ return (struct bpf_link *)link;
+}
+
enum bpf_perf_event_ret
bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
void **copy_mem, size_t *copy_size,
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 5082a5ebb0c2..1bf66c4a9330 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -169,6 +169,9 @@ struct bpf_link;
LIBBPF_API int bpf_link__destroy(struct bpf_link *link);
+LIBBPF_API struct bpf_link *
+bpf_program__attach_perf_event(struct bpf_program *prog, int pfd);
+
struct bpf_insn;
/*
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 3cde850fc8da..756f5aa802e9 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -169,6 +169,7 @@ LIBBPF_0.0.4 {
global:
bpf_link__destroy;
bpf_object__load_xattr;
+ bpf_program__attach_perf_event;
btf_dump__dump_type;
btf_dump__free;
btf_dump__new;
--
2.17.1
^ permalink raw reply related
* [PATCH v5 bpf-next 1/9] libbpf: make libbpf_strerror_r agnostic to sign of error
From: Andrii Nakryiko @ 2019-07-01 23:58 UTC (permalink / raw)
To: andrii.nakryiko, bpf, netdev, ast, daniel, kernel-team, yhs
Cc: Andrii Nakryiko
In-Reply-To: <20190701235903.660141-1-andriin@fb.com>
It's often inconvenient to switch sign of error when passing it into
libbpf_strerror_r. It's better for it to handle that automatically.
Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Reviewed-by: Stanislav Fomichev <sdf@google.com>
Acked-by: Song Liu <songliubraving@fb.com>
---
tools/lib/bpf/str_error.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/lib/bpf/str_error.c b/tools/lib/bpf/str_error.c
index 00e48ac5b806..b8064eedc177 100644
--- a/tools/lib/bpf/str_error.c
+++ b/tools/lib/bpf/str_error.c
@@ -11,7 +11,7 @@
*/
char *libbpf_strerror_r(int err, char *dst, int len)
{
- int ret = strerror_r(err, dst, len);
+ int ret = strerror_r(err < 0 ? -err : err, dst, len);
if (ret)
snprintf(dst, len, "ERROR: strerror_r(%d)=%d", err, ret);
return dst;
--
2.17.1
^ permalink raw reply related
* Re: net: check before dereferencing netdev_ops during busy poll
From: Josh Elsasser @ 2019-07-01 23:52 UTC (permalink / raw)
To: Matteo Croce
Cc: Greg Kroah-Hartman, Sasha Levin, stable, netdev, LKML,
David Miller
In-Reply-To: <CAGnkfhx3ykbEsW+=FtpMFWU=_Vnie7RpPYWpWqa1S1HPMXj9kw@mail.gmail.com>
On Jul 1, 2019, at 11:03 AM, Matteo Croce <mcroce@redhat.com> wrote:
> Josh, as you are the original author, can you please resend it to -stable?
> Feel free to add this tag:
>
> Tested-by: Matteo Croce <mcroce@redhat.com>
For sure. Resent with your Tested-by, along with a second patch that applies
to the 4.4.y LTS kernel.
I'm still a little hazy on how net fixes work for older LTS releases, so I
hope I've sent these properly. I can respin if necessary.
^ permalink raw reply
* [PATCH 4.4.y] net: check before dereferencing netdev_ops during busy poll
From: Josh Elsasser @ 2019-07-01 23:48 UTC (permalink / raw)
To: stable
Cc: Josh Elsasser, gregkh, netdev, David S. Miller, Jiri Pirko,
Edward Cree, Eric Dumazet, Alexander Duyck, Matteo Croce
init_dummy_netdev() leaves its netdev_ops pointer zeroed. This leads
to a NULL pointer dereference when sk_busy_loop fires against an iwlwifi
wireless adapter and checks napi->dev->netdev_ops->ndo_busy_poll.
Avoid this by ensuring napi->dev->netdev_ops is valid before following
the pointer, avoiding the following panic when busy polling on a dummy
netdev:
BUG: unable to handle kernel NULL pointer dereference at 00000000000000c8
IP: [<ffffffff817b4b72>] sk_busy_loop+0x92/0x2f0
Call Trace:
[<ffffffff815a3134>] ? uart_write_room+0x74/0xf0
[<ffffffff817964a9>] sock_poll+0x99/0xa0
[<ffffffff81223142>] do_sys_poll+0x2e2/0x520
[<ffffffff8118d3fc>] ? get_page_from_freelist+0x3bc/0xa30
[<ffffffff810ada22>] ? update_curr+0x62/0x140
[<ffffffff811ea671>] ? __slab_free+0xa1/0x2a0
[<ffffffff811ea671>] ? __slab_free+0xa1/0x2a0
[<ffffffff8179dbb1>] ? skb_free_head+0x21/0x30
[<ffffffff81221bd0>] ? poll_initwait+0x50/0x50
[<ffffffff811eaa36>] ? kmem_cache_free+0x1c6/0x1e0
[<ffffffff815a4884>] ? uart_write+0x124/0x1d0
[<ffffffff810bd1cd>] ? remove_wait_queue+0x4d/0x60
[<ffffffff810bd224>] ? __wake_up+0x44/0x50
[<ffffffff81582731>] ? tty_write_unlock+0x31/0x40
[<ffffffff8158c5c6>] ? tty_ldisc_deref+0x16/0x20
[<ffffffff81584820>] ? tty_write+0x1e0/0x2f0
[<ffffffff81587e50>] ? process_echoes+0x80/0x80
[<ffffffff8120c17b>] ? __vfs_write+0x2b/0x130
[<ffffffff8120d09a>] ? vfs_write+0x15a/0x1a0
[<ffffffff81223455>] SyS_poll+0x75/0x100
[<ffffffff819a6524>] entry_SYSCALL_64_fastpath+0x24/0xcf
Commit 79e7fff47b7b ("net: remove support for per driver ndo_busy_poll()")
indirectly fixed this upstream in linux-4.11 by removing the offending
pointer usage. No other users of napi->dev touch its netdev_ops.
Fixes: 8b80cda536ea ("net: rename include/net/ll_poll.h to include/net/busy_poll.h") # 4.4.y
Signed-off-by: Josh Elsasser <jelsasser@appneta.com>
---
This is a straightforward backport of the 4.9.y fix[1] for this crash, which doesn't
apply to the older LTS releases. Only build-tested on 4.4.y, as I don't have access
to wireless hardware and firmware that runs on older LTS kernels.
[1]: https://lore.kernel.org/stable/20190701234143.72631-1-jelsasser@appneta.com/T/#u
include/net/busy_poll.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h
index 1d67fb6b23a0..6d238506d49b 100644
--- a/include/net/busy_poll.h
+++ b/include/net/busy_poll.h
@@ -93,7 +93,7 @@ static inline bool sk_busy_loop(struct sock *sk, int nonblock)
goto out;
ops = napi->dev->netdev_ops;
- if (!ops->ndo_busy_poll)
+ if (!ops || !ops->ndo_busy_poll)
goto out;
do {
--
2.20.1
^ permalink raw reply related
* Re: [PATCH mlx5-next 00/18] Mellanox, mlx5 E-Switch and low level updates
From: Saeed Mahameed @ 2019-07-01 23:46 UTC (permalink / raw)
To: Saeed Mahameed
Cc: Leon Romanovsky, netdev@vger.kernel.org,
linux-rdma@vger.kernel.org
In-Reply-To: <20190628223516.9368-1-saeedm@mellanox.com>
On Fri, Jun 28, 2019 at 3:35 PM Saeed Mahameed <saeedm@mellanox.com> wrote:
>
> Hi All,
>
> This series includes some low level updates mainly in the E-Switch
> netdev and rdma vport representors areas.
>
> From Parav and Huy:
> 1) Added hardware bits and structures definitions for sub-functions
> 2) Small code cleanup and improvement for PF pci driver.
>
> From Bodong:
> 3) Use the correct name semantics of vport index and vport number
> 4) Cleanup the rep and netdev reference when unloading IB rep.
> 5) Bluefield (ECPF) updates and refactoring for better E-Switch
> management on ECPF embedded CPU NIC:
> 5.1) Consolidate querying eswitch number of VFs
> 5.2) Register event handler at the correct E-Switch init stage
> 5.3) Setup PF's inline mode and vlan pop when the ECPF is the
> E-Swtich manager ( the host PF is basically a VF ).
> 5.4) Handle Vport UC address changes in switchdev mode.
>
> From Shay:
> 6) Add support for MCQI and MCQS hardware registers.
>
> In case of no objections these patches will be applied to mlx5-next and
> will be sent later as pull request to both rdma-next and net-next trees.
Applied to mlx5-next,
Thanks!
^ permalink raw reply
* [PATCH RESEND 4.9.y] net: check before dereferencing netdev_ops during busy poll
From: Josh Elsasser @ 2019-07-01 23:41 UTC (permalink / raw)
To: stable
Cc: Josh Elsasser, gregkh, netdev, David S. Miller, Eric Dumazet,
Matteo Croce
init_dummy_netdev() leaves its netdev_ops pointer zeroed. This leads
to a NULL pointer dereference when sk_busy_loop fires against an iwlwifi
wireless adapter and checks napi->dev->netdev_ops->ndo_busy_poll.
Avoid this by ensuring napi->dev->netdev_ops is valid before following
the pointer, avoiding the following panic when busy polling on a dummy
netdev:
BUG: unable to handle kernel NULL pointer dereference at 00000000000000c8
IP: [<ffffffff817b4b72>] sk_busy_loop+0x92/0x2f0
Call Trace:
[<ffffffff815a3134>] ? uart_write_room+0x74/0xf0
[<ffffffff817964a9>] sock_poll+0x99/0xa0
[<ffffffff81223142>] do_sys_poll+0x2e2/0x520
[<ffffffff8118d3fc>] ? get_page_from_freelist+0x3bc/0xa30
[<ffffffff810ada22>] ? update_curr+0x62/0x140
[<ffffffff811ea671>] ? __slab_free+0xa1/0x2a0
[<ffffffff811ea671>] ? __slab_free+0xa1/0x2a0
[<ffffffff8179dbb1>] ? skb_free_head+0x21/0x30
[<ffffffff81221bd0>] ? poll_initwait+0x50/0x50
[<ffffffff811eaa36>] ? kmem_cache_free+0x1c6/0x1e0
[<ffffffff815a4884>] ? uart_write+0x124/0x1d0
[<ffffffff810bd1cd>] ? remove_wait_queue+0x4d/0x60
[<ffffffff810bd224>] ? __wake_up+0x44/0x50
[<ffffffff81582731>] ? tty_write_unlock+0x31/0x40
[<ffffffff8158c5c6>] ? tty_ldisc_deref+0x16/0x20
[<ffffffff81584820>] ? tty_write+0x1e0/0x2f0
[<ffffffff81587e50>] ? process_echoes+0x80/0x80
[<ffffffff8120c17b>] ? __vfs_write+0x2b/0x130
[<ffffffff8120d09a>] ? vfs_write+0x15a/0x1a0
[<ffffffff81223455>] SyS_poll+0x75/0x100
[<ffffffff819a6524>] entry_SYSCALL_64_fastpath+0x24/0xcf
Commit 79e7fff47b7b ("net: remove support for per driver ndo_busy_poll()")
indirectly fixed this upstream in linux-4.11 by removing the offending
pointer usage. No other users of napi->dev touch its netdev_ops.
Fixes: ce6aea93f751 ("net: network drivers no longer need to implement ndo_busy_poll()") # 4.9.y
Signed-off-by: Josh Elsasser <jelsasser@appneta.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Tested-by: Matteo Croce <mcroce@redhat.com>
---
No changes since V2[1], resent as per discussiond on -stable[2]. I hope
this is the correct way to send net fixes for older LTS releases, I'm
going off of the latest netdev FAQ:
For earlier stable releases, each stable branch maintainer is supposed
to take care of them. If you find any patch is missing from an earlier
stable branch, please notify stable@vger.kernel.org with either a commit
ID or a formal patch backported, and CC Dave and other relevant networking
developers.
[1]: https://patchwork.ozlabs.org/patch/884986/
[2]: https://lore.kernel.org/stable/CAGnkfhx3ykbEsW+=FtpMFWU=_Vnie7RpPYWpWqa1S1HPMXj9kw@mail.gmail.com/
net/core/dev.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/net/core/dev.c b/net/core/dev.c
index 4e10bae5e3da..f693afe608d7 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5083,7 +5083,10 @@ bool sk_busy_loop(struct sock *sk, int nonblock)
goto out;
/* Note: ndo_busy_poll method is optional in linux-4.5 */
- busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
+ if (napi->dev->netdev_ops)
+ busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
+ else
+ busy_poll = NULL;
do {
rc = 0;
--
2.20.1
^ permalink raw reply related
* Re: [PATCH bpf-next 6/8] selftests/bpf: test BPF_SOCK_OPS_RTT_CB
From: Y Song @ 2019-07-01 23:40 UTC (permalink / raw)
To: Stanislav Fomichev
Cc: netdev, bpf, David Miller, Alexei Starovoitov, Daniel Borkmann,
Eric Dumazet, Priyaranjan Jha, Yuchung Cheng,
Soheil Hassas Yeganeh
In-Reply-To: <20190701204821.44230-7-sdf@google.com>
On Mon, Jul 1, 2019 at 1:49 PM Stanislav Fomichev <sdf@google.com> wrote:
>
> Make sure the callback is invoked for syn-ack and data packet.
>
> Cc: Eric Dumazet <edumazet@google.com>
> Cc: Priyaranjan Jha <priyarjha@google.com>
> Cc: Yuchung Cheng <ycheng@google.com>
> Cc: Soheil Hassas Yeganeh <soheil@google.com>
> Signed-off-by: Stanislav Fomichev <sdf@google.com>
> ---
> tools/testing/selftests/bpf/Makefile | 3 +-
> tools/testing/selftests/bpf/progs/tcp_rtt.c | 61 +++++
> tools/testing/selftests/bpf/test_tcp_rtt.c | 253 ++++++++++++++++++++
> 3 files changed, 316 insertions(+), 1 deletion(-)
> create mode 100644 tools/testing/selftests/bpf/progs/tcp_rtt.c
> create mode 100644 tools/testing/selftests/bpf/test_tcp_rtt.c
>
> diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
> index de1754a8f5fe..2620406a53ec 100644
> --- a/tools/testing/selftests/bpf/Makefile
> +++ b/tools/testing/selftests/bpf/Makefile
> @@ -27,7 +27,7 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test
> test_cgroup_storage test_select_reuseport test_section_names \
> test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \
> test_btf_dump test_cgroup_attach xdping test_sockopt test_sockopt_sk \
> - test_sockopt_multi
> + test_sockopt_multi test_tcp_rtt
>
> BPF_OBJ_FILES = $(patsubst %.c,%.o, $(notdir $(wildcard progs/*.c)))
> TEST_GEN_FILES = $(BPF_OBJ_FILES)
> @@ -107,6 +107,7 @@ $(OUTPUT)/test_cgroup_attach: cgroup_helpers.c
> $(OUTPUT)/test_sockopt: cgroup_helpers.c
> $(OUTPUT)/test_sockopt_sk: cgroup_helpers.c
> $(OUTPUT)/test_sockopt_multi: cgroup_helpers.c
> +$(OUTPUT)/test_tcp_rtt: cgroup_helpers.c
>
> .PHONY: force
>
> diff --git a/tools/testing/selftests/bpf/progs/tcp_rtt.c b/tools/testing/selftests/bpf/progs/tcp_rtt.c
> new file mode 100644
> index 000000000000..233bdcb1659e
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/progs/tcp_rtt.c
> @@ -0,0 +1,61 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include <linux/bpf.h>
> +#include "bpf_helpers.h"
> +
> +char _license[] SEC("license") = "GPL";
> +__u32 _version SEC("version") = 1;
> +
> +struct tcp_rtt_storage {
> + __u32 invoked;
> + __u32 dsack_dups;
> + __u32 delivered;
> + __u32 delivered_ce;
> + __u32 icsk_retransmits;
> +};
> +
> +struct bpf_map_def SEC("maps") socket_storage_map = {
> + .type = BPF_MAP_TYPE_SK_STORAGE,
> + .key_size = sizeof(int),
> + .value_size = sizeof(struct tcp_rtt_storage),
> + .map_flags = BPF_F_NO_PREALLOC,
> +};
> +BPF_ANNOTATE_KV_PAIR(socket_storage_map, int, struct tcp_rtt_storage);
> +
> +SEC("sockops")
> +int _sockops(struct bpf_sock_ops *ctx)
> +{
> + struct tcp_rtt_storage *storage;
> + struct bpf_tcp_sock *tcp_sk;
> + int op = (int) ctx->op;
> + struct bpf_sock *sk;
> +
> + sk = ctx->sk;
> + if (!sk)
> + return 1;
> +
> + storage = bpf_sk_storage_get(&socket_storage_map, sk, 0,
> + BPF_SK_STORAGE_GET_F_CREATE);
> + if (!storage)
> + return 1;
> +
> + if (op == BPF_SOCK_OPS_TCP_CONNECT_CB) {
> + bpf_sock_ops_cb_flags_set(ctx, BPF_SOCK_OPS_RTT_CB_FLAG);
> + return 1;
> + }
> +
> + if (op != BPF_SOCK_OPS_RTT_CB)
> + return 1;
> +
> + tcp_sk = bpf_tcp_sock(sk);
> + if (!tcp_sk)
> + return 1;
> +
> + storage->invoked++;
> +
> + storage->dsack_dups = tcp_sk->dsack_dups;
> + storage->delivered = tcp_sk->delivered;
> + storage->delivered_ce = tcp_sk->delivered_ce;
> + storage->icsk_retransmits = tcp_sk->icsk_retransmits;
> +
> + return 1;
> +}
> diff --git a/tools/testing/selftests/bpf/test_tcp_rtt.c b/tools/testing/selftests/bpf/test_tcp_rtt.c
> new file mode 100644
> index 000000000000..413fd8514adc
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/test_tcp_rtt.c
> @@ -0,0 +1,253 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include <error.h>
> +#include <errno.h>
> +#include <stdio.h>
> +#include <unistd.h>
> +#include <sys/types.h>
> +#include <sys/socket.h>
> +#include <netinet/in.h>
> +#include <pthread.h>
> +
> +#include <linux/filter.h>
> +#include <bpf/bpf.h>
> +#include <bpf/libbpf.h>
> +
> +#include "bpf_rlimit.h"
> +#include "bpf_util.h"
> +#include "cgroup_helpers.h"
> +
> +#define CG_PATH "/tcp_rtt"
> +
> +struct tcp_rtt_storage {
> + __u32 invoked;
> + __u32 dsack_dups;
> + __u32 delivered;
> + __u32 delivered_ce;
> + __u32 icsk_retransmits;
> +};
> +
> +static void send_byte(int fd)
> +{
> + char b = 0x55;
> +
> + if (write(fd, &b, sizeof(b)) != 1)
> + error(1, errno, "Failed to send single byte");
> +}
> +
> +static int verify_sk(int map_fd, int client_fd, const char *msg, __u32 invoked,
> + __u32 dsack_dups, __u32 delivered, __u32 delivered_ce,
> + __u32 icsk_retransmits)
> +{
> + int err = 0;
> + struct tcp_rtt_storage val;
> +
> + if (bpf_map_lookup_elem(map_fd, &client_fd, &val) < 0)
> + error(1, errno, "Failed to read socket storage");
> +
> + if (val.invoked != invoked) {
> + log_err("%s: unexpected bpf_tcp_sock.invoked %d != %d",
> + msg, val.invoked, invoked);
> + err++;
> + }
> +
> + if (val.dsack_dups != dsack_dups) {
> + log_err("%s: unexpected bpf_tcp_sock.dsack_dups %d != %d",
> + msg, val.dsack_dups, dsack_dups);
> + err++;
> + }
> +
> + if (val.delivered != delivered) {
> + log_err("%s: unexpected bpf_tcp_sock.delivered %d != %d",
> + msg, val.delivered, delivered);
> + err++;
> + }
> +
> + if (val.delivered_ce != delivered_ce) {
> + log_err("%s: unexpected bpf_tcp_sock.delivered_ce %d != %d",
> + msg, val.delivered_ce, delivered_ce);
> + err++;
> + }
> +
> + if (val.icsk_retransmits != icsk_retransmits) {
> + log_err("%s: unexpected bpf_tcp_sock.icsk_retransmits %d != %d",
> + msg, val.icsk_retransmits, icsk_retransmits);
> + err++;
> + }
> +
> + return err;
> +}
> +
> +static int connect_to_server(int server_fd)
> +{
> + struct sockaddr_storage addr;
> + socklen_t len = sizeof(addr);
> + int fd;
> +
> + fd = socket(AF_INET, SOCK_STREAM, 0);
> + if (fd < 0) {
> + log_err("Failed to create client socket");
> + return -1;
> + }
> +
> + if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) {
> + log_err("Failed to get server addr");
> + goto out;
> + }
> +
> + if (connect(fd, (const struct sockaddr *)&addr, len) < 0) {
> + log_err("Fail to connect to server");
> + goto out;
> + }
> +
> + return fd;
> +
> +out:
> + close(fd);
> + return -1;
> +}
> +
> +static int run_test(int cgroup_fd, int server_fd)
> +{
> + struct bpf_prog_load_attr attr = {
> + .prog_type = BPF_PROG_TYPE_SOCK_OPS,
> + .file = "./tcp_rtt.o",
> + .expected_attach_type = BPF_CGROUP_SOCK_OPS,
> + };
> + struct bpf_program *prog;
> + struct bpf_object *obj;
> + struct bpf_map *map;
> + int client_fd;
> + int ignored;
> + int map_fd;
> + int err;
> +
> + err = bpf_prog_load_xattr(&attr, &obj, &ignored);
> + if (err) {
> + log_err("Failed to load BPF object");
> + return -1;
> + }
The third argument of bpf_prog_load_xattr is prog_fd.
If you have it, you do not need the below retrieving prog_fd
by bpf_program__fd(prog).
> +
> + map = bpf_map__next(NULL, obj);
> + map_fd = bpf_map__fd(map);
> +
> + prog = bpf_program__next(NULL, obj);
> + err = bpf_prog_attach(bpf_program__fd(prog), cgroup_fd,
> + BPF_CGROUP_SOCK_OPS, 0);
> + if (err) {
> + log_err("Failed to attach BPF program");
> + goto close_bpf_object;
> + }
> +
> + client_fd = connect_to_server(server_fd);
> + if (client_fd < 0) {
> + err = -1;
> + goto close_bpf_object;
> + }
> +
> + err += verify_sk(map_fd, client_fd, "syn-ack",
> + /*invoked=*/1,
> + /*dsack_dups=*/0,
> + /*delivered=*/1,
> + /*delivered_ce=*/0,
> + /*icsk_retransmits=*/0);
> +
> + send_byte(client_fd);
> +
> + err += verify_sk(map_fd, client_fd, "first payload byte",
> + /*invoked=*/2,
> + /*dsack_dups=*/0,
> + /*delivered=*/2,
> + /*delivered_ce=*/0,
> + /*icsk_retransmits=*/0);
> +
> + close(client_fd);
> +
> +close_bpf_object:
> + bpf_object__close(obj);
> + return err;
> +}
> +
> +static int start_server(void)
> +{
> + struct sockaddr_in addr = {
> + .sin_family = AF_INET,
> + .sin_addr.s_addr = htonl(INADDR_LOOPBACK),
> + };
> + int fd;
> +
> + fd = socket(AF_INET, SOCK_STREAM, 0);
> + if (fd < 0) {
> + log_err("Failed to create server socket");
> + return -1;
> + }
> +
> + if (bind(fd, (const struct sockaddr *)&addr, sizeof(addr)) < 0) {
> + log_err("Failed to bind socket");
> + close(fd);
> + return -1;
> + }
> +
> + return fd;
> +}
> +
> +static void *server_thread(void *arg)
> +{
> + struct sockaddr_storage addr;
> + socklen_t len = sizeof(addr);
> + int fd = *(int *)arg;
> + int client_fd;
> +
> + if (listen(fd, 1) < 0)
> + error(1, errno, "Failed to listed on socket");
The error() here only reports the error, right? In case of error,
should the control jumps to the end of this function and return?
The same for several error() calls below.
> +
> + client_fd = accept(fd, (struct sockaddr *)&addr, &len);
> + if (client_fd < 0)
> + error(1, errno, "Failed to accept client");
> +
> + if (accept(fd, (struct sockaddr *)&addr, &len) >= 0)
> + error(1, errno, "Unexpected success in second accept");
What is the purpose of this second default to-be-failed accept() call?
> +
> + close(client_fd);
> +
> + return NULL;
> +}
> +
> +int main(int args, char **argv)
> +{
> + int server_fd, cgroup_fd;
> + int err = EXIT_SUCCESS;
> + pthread_t tid;
> +
> + if (setup_cgroup_environment())
> + goto cleanup_obj;
> +
> + cgroup_fd = create_and_get_cgroup(CG_PATH);
> + if (cgroup_fd < 0)
> + goto cleanup_cgroup_env;
> +
> + if (join_cgroup(CG_PATH))
> + goto cleanup_cgroup;
> +
> + server_fd = start_server();
> + if (server_fd < 0) {
> + err = EXIT_FAILURE;
> + goto cleanup_cgroup;
> + }
> +
> + pthread_create(&tid, NULL, server_thread, (void *)&server_fd);
> +
> + if (run_test(cgroup_fd, server_fd))
> + err = EXIT_FAILURE;
> +
> + close(server_fd);
> +
> + printf("test_sockopt_sk: %s\n",
> + err == EXIT_SUCCESS ? "PASSED" : "FAILED");
> +
> +cleanup_cgroup:
> + close(cgroup_fd);
> +cleanup_cgroup_env:
> + cleanup_cgroup_environment();
> +cleanup_obj:
> + return err;
> +}
> --
> 2.22.0.410.gd8fdbe21b5-goog
>
^ permalink raw reply
* Re: [PATCH net-next v3 0/4] Add gve driver
From: Catherine Sullivan @ 2019-07-01 23:37 UTC (permalink / raw)
To: Jakub Kicinski; +Cc: netdev
In-Reply-To: <20190629112829.698d2d8b@cakuba.netronome.com>
Done in v4, thanks!
On Sat, Jun 29, 2019 at 11:28 AM Jakub Kicinski
<jakub.kicinski@netronome.com> wrote:
>
> On Fri, 28 Jun 2019 16:07:29 -0700, Catherine Sullivan wrote:
> > This patch series adds the gve driver which will support the
> > Compute Engine Virtual NIC that will be available in the future.
>
> Looks like buildbot also found a bunch of bad endian casts.
> Please make sure the driver builds with no warnings with W=1 C=1.
^ permalink raw reply
* Re: [net-next 4/4] gve: Add ethtool support
From: Catherine Sullivan @ 2019-07-01 23:37 UTC (permalink / raw)
To: Florian Fainelli
Cc: netdev, Sagi Shahar, Jon Olson, Willem de Bruijn, Luigi Rizzo
In-Reply-To: <efa71935-3ec3-0df9-2d9c-25cb9e134a40@gmail.com>
On Fri, Jun 28, 2019 at 7:44 PM Florian Fainelli <f.fainelli@gmail.com> wrote:
>
>
>
> On 6/26/2019 11:52 AM, Catherine Sullivan wrote:
>
> [snip]
>
> > +static void
> > +gve_get_ethtool_stats(struct net_device *netdev,
> > + struct ethtool_stats *stats, u64 *data)
> > +{
> > + struct gve_priv *priv = netdev_priv(netdev);
> > + u64 rx_pkts, rx_bytes, tx_pkts, tx_bytes;
> > + int ring;
> > + int i;
> > +
> > + ASSERT_RTNL();
> > +
> > + if (!netif_carrier_ok(netdev))
> > + return;
> > +
> > + for (rx_pkts = 0, rx_bytes = 0, ring = 0;
> > + ring < priv->rx_cfg.num_queues; ring++) {
> > + rx_pkts += priv->rx[ring].rpackets;
> > + rx_bytes += priv->rx[ring].rbytes;
> > + }
> > + for (tx_pkts = 0, tx_bytes = 0, ring = 0;
> > + ring < priv->tx_cfg.num_queues; ring++) {
> > + tx_pkts += priv->tx[ring].pkt_done;
> > + tx_bytes += priv->tx[ring].bytes_done;
> > + }
>
> Maybe you do not need to support 32-bit guests with that driver, but you
> might as well be correct and use the include/linux/u64_stats_sync.h
> primitives to help return consistent 64-bit stats on 32-bit machines.
Done in v4, thanks.
>
> [snip]
>
> > +int gve_adjust_queues(struct gve_priv *priv,
> > + struct gve_queue_config new_rx_config,
> > + struct gve_queue_config new_tx_config)
> > +{
> > + int err;
> > +
> > + if (netif_carrier_ok(priv->dev)) {
>
> Should not that be netif_running()?
> --
> Florian
^ permalink raw reply
* Re: [PATCH net-next 1/3] devlink: Introduce PCI PF port flavour and port attribute
From: Jakub Kicinski @ 2019-07-01 23:26 UTC (permalink / raw)
To: jiri; +Cc: Parav Pandit, netdev, saeedm
In-Reply-To: <20190701122734.18770-2-parav@mellanox.com>
On Mon, 1 Jul 2019 07:27:32 -0500, Parav Pandit wrote:
> In an eswitch, PCI PF may have port which is normally represented
> using a representor netdevice.
> To have better visibility of eswitch port, its association with
> PF, a representor netdevice and port number, introduce a PCI PF port
> flavour and port attriute.
>
> When devlink port flavour is PCI PF, fill up PCI PF attributes of the
> port.
>
> Extend port name creation using PCI PF number on best effort basis.
> So that vendor drivers can skip defining their own scheme.
>
> $ devlink port show
> pci/0000:05:00.0/0: type eth netdev eth0 flavour pcipf pfnum 0
>
> Acked-by: Jiri Pirko <jiri@mellanox.com>
> Signed-off-by: Parav Pandit <parav@mellanox.com>
> ---
> include/net/devlink.h | 11 ++++++
> include/uapi/linux/devlink.h | 5 +++
> net/core/devlink.c | 71 +++++++++++++++++++++++++++++-------
> 3 files changed, 73 insertions(+), 14 deletions(-)
>
> diff --git a/include/net/devlink.h b/include/net/devlink.h
> index 6625ea068d5e..8db9c0e83fb5 100644
> --- a/include/net/devlink.h
> +++ b/include/net/devlink.h
> @@ -38,6 +38,10 @@ struct devlink {
> char priv[0] __aligned(NETDEV_ALIGN);
> };
>
> +struct devlink_port_pci_pf_attrs {
Why the named structure? Anonymous one should be just fine?
> + u16 pf; /* Associated PCI PF for this port. */
> +};
> +
> struct devlink_port_attrs {
> u8 set:1,
> split:1,
> @@ -46,6 +50,9 @@ struct devlink_port_attrs {
> u32 port_number; /* same value as "split group" */
> u32 split_subport_number;
> struct netdev_phys_item_id switch_id;
> + union {
> + struct devlink_port_pci_pf_attrs pci_pf;
> + };
> };
>
> struct devlink_port {
> @@ -590,6 +597,10 @@ void devlink_port_attrs_set(struct devlink_port *devlink_port,
> u32 split_subport_number,
> const unsigned char *switch_id,
> unsigned char switch_id_len);
> +void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port,
> + u32 port_number,
> + const unsigned char *switch_id,
> + unsigned char switch_id_len, u16 pf);
> int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
> u32 size, u16 ingress_pools_count,
> u16 egress_pools_count, u16 ingress_tc_count,
> diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h
> index 5287b42c181f..f7323884c3fe 100644
> --- a/include/uapi/linux/devlink.h
> +++ b/include/uapi/linux/devlink.h
> @@ -169,6 +169,10 @@ enum devlink_port_flavour {
> DEVLINK_PORT_FLAVOUR_DSA, /* Distributed switch architecture
> * interconnect port.
> */
> + DEVLINK_PORT_FLAVOUR_PCI_PF, /* Represents eswitch port for
> + * the PCI PF. It is an internal
> + * port that faces the PCI PF.
> + */
> };
>
> enum devlink_param_cmode {
> @@ -337,6 +341,7 @@ enum devlink_attr {
> DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE, /* u64 */
> DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL, /* u64 */
>
> + DEVLINK_ATTR_PORT_PCI_PF_NUMBER, /* u16 */
> /* add new attributes above here, update the policy in devlink.c */
>
> __DEVLINK_ATTR_MAX,
> diff --git a/net/core/devlink.c b/net/core/devlink.c
> index 89c533778135..001f9e2c96f0 100644
> --- a/net/core/devlink.c
> +++ b/net/core/devlink.c
> @@ -517,6 +517,11 @@ static int devlink_nl_port_attrs_put(struct sk_buff *msg,
> return -EMSGSIZE;
> if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER, attrs->port_number))
> return -EMSGSIZE;
Why would we report network port information for PF and VF port
flavours?
> + if (devlink_port->attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_PF) {
> + if (nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
> + attrs->pci_pf.pf))
> + return -EMSGSIZE;
> + }
> if (!attrs->split)
> return 0;
> if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP, attrs->port_number))
^ permalink raw reply
* Re: [PATCH v4 bpf-next 6/9] libbpf: add raw tracepoint attach API
From: Andrii Nakryiko @ 2019-07-01 23:09 UTC (permalink / raw)
To: Yonghong Song
Cc: Andrii Nakryiko, bpf@vger.kernel.org, netdev@vger.kernel.org,
Alexei Starovoitov, daniel@iogearbox.net, Kernel Team,
sdf@fomichev.me, Song Liu
In-Reply-To: <cf314380-a96e-eae6-6fb5-b136c50cec71@fb.com>
On Mon, Jul 1, 2019 at 4:04 PM Yonghong Song <yhs@fb.com> wrote:
>
>
>
> On 7/1/19 3:26 PM, Andrii Nakryiko wrote:
> > On Mon, Jul 1, 2019 at 10:13 AM Yonghong Song <yhs@fb.com> wrote:
> >>
> >>
> >>
> >> On 6/28/19 8:49 PM, Andrii Nakryiko wrote:
> >>> Add a wrapper utilizing bpf_link "infrastructure" to allow attaching BPF
> >>> programs to raw tracepoints.
> >>>
> >>> Signed-off-by: Andrii Nakryiko <andriin@fb.com>
> >>> Acked-by: Song Liu <songliubraving@fb.com>
> >>> ---
> >>> tools/lib/bpf/libbpf.c | 37 +++++++++++++++++++++++++++++++++++++
> >>> tools/lib/bpf/libbpf.h | 3 +++
> >>> tools/lib/bpf/libbpf.map | 1 +
> >>> 3 files changed, 41 insertions(+)
> >>>
> >>> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
> >>> index 8ad4f915df38..f8c7a7ecb35e 100644
> >>> --- a/tools/lib/bpf/libbpf.c
> >>> +++ b/tools/lib/bpf/libbpf.c
> >>> @@ -4263,6 +4263,43 @@ struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
> >>> return link;
> >>> }
> >>>
> >>> +static int bpf_link__destroy_fd(struct bpf_link *link)
> >>> +{
> >>> + struct bpf_link_fd *l = (void *)link;
> >>> +
> >>> + return close(l->fd);
> >>> +}
> >>> +
> >>> +struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
> >>> + const char *tp_name)
> >>> +{
> >>> + char errmsg[STRERR_BUFSIZE];
> >>> + struct bpf_link_fd *link;
> >>> + int prog_fd, pfd;
> >>> +
> >>> + prog_fd = bpf_program__fd(prog);
> >>> + if (prog_fd < 0) {
> >>> + pr_warning("program '%s': can't attach before loaded\n",
> >>> + bpf_program__title(prog, false));
> >>> + return ERR_PTR(-EINVAL);
> >>> + }
> >>> +
> >>> + link = malloc(sizeof(*link));
> >>> + link->link.destroy = &bpf_link__destroy_fd;
> >>
> >> You can move the "link = malloc(...)" etc. after
> >> bpf_raw_tracepoint_open(). That way, you do not need to free(link)
> >> in the error case.
> >
> > It's either `free(link)` here, or `close(pfd)` if malloc fails after
> > we attached program. Either way extra clean up is needed. I went with
> > the first one.
>
> Okay with me.
> BTW, do you want to check whether malloc() may failure and link may be NULL?
Oh, you have a fresh eye! ;) fixed.
>
> >>
> >>> +
> >>> + pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
> >>> + if (pfd < 0) {
> >>> + pfd = -errno;
> >>> + free(link);
> >>> + pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n",
> >>> + bpf_program__title(prog, false), tp_name,
> >>> + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
> >>> + return ERR_PTR(pfd);
> >>> + }
> >>> + link->fd = pfd;
> >>> + return (struct bpf_link *)link;
> >>> +}
> >>> +
> >>> enum bpf_perf_event_ret
> >>> bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
> >>> void **copy_mem, size_t *copy_size,
> >>> diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
> >>> index 60611f4b4e1d..f55933784f95 100644
> >>> --- a/tools/lib/bpf/libbpf.h
> >>> +++ b/tools/lib/bpf/libbpf.h
> >>> @@ -182,6 +182,9 @@ LIBBPF_API struct bpf_link *
> >>> bpf_program__attach_tracepoint(struct bpf_program *prog,
> >>> const char *tp_category,
> >>> const char *tp_name);
> >>> +LIBBPF_API struct bpf_link *
> >>> +bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
> >>> + const char *tp_name);
> >>>
> >>> struct bpf_insn;
> >>>
> >>> diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
> >>> index 3c618b75ef65..e6b7d4edbc93 100644
> >>> --- a/tools/lib/bpf/libbpf.map
> >>> +++ b/tools/lib/bpf/libbpf.map
> >>> @@ -171,6 +171,7 @@ LIBBPF_0.0.4 {
> >>> bpf_object__load_xattr;
> >>> bpf_program__attach_kprobe;
> >>> bpf_program__attach_perf_event;
> >>> + bpf_program__attach_raw_tracepoint;
> >>> bpf_program__attach_tracepoint;
> >>> bpf_program__attach_uprobe;
> >>> btf_dump__dump_type;
> >>>
^ permalink raw reply
* Re: [PATCH v4 bpf-next 7/9] selftests/bpf: switch test to new attach_perf_event API
From: Yonghong Song @ 2019-07-01 23:05 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Andrii Nakryiko, bpf@vger.kernel.org, netdev@vger.kernel.org,
Alexei Starovoitov, daniel@iogearbox.net, Kernel Team,
sdf@fomichev.me, Song Liu
In-Reply-To: <CAEf4BzbDP=e+jVUBJjCUpPCewxp7-Uwq9L5TuPfUzn9j9MxUeg@mail.gmail.com>
On 7/1/19 3:32 PM, Andrii Nakryiko wrote:
> On Mon, Jul 1, 2019 at 10:16 AM Yonghong Song <yhs@fb.com> wrote:
>>
>>
>>
>> On 6/28/19 8:49 PM, Andrii Nakryiko wrote:
>>> Use new bpf_program__attach_perf_event() in test previously relying on
>>> direct ioctl manipulations.
>>>
>>> Signed-off-by: Andrii Nakryiko <andriin@fb.com>
>>> Reviewed-by: Stanislav Fomichev <sdf@google.com>
>>> Acked-by: Song Liu <songliubraving@fb.com>
>>> ---
>>> .../bpf/prog_tests/stacktrace_build_id_nmi.c | 31 +++++++++----------
>>> 1 file changed, 15 insertions(+), 16 deletions(-)
>>>
>>> diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c
>>> index 1c1a2f75f3d8..9557b7dfb782 100644
>>> --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c
>>> +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_build_id_nmi.c
>>> @@ -17,6 +17,7 @@ static __u64 read_perf_max_sample_freq(void)
>>> void test_stacktrace_build_id_nmi(void)
>>> {
>>> int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
>>> + const char *prog_name = "tracepoint/random/urandom_read";
>>> const char *file = "./test_stacktrace_build_id.o";
>>> int err, pmu_fd, prog_fd;
>>> struct perf_event_attr attr = {
>>> @@ -25,7 +26,9 @@ void test_stacktrace_build_id_nmi(void)
>>> .config = PERF_COUNT_HW_CPU_CYCLES,
>>> };
>>> __u32 key, previous_key, val, duration = 0;
>>> + struct bpf_program *prog;
>>> struct bpf_object *obj;
>>> + struct bpf_link *link;
>>> char buf[256];
>>> int i, j;
>>> struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
>>> @@ -39,6 +42,10 @@ void test_stacktrace_build_id_nmi(void)
>>> if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
>>> return;
>>>
>>> + prog = bpf_object__find_program_by_title(obj, prog_name);
>>> + if (CHECK(!prog, "find_prog", "prog '%s' not found\n", prog_name))
>>> + goto close_prog;
>>> +
>>> pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
>>> 0 /* cpu 0 */, -1 /* group id */,
>>> 0 /* flags */);
>>> @@ -47,15 +54,12 @@ void test_stacktrace_build_id_nmi(void)
>>> pmu_fd, errno))
>>> goto close_prog;
>>>
>>> - err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
>>> - if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
>>> - err, errno))
>>> - goto close_pmu;
>>> -
>>> - err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
>>> - if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
>>> - err, errno))
>>> - goto disable_pmu;
>>> + link = bpf_program__attach_perf_event(prog, pmu_fd);
>>> + if (CHECK(IS_ERR(link), "attach_perf_event",
>>> + "err %ld\n", PTR_ERR(link))) {
>>> + close(pmu_fd);
>>> + goto close_prog;
>>> + }
>>>
>>> /* find map fds */
>>> control_map_fd = bpf_find_map(__func__, obj, "control_map");
>>> @@ -134,8 +138,7 @@ void test_stacktrace_build_id_nmi(void)
>>> * try it one more time.
>>> */
>>> if (build_id_matches < 1 && retry--) {
>>> - ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
>>> - close(pmu_fd);
>>> + bpf_link__destroy(link);
>>> bpf_object__close(obj);
>>> printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
>>> __func__);
>>> @@ -154,11 +157,7 @@ void test_stacktrace_build_id_nmi(void)
>>> */
>>>
>>> disable_pmu:
>>> - ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
>>> -
>>> -close_pmu:
>>> - close(pmu_fd);
>>> -
>>> + bpf_link__destroy(link);
>>
>> There is a problem in bpf_link__destroy(link).
>> The "link = bpf_program__attach_perf_event(prog, pmu_fd)"
>> may be an error pointer (IS_ERR(link) is true), in which
>> case, link should be reset to NULL and then call
>> bpf_link__destroy(link). Otherwise, the program may
>> segfault or function incorrectly.
>
> Not really, if bpf_program__attach_perf_event fails and IS_ERR(link)
> is true, we'll close pmu_fd explicitly and `goto close_prog` bypassing
> bpf_link__destroy. `goto disable_pmu` is done only after we
> successfully established attached link.
>
> So unless I still miss something, I think this will work reliably.
Double checked again. You are correct. We do not have issues here.
>
>>
>>> close_prog:
>>> bpf_object__close(obj);
>>> }
>>>
^ permalink raw reply
* Re: [PATCH v4 bpf-next 6/9] libbpf: add raw tracepoint attach API
From: Yonghong Song @ 2019-07-01 23:04 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Andrii Nakryiko, bpf@vger.kernel.org, netdev@vger.kernel.org,
Alexei Starovoitov, daniel@iogearbox.net, Kernel Team,
sdf@fomichev.me, Song Liu
In-Reply-To: <CAEf4BzYRpgE5VPwuv2zkXZ2N9BQVPASvYbtsZDM10C1kwdX3eg@mail.gmail.com>
On 7/1/19 3:26 PM, Andrii Nakryiko wrote:
> On Mon, Jul 1, 2019 at 10:13 AM Yonghong Song <yhs@fb.com> wrote:
>>
>>
>>
>> On 6/28/19 8:49 PM, Andrii Nakryiko wrote:
>>> Add a wrapper utilizing bpf_link "infrastructure" to allow attaching BPF
>>> programs to raw tracepoints.
>>>
>>> Signed-off-by: Andrii Nakryiko <andriin@fb.com>
>>> Acked-by: Song Liu <songliubraving@fb.com>
>>> ---
>>> tools/lib/bpf/libbpf.c | 37 +++++++++++++++++++++++++++++++++++++
>>> tools/lib/bpf/libbpf.h | 3 +++
>>> tools/lib/bpf/libbpf.map | 1 +
>>> 3 files changed, 41 insertions(+)
>>>
>>> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
>>> index 8ad4f915df38..f8c7a7ecb35e 100644
>>> --- a/tools/lib/bpf/libbpf.c
>>> +++ b/tools/lib/bpf/libbpf.c
>>> @@ -4263,6 +4263,43 @@ struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
>>> return link;
>>> }
>>>
>>> +static int bpf_link__destroy_fd(struct bpf_link *link)
>>> +{
>>> + struct bpf_link_fd *l = (void *)link;
>>> +
>>> + return close(l->fd);
>>> +}
>>> +
>>> +struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
>>> + const char *tp_name)
>>> +{
>>> + char errmsg[STRERR_BUFSIZE];
>>> + struct bpf_link_fd *link;
>>> + int prog_fd, pfd;
>>> +
>>> + prog_fd = bpf_program__fd(prog);
>>> + if (prog_fd < 0) {
>>> + pr_warning("program '%s': can't attach before loaded\n",
>>> + bpf_program__title(prog, false));
>>> + return ERR_PTR(-EINVAL);
>>> + }
>>> +
>>> + link = malloc(sizeof(*link));
>>> + link->link.destroy = &bpf_link__destroy_fd;
>>
>> You can move the "link = malloc(...)" etc. after
>> bpf_raw_tracepoint_open(). That way, you do not need to free(link)
>> in the error case.
>
> It's either `free(link)` here, or `close(pfd)` if malloc fails after
> we attached program. Either way extra clean up is needed. I went with
> the first one.
Okay with me.
BTW, do you want to check whether malloc() may failure and link may be NULL?
>>
>>> +
>>> + pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
>>> + if (pfd < 0) {
>>> + pfd = -errno;
>>> + free(link);
>>> + pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n",
>>> + bpf_program__title(prog, false), tp_name,
>>> + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
>>> + return ERR_PTR(pfd);
>>> + }
>>> + link->fd = pfd;
>>> + return (struct bpf_link *)link;
>>> +}
>>> +
>>> enum bpf_perf_event_ret
>>> bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
>>> void **copy_mem, size_t *copy_size,
>>> diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
>>> index 60611f4b4e1d..f55933784f95 100644
>>> --- a/tools/lib/bpf/libbpf.h
>>> +++ b/tools/lib/bpf/libbpf.h
>>> @@ -182,6 +182,9 @@ LIBBPF_API struct bpf_link *
>>> bpf_program__attach_tracepoint(struct bpf_program *prog,
>>> const char *tp_category,
>>> const char *tp_name);
>>> +LIBBPF_API struct bpf_link *
>>> +bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
>>> + const char *tp_name);
>>>
>>> struct bpf_insn;
>>>
>>> diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
>>> index 3c618b75ef65..e6b7d4edbc93 100644
>>> --- a/tools/lib/bpf/libbpf.map
>>> +++ b/tools/lib/bpf/libbpf.map
>>> @@ -171,6 +171,7 @@ LIBBPF_0.0.4 {
>>> bpf_object__load_xattr;
>>> bpf_program__attach_kprobe;
>>> bpf_program__attach_perf_event;
>>> + bpf_program__attach_raw_tracepoint;
>>> bpf_program__attach_tracepoint;
>>> bpf_program__attach_uprobe;
>>> btf_dump__dump_type;
>>>
^ permalink raw reply
* Re: [PATCH v4 bpf-next 8/9] selftests/bpf: add kprobe/uprobe selftests
From: Andrii Nakryiko @ 2019-07-01 23:02 UTC (permalink / raw)
To: Yonghong Song
Cc: Andrii Nakryiko, bpf@vger.kernel.org, netdev@vger.kernel.org,
Alexei Starovoitov, daniel@iogearbox.net, Kernel Team,
sdf@fomichev.me, Song Liu
In-Reply-To: <8c79e0b2-1194-ab60-fda9-62ba4bbe019f@fb.com>
On Mon, Jul 1, 2019 at 10:18 AM Yonghong Song <yhs@fb.com> wrote:
>
>
>
> On 6/28/19 8:49 PM, Andrii Nakryiko wrote:
> > Add tests verifying kprobe/kretprobe/uprobe/uretprobe APIs work as
> > expected.
> >
> > Signed-off-by: Andrii Nakryiko <andriin@fb.com>
> > Reviewed-by: Stanislav Fomichev <sdf@google.com>
> > Acked-by: Song Liu <songliubraving@fb.com>
> > ---
> > .../selftests/bpf/prog_tests/attach_probe.c | 155 ++++++++++++++++++
> > .../selftests/bpf/progs/test_attach_probe.c | 55 +++++++
> > 2 files changed, 210 insertions(+)
> > create mode 100644 tools/testing/selftests/bpf/prog_tests/attach_probe.c
> > create mode 100644 tools/testing/selftests/bpf/progs/test_attach_probe.c
> >
> > diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
> > new file mode 100644
> > index 000000000000..f22929063c58
> > --- /dev/null
> > +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
> > @@ -0,0 +1,155 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +#include <test_progs.h>
> > +
> > +ssize_t get_base_addr() {
> > + size_t start;
> > + char buf[256];
> > + FILE *f;
> > +
> > + f = fopen("/proc/self/maps", "r");
> > + if (!f)
> > + return -errno;
> > +
> > + while (fscanf(f, "%zx-%*x %s %*s\n", &start, buf) == 2) {
> > + if (strcmp(buf, "r-xp") == 0) {
> > + fclose(f);
> > + return start;
> > + }
> > + }
> > +
> > + fclose(f);
> > + return -EINVAL;
> > +}
> > +
> > +void test_attach_probe(void)
> > +{
> > + const char *kprobe_name = "kprobe/sys_nanosleep";
> > + const char *kretprobe_name = "kretprobe/sys_nanosleep";
> > + const char *uprobe_name = "uprobe/trigger_func";
> > + const char *uretprobe_name = "uretprobe/trigger_func";
> > + const int kprobe_idx = 0, kretprobe_idx = 1;
> > + const int uprobe_idx = 2, uretprobe_idx = 3;
> > + const char *file = "./test_attach_probe.o";
> > + struct bpf_program *kprobe_prog, *kretprobe_prog;
> > + struct bpf_program *uprobe_prog, *uretprobe_prog;
> > + struct bpf_object *obj;
> > + int err, prog_fd, duration = 0, res;
> > + struct bpf_link *kprobe_link = NULL;
> > + struct bpf_link *kretprobe_link = NULL;
> > + struct bpf_link *uprobe_link = NULL;
> > + struct bpf_link *uretprobe_link = NULL;
> > + int results_map_fd;
> > + size_t uprobe_offset;
> > + ssize_t base_addr;
> > +
> > + base_addr = get_base_addr();
> > + if (CHECK(base_addr < 0, "get_base_addr",
> > + "failed to find base addr: %zd", base_addr))
> > + return;
> > + uprobe_offset = (size_t)&get_base_addr - base_addr;
> > +
> > + /* load programs */
> > + err = bpf_prog_load(file, BPF_PROG_TYPE_KPROBE, &obj, &prog_fd);
> > + if (CHECK(err, "obj_load", "err %d errno %d\n", err, errno))
> > + return;
> > +
> > + kprobe_prog = bpf_object__find_program_by_title(obj, kprobe_name);
> > + if (CHECK(!kprobe_prog, "find_probe",
> > + "prog '%s' not found\n", kprobe_name))
> > + goto cleanup;
> > + kretprobe_prog = bpf_object__find_program_by_title(obj, kretprobe_name);
> > + if (CHECK(!kretprobe_prog, "find_probe",
> > + "prog '%s' not found\n", kretprobe_name))
> > + goto cleanup;
> > + uprobe_prog = bpf_object__find_program_by_title(obj, uprobe_name);
> > + if (CHECK(!uprobe_prog, "find_probe",
> > + "prog '%s' not found\n", uprobe_name))
> > + goto cleanup;
> > + uretprobe_prog = bpf_object__find_program_by_title(obj, uretprobe_name);
> > + if (CHECK(!uretprobe_prog, "find_probe",
> > + "prog '%s' not found\n", uretprobe_name))
> > + goto cleanup;
> > +
> > + /* load maps */
> > + results_map_fd = bpf_find_map(__func__, obj, "results_map");
> > + if (CHECK(results_map_fd < 0, "find_results_map",
> > + "err %d\n", results_map_fd))
> > + goto cleanup;
> > +
> > + kprobe_link = bpf_program__attach_kprobe(kprobe_prog,
> > + false /* retprobe */,
> > + "sys_nanosleep");
> > + if (CHECK(IS_ERR(kprobe_link), "attach_kprobe",
> > + "err %ld\n", PTR_ERR(kprobe_link)))
> > + goto cleanup;
> > +
> > + kretprobe_link = bpf_program__attach_kprobe(kretprobe_prog,
> > + true /* retprobe */,
> > + "sys_nanosleep");
> > + if (CHECK(IS_ERR(kretprobe_link), "attach_kretprobe",
> > + "err %ld\n", PTR_ERR(kretprobe_link)))
> > + goto cleanup;
> > +
> > + uprobe_link = bpf_program__attach_uprobe(uprobe_prog,
> > + false /* retprobe */,
> > + 0 /* self pid */,
> > + "/proc/self/exe",
> > + uprobe_offset);
> > + if (CHECK(IS_ERR(uprobe_link), "attach_uprobe",
> > + "err %ld\n", PTR_ERR(uprobe_link)))
> > + goto cleanup;
> > +
> > + uretprobe_link = bpf_program__attach_uprobe(uretprobe_prog,
> > + true /* retprobe */,
> > + -1 /* any pid */,
> > + "/proc/self/exe",
> > + uprobe_offset);
> > + if (CHECK(IS_ERR(uretprobe_link), "attach_uretprobe",
> > + "err %ld\n", PTR_ERR(uretprobe_link)))
> > + goto cleanup;
> > +
> > + /* trigger & validate kprobe && kretprobe */
> > + usleep(1);
> > +
> > + err = bpf_map_lookup_elem(results_map_fd, &kprobe_idx, &res);
> > + if (CHECK(err, "get_kprobe_res",
> > + "failed to get kprobe res: %d\n", err))
> > + goto cleanup;
> > + if (CHECK(res != kprobe_idx + 1, "check_kprobe_res",
> > + "wrong kprobe res: %d\n", res))
> > + goto cleanup;
> > +
> > + err = bpf_map_lookup_elem(results_map_fd, &kretprobe_idx, &res);
> > + if (CHECK(err, "get_kretprobe_res",
> > + "failed to get kretprobe res: %d\n", err))
> > + goto cleanup;
> > + if (CHECK(res != kretprobe_idx + 1, "check_kretprobe_res",
> > + "wrong kretprobe res: %d\n", res))
> > + goto cleanup;
> > +
> > + /* trigger & validate uprobe & uretprobe */
> > + get_base_addr();
> > +
> > + err = bpf_map_lookup_elem(results_map_fd, &uprobe_idx, &res);
> > + if (CHECK(err, "get_uprobe_res",
> > + "failed to get uprobe res: %d\n", err))
> > + goto cleanup;
> > + if (CHECK(res != uprobe_idx + 1, "check_uprobe_res",
> > + "wrong uprobe res: %d\n", res))
> > + goto cleanup;
> > +
> > + err = bpf_map_lookup_elem(results_map_fd, &uretprobe_idx, &res);
> > + if (CHECK(err, "get_uretprobe_res",
> > + "failed to get uretprobe res: %d\n", err))
> > + goto cleanup;
> > + if (CHECK(res != uretprobe_idx + 1, "check_uretprobe_res",
> > + "wrong uretprobe res: %d\n", res))
> > + goto cleanup;
> > +
> > +cleanup:
> > + bpf_link__destroy(kprobe_link);
> > + bpf_link__destroy(kretprobe_link);
> > + bpf_link__destroy(uprobe_link);
> > + bpf_link__destroy(uretprobe_link);
>
> if any error happens, kprobe_link etc. will be a non-NULL pointer
> indicating an error. the above bpf_link__destroy() won't work
> properly. The same for patch #9.
Yes, you are absolutely right. Fixing.
I just also double-checked patch #9, it doesn't have this problem:
- stacktrace_build_id.c and stacktrace_map.c never call destroy before
bpf_link is successfully established.
- stacktrace_map_raw_tp.c uses if (!IS_ERR_OR_NULL(link)) check before
bpf_link__destroy.
So I think it's just this one.
>
> > + bpf_object__close(obj);
> > +}
> > diff --git a/tools/testing/selftests/bpf/progs/test_attach_probe.c b/tools/testing/selftests/bpf/progs/test_attach_probe.c
> > new file mode 100644
> > index 000000000000..7a7c5cd728c8
> > --- /dev/null
> > +++ b/tools/testing/selftests/bpf/progs/test_attach_probe.c
> > @@ -0,0 +1,55 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +// Copyright (c) 2017 Facebook
> > +
> > +#include <linux/ptrace.h>
> > +#include <linux/bpf.h>
> > +#include "bpf_helpers.h"
> > +
> > +struct {
> > + int type;
> > + int max_entries;
> > + int *key;
> > + int *value;
> > +} results_map SEC(".maps") = {
> > + .type = BPF_MAP_TYPE_ARRAY,
> > + .max_entries = 4,
> > +};
> > +
> > +SEC("kprobe/sys_nanosleep")
> > +int handle_sys_nanosleep_entry(struct pt_regs *ctx)
> > +{
> > + const int key = 0, value = 1;
> > +
> > + bpf_map_update_elem(&results_map, &key, &value, 0);
> > + return 0;
> > +}
> > +
> > +SEC("kretprobe/sys_nanosleep")
> > +int handle_sys_getpid_return(struct pt_regs *ctx)
> > +{
> > + const int key = 1, value = 2;
> > +
> > + bpf_map_update_elem(&results_map, &key, &value, 0);
> > + return 0;
> > +}
> > +
> > +SEC("uprobe/trigger_func")
> > +int handle_uprobe_entry(struct pt_regs *ctx)
> > +{
> > + const int key = 2, value = 3;
> > +
> > + bpf_map_update_elem(&results_map, &key, &value, 0);
> > + return 0;
> > +}
> > +
> > +SEC("uretprobe/trigger_func")
> > +int handle_uprobe_return(struct pt_regs *ctx)
> > +{
> > + const int key = 3, value = 4;
> > +
> > + bpf_map_update_elem(&results_map, &key, &value, 0);
> > + return 0;
> > +}
> > +
> > +char _license[] SEC("license") = "GPL";
> > +__u32 _version SEC("version") = 1;
> >
^ permalink raw reply
* Re: [PATCH v4 bpf-next 3/9] libbpf: add ability to attach/detach BPF program to perf event
From: Yonghong Song @ 2019-07-01 23:02 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Andrii Nakryiko, bpf@vger.kernel.org, netdev@vger.kernel.org,
Alexei Starovoitov, daniel@iogearbox.net, Kernel Team,
sdf@fomichev.me, Song Liu
In-Reply-To: <CAEf4Bzbz+bnM2E8aGP-eWtqDBepQ0Rc_KU-n+FQHnOrnFAWKwg@mail.gmail.com>
On 7/1/19 2:57 PM, Andrii Nakryiko wrote:
> On Mon, Jul 1, 2019 at 10:03 AM Yonghong Song <yhs@fb.com> wrote:
>>
>>
>>
>> On 6/28/19 8:49 PM, Andrii Nakryiko wrote:
>>> bpf_program__attach_perf_event allows to attach BPF program to existing
>>> perf event hook, providing most generic and most low-level way to attach BPF
>>> programs. It returns struct bpf_link, which should be passed to
>>> bpf_link__destroy to detach and free resources, associated with a link.
>>>
>>> Signed-off-by: Andrii Nakryiko <andriin@fb.com>
>>> ---
>>> tools/lib/bpf/libbpf.c | 61 ++++++++++++++++++++++++++++++++++++++++
>>> tools/lib/bpf/libbpf.h | 3 ++
>>> tools/lib/bpf/libbpf.map | 1 +
>>> 3 files changed, 65 insertions(+)
>>>
>>> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
>>> index 455795e6f8af..98c155ec3bfa 100644
>>> --- a/tools/lib/bpf/libbpf.c
>>> +++ b/tools/lib/bpf/libbpf.c
>>> @@ -32,6 +32,7 @@
>>> #include <linux/limits.h>
>>> #include <linux/perf_event.h>
>>> #include <linux/ring_buffer.h>
>>> +#include <sys/ioctl.h>
>>> #include <sys/stat.h>
>>> #include <sys/types.h>
>>> #include <sys/vfs.h>
>>> @@ -3958,6 +3959,66 @@ int bpf_link__destroy(struct bpf_link *link)
>>> return err;
>>> }
>>>
>>> +struct bpf_link_fd {
>>> + struct bpf_link link; /* has to be at the top of struct */
>>> + int fd; /* hook FD */
>>> +};
>>> +
>>> +static int bpf_link__destroy_perf_event(struct bpf_link *link)
>>> +{
>>> + struct bpf_link_fd *l = (void *)link;
>>> + int err;
>>> +
>>> + if (l->fd < 0)
>>> + return 0;
>>> +
>>> + err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0);
>>> + if (err)
>>> + err = -errno;
>>> +
>>> + close(l->fd);
>>> + return err;
>>> +}
>>> +
>>> +struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
>>> + int pfd)
>>> +{
>>> + char errmsg[STRERR_BUFSIZE];
>>> + struct bpf_link_fd *link;
>>> + int prog_fd, err;
>>> +
>>> + prog_fd = bpf_program__fd(prog);
>>> + if (prog_fd < 0) {
>>> + pr_warning("program '%s': can't attach before loaded\n",
>>> + bpf_program__title(prog, false));
>>> + return ERR_PTR(-EINVAL);
>>> + }
>>
>> should we check validity of pfd here?
>> If pfd < 0, we just return ERR_PTR(-EINVAL)?
>
> I can add that. I didn't do it, because in general, you can provide fd
>> = 0 which is still not a valid FD for PERF_EVENT_IOC_SET_BPF and
> PERF_EVENT_IOC_ENABLE, so in general we can't detect this reliably.
I just want a check for validity of input parameter which will failure
later with dedicated error message. But the same negative pfd will
be printed in ioctl error message. I am okay with this.
>
>> This way, in bpf_link__destroy_perf_event(), we do not need to check
>> l->fd < 0 since it will be always nonnegative.
>
> That check is not needed anyway, because even if pfd < 0, ioctl should
> fail and return error. I'll remove that check.
>
>>
>>> +
>>> + link = malloc(sizeof(*link));
>>> + if (!link)
>>> + return ERR_PTR(-ENOMEM);
>>> + link->link.destroy = &bpf_link__destroy_perf_event;
>>> + link->fd = pfd;
>>> +
>>> + if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
>>> + err = -errno;
>>> + free(link);
>>> + pr_warning("program '%s': failed to attach to pfd %d: %s\n",
>>> + bpf_program__title(prog, false), pfd,
>>> + libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
>>> + return ERR_PTR(err);
>>> + }
>>> + if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
>>> + err = -errno;
>>> + free(link);
>>> + pr_warning("program '%s': failed to enable pfd %d: %s\n",
>>> + bpf_program__title(prog, false), pfd,
>>> + libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
>>> + return ERR_PTR(err);
>>> + }
>>> + return (struct bpf_link *)link;
>>> +}
>>> +
>>> enum bpf_perf_event_ret
>>> bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
>>> void **copy_mem, size_t *copy_size,
>>> diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
>>> index 5082a5ebb0c2..1bf66c4a9330 100644
>>> --- a/tools/lib/bpf/libbpf.h
>>> +++ b/tools/lib/bpf/libbpf.h
>>> @@ -169,6 +169,9 @@ struct bpf_link;
>>>
>>> LIBBPF_API int bpf_link__destroy(struct bpf_link *link);
>>>
>>> +LIBBPF_API struct bpf_link *
>>> +bpf_program__attach_perf_event(struct bpf_program *prog, int pfd);
>>> +
>>> struct bpf_insn;
>>>
>>> /*
>>> diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
>>> index 3cde850fc8da..756f5aa802e9 100644
>>> --- a/tools/lib/bpf/libbpf.map
>>> +++ b/tools/lib/bpf/libbpf.map
>>> @@ -169,6 +169,7 @@ LIBBPF_0.0.4 {
>>> global:
>>> bpf_link__destroy;
>>> bpf_object__load_xattr;
>>> + bpf_program__attach_perf_event;
>>> btf_dump__dump_type;
>>> btf_dump__free;
>>> btf_dump__new;
>>>
^ permalink raw reply
* [PATCH net-next v4 4/4] gve: Add ethtool support
From: Catherine Sullivan @ 2019-07-01 22:57 UTC (permalink / raw)
To: netdev
Cc: Catherine Sullivan, Sagi Shahar, Jon Olson, Willem de Bruijn,
Luigi Rizzo
In-Reply-To: <20190701225755.209250-1-csully@google.com>
Add support for the following ethtool commands:
ethtool -s|--change devname [msglvl N] [msglevel type on|off]
ethtool -S|--statistics devname
ethtool -i|--driver devname
ethtool -l|--show-channels devname
ethtool -L|--set-channels devname
ethtool -g|--show-ring devname
ethtool --reset devname
Signed-off-by: Catherine Sullivan <csully@google.com>
Signed-off-by: Sagi Shahar <sagis@google.com>
Signed-off-by: Jon Olson <jonolson@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Luigi Rizzo <lrizzo@google.com>
---
drivers/net/ethernet/google/gve/Makefile | 2 +-
drivers/net/ethernet/google/gve/gve.h | 4 +
drivers/net/ethernet/google/gve/gve_ethtool.c | 243 ++++++++++++++++++
drivers/net/ethernet/google/gve/gve_main.c | 41 ++-
4 files changed, 288 insertions(+), 2 deletions(-)
create mode 100644 drivers/net/ethernet/google/gve/gve_ethtool.c
diff --git a/drivers/net/ethernet/google/gve/Makefile b/drivers/net/ethernet/google/gve/Makefile
index a1890c93705b..3354ce40eb97 100644
--- a/drivers/net/ethernet/google/gve/Makefile
+++ b/drivers/net/ethernet/google/gve/Makefile
@@ -1,4 +1,4 @@
# Makefile for the Google virtual Ethernet (gve) driver
obj-$(CONFIG_GVE) += gve.o
-gve-objs := gve_main.o gve_tx.o gve_rx.o gve_adminq.o
+gve-objs := gve_main.o gve_tx.o gve_rx.o gve_ethtool.o gve_adminq.o
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 7960d5532078..92372dc43be8 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -452,4 +452,8 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown);
int gve_adjust_queues(struct gve_priv *priv,
struct gve_queue_config new_rx_config,
struct gve_queue_config new_tx_config);
+/* exported by ethtool.c */
+extern const struct ethtool_ops gve_ethtool_ops;
+/* needed by ethtool */
+extern const char gve_version_str[];
#endif /* _GVE_H_ */
diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c
new file mode 100644
index 000000000000..52947d668e6d
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_ethtool.c
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#include <linux/rtnetlink.h>
+#include "gve.h"
+
+static void gve_get_drvinfo(struct net_device *netdev,
+ struct ethtool_drvinfo *info)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ strlcpy(info->driver, "gve", sizeof(info->driver));
+ strlcpy(info->version, gve_version_str, sizeof(info->version));
+ strlcpy(info->bus_info, pci_name(priv->pdev), sizeof(info->bus_info));
+}
+
+static void gve_set_msglevel(struct net_device *netdev, u32 value)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ priv->msg_enable = value;
+}
+
+static u32 gve_get_msglevel(struct net_device *netdev)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ return priv->msg_enable;
+}
+
+static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = {
+ "rx_packets", "tx_packets", "rx_bytes", "tx_bytes",
+ "rx_dropped", "tx_dropped", "tx_timeouts",
+};
+
+#define GVE_MAIN_STATS_LEN ARRAY_SIZE(gve_gstrings_main_stats)
+#define NUM_GVE_TX_CNTS 5
+#define NUM_GVE_RX_CNTS 2
+
+static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+ char *s = (char *)data;
+ int i;
+
+ if (stringset != ETH_SS_STATS)
+ return;
+
+ memcpy(s, *gve_gstrings_main_stats,
+ sizeof(gve_gstrings_main_stats));
+ s += sizeof(gve_gstrings_main_stats);
+ for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+ snprintf(s, ETH_GSTRING_LEN, "rx_desc_cnt[%u]", i);
+ s += ETH_GSTRING_LEN;
+ snprintf(s, ETH_GSTRING_LEN, "rx_desc_fill_cnt[%u]", i);
+ s += ETH_GSTRING_LEN;
+ }
+ for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+ snprintf(s, ETH_GSTRING_LEN, "tx_req[%u]", i);
+ s += ETH_GSTRING_LEN;
+ snprintf(s, ETH_GSTRING_LEN, "tx_done[%u]", i);
+ s += ETH_GSTRING_LEN;
+ snprintf(s, ETH_GSTRING_LEN, "tx_wake[%u]", i);
+ s += ETH_GSTRING_LEN;
+ snprintf(s, ETH_GSTRING_LEN, "tx_stop[%u]", i);
+ s += ETH_GSTRING_LEN;
+ snprintf(s, ETH_GSTRING_LEN, "tx_event_counter[%u]", i);
+ s += ETH_GSTRING_LEN;
+ }
+}
+
+static int gve_get_sset_count(struct net_device *netdev, int sset)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ switch (sset) {
+ case ETH_SS_STATS:
+ return GVE_MAIN_STATS_LEN +
+ (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) +
+ (priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static void
+gve_get_ethtool_stats(struct net_device *netdev,
+ struct ethtool_stats *stats, u64 *data)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+ u64 rx_pkts, rx_bytes, tx_pkts, tx_bytes;
+ unsigned int start;
+ int ring;
+ int i;
+
+ ASSERT_RTNL();
+
+ for (rx_pkts = 0, rx_bytes = 0, ring = 0;
+ ring < priv->rx_cfg.num_queues; ring++) {
+ if (priv->rx) {
+ do {
+ u64_stats_fetch_begin(&priv->rx[ring].statss);
+ rx_pkts += priv->rx[ring].rpackets;
+ rx_bytes += priv->rx[ring].rbytes;
+ } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
+ start));
+ }
+ }
+ for (tx_pkts = 0, tx_bytes = 0, ring = 0;
+ ring < priv->tx_cfg.num_queues; ring++) {
+ if (priv->tx) {
+ do {
+ u64_stats_fetch_begin(&priv->tx[ring].statss);
+ tx_pkts += priv->tx[ring].pkt_done;
+ tx_bytes += priv->tx[ring].bytes_done;
+ } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
+ start));
+ }
+ }
+
+ i = 0;
+ data[i++] = rx_pkts;
+ data[i++] = tx_pkts;
+ data[i++] = rx_bytes;
+ data[i++] = tx_bytes;
+ /* Skip rx_dropped and tx_dropped */
+ i += 2;
+ data[i++] = priv->tx_timeo_cnt;
+ i = GVE_MAIN_STATS_LEN;
+
+ /* walk RX rings */
+ if (priv->rx) {
+ for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
+ struct gve_rx_ring *rx = &priv->rx[ring];
+
+ data[i++] = rx->desc.cnt;
+ data[i++] = rx->desc.fill_cnt;
+ }
+ } else {
+ i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS;
+ }
+ /* walk TX rings */
+ if (priv->tx) {
+ for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
+ struct gve_tx_ring *tx = &priv->tx[ring];
+
+ data[i++] = tx->req;
+ data[i++] = tx->done;
+ data[i++] = tx->wake_queue;
+ data[i++] = tx->stop_queue;
+ data[i++] = be32_to_cpu(gve_tx_load_event_counter(priv,
+ tx));
+ }
+ } else {
+ i += priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS;
+ }
+}
+
+static void gve_get_channels(struct net_device *netdev,
+ struct ethtool_channels *cmd)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ cmd->max_rx = priv->rx_cfg.max_queues;
+ cmd->max_tx = priv->tx_cfg.max_queues;
+ cmd->max_other = 0;
+ cmd->max_combined = 0;
+ cmd->rx_count = priv->rx_cfg.num_queues;
+ cmd->tx_count = priv->tx_cfg.num_queues;
+ cmd->other_count = 0;
+ cmd->combined_count = 0;
+}
+
+static int gve_set_channels(struct net_device *netdev,
+ struct ethtool_channels *cmd)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+ struct gve_queue_config new_tx_cfg = priv->tx_cfg;
+ struct gve_queue_config new_rx_cfg = priv->rx_cfg;
+ struct ethtool_channels old_settings;
+ int new_tx = cmd->tx_count;
+ int new_rx = cmd->rx_count;
+
+ gve_get_channels(netdev, &old_settings);
+
+ /* Changing combined is not allowed allowed */
+ if (cmd->combined_count != old_settings.combined_count)
+ return -EINVAL;
+
+ if (!new_rx || !new_tx)
+ return -EINVAL;
+
+ if (!netif_carrier_ok(netdev)) {
+ priv->tx_cfg.num_queues = new_tx;
+ priv->rx_cfg.num_queues = new_rx;
+ return 0;
+ }
+
+ new_tx_cfg.num_queues = new_tx;
+ new_rx_cfg.num_queues = new_rx;
+
+ return gve_adjust_queues(priv, new_rx_cfg, new_tx_cfg);
+}
+
+static void gve_get_ringparam(struct net_device *netdev,
+ struct ethtool_ringparam *cmd)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ cmd->rx_max_pending = priv->rx_desc_cnt;
+ cmd->tx_max_pending = priv->tx_desc_cnt;
+ cmd->rx_pending = priv->rx_desc_cnt;
+ cmd->tx_pending = priv->tx_desc_cnt;
+}
+
+static int gve_user_reset(struct net_device *netdev, u32 *flags)
+{
+ struct gve_priv *priv = netdev_priv(netdev);
+
+ if (*flags == ETH_RESET_ALL) {
+ *flags = 0;
+ return gve_reset(priv, true);
+ }
+
+ return -EOPNOTSUPP;
+}
+
+const struct ethtool_ops gve_ethtool_ops = {
+ .get_drvinfo = gve_get_drvinfo,
+ .get_strings = gve_get_strings,
+ .get_sset_count = gve_get_sset_count,
+ .get_ethtool_stats = gve_get_ethtool_stats,
+ .set_msglevel = gve_set_msglevel,
+ .get_msglevel = gve_get_msglevel,
+ .set_channels = gve_set_channels,
+ .get_channels = gve_get_channels,
+ .get_link = ethtool_op_get_link,
+ .get_ringparam = gve_get_ringparam,
+ .reset = gve_user_reset,
+};
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 126d6533b965..6a147ed4627f 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -23,7 +23,7 @@
#define GVE_VERSION "1.0.0"
#define GVE_VERSION_PREFIX "GVE-"
-static const char gve_version_str[] = GVE_VERSION;
+const char gve_version_str[] = GVE_VERSION;
static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
@@ -746,6 +746,44 @@ static int gve_close(struct net_device *dev)
return gve_reset_recovery(priv, false);
}
+int gve_adjust_queues(struct gve_priv *priv,
+ struct gve_queue_config new_rx_config,
+ struct gve_queue_config new_tx_config)
+{
+ int err;
+
+ if (netif_carrier_ok(priv->dev)) {
+ /* To make this process as simple as possible we teardown the
+ * device, set the new configuration, and then bring the device
+ * up again.
+ */
+ err = gve_close(priv->dev);
+ /* we have already tried to reset in close,
+ * just fail at this point
+ */
+ if (err)
+ return err;
+ priv->tx_cfg = new_tx_config;
+ priv->rx_cfg = new_rx_config;
+
+ err = gve_open(priv->dev);
+ if (err)
+ goto err;
+
+ return 0;
+ }
+ /* Set the config for the next up. */
+ priv->tx_cfg = new_tx_config;
+ priv->rx_cfg = new_rx_config;
+
+ return 0;
+err:
+ netif_err(priv, drv, priv->dev,
+ "Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n");
+ gve_turndown(priv);
+ return err;
+}
+
static void gve_turndown(struct gve_priv *priv)
{
int idx;
@@ -1082,6 +1120,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
SET_NETDEV_DEV(dev, &pdev->dev);
pci_set_drvdata(pdev, dev);
+ dev->ethtool_ops = &gve_ethtool_ops;
dev->netdev_ops = &gve_netdev_ops;
/* advertise features */
dev->hw_features = NETIF_F_HIGHDMA;
--
2.22.0.410.gd8fdbe21b5-goog
^ permalink raw reply related
* [PATCH net-next v4 3/4] gve: Add workqueue and reset support
From: Catherine Sullivan @ 2019-07-01 22:57 UTC (permalink / raw)
To: netdev
Cc: Catherine Sullivan, Sagi Shahar, Jon Olson, Willem de Bruijn,
Luigi Rizzo
In-Reply-To: <20190701225755.209250-1-csully@google.com>
Add support for the workqueue to handle management interrupts and
support for resets.
Signed-off-by: Catherine Sullivan <csully@google.com>
Signed-off-by: Sagi Shahar <sagis@google.com>
Signed-off-by: Jon Olson <jonolson@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Luigi Rizzo <lrizzo@google.com>
---
.../networking/device_drivers/google/gve.rst | 11 +
drivers/net/ethernet/google/gve/gve.h | 62 ++++++
drivers/net/ethernet/google/gve/gve_main.c | 190 +++++++++++++++++-
drivers/net/ethernet/google/gve/gve_rx.c | 5 +-
4 files changed, 260 insertions(+), 8 deletions(-)
diff --git a/Documentation/networking/device_drivers/google/gve.rst b/Documentation/networking/device_drivers/google/gve.rst
index df8974fb3270..793693cef6e3 100644
--- a/Documentation/networking/device_drivers/google/gve.rst
+++ b/Documentation/networking/device_drivers/google/gve.rst
@@ -40,6 +40,8 @@ The driver interacts with the device in the following ways:
- See gve_register.h for more detail
- Admin Queue
- See description below
+ - Reset
+ - At any time the device can be reset
- Interrupts
- See supported interrupts below
- Transmit and Receive Queues
@@ -69,6 +71,12 @@ the following (with proper locking):
The device will update the status field in each AQ command reported as
executed through the ADMIN_QUEUE_EVENT_COUNTER register.
+Device Resets
+-------------
+A device reset is triggered by writing 0x0 to the AQ PFN register.
+This causes the device to release all resources allocated by the
+driver, including the AQ itself.
+
Interrupts
----------
The following interrupts are supported by the driver:
@@ -78,6 +86,9 @@ Management Interrupt
The management interrupt is used by the device to tell the driver to
look at the GVE_DEVICE_STATUS register.
+The handler for the management irq simply queues the service task in
+the workqueue to check the register and acks the irq.
+
Notification Block Interrupts
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The notification block interrupts are used to tell the driver to poll
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 110ae46adc69..7960d5532078 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -205,9 +205,18 @@ struct gve_priv {
u32 adminq_mask; /* masks prod_cnt to adminq size */
u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
+ struct workqueue_struct *gve_wq;
+ struct work_struct service_task;
+ unsigned long service_task_flags;
unsigned long state_flags;
};
+enum gve_service_task_flags {
+ GVE_PRIV_FLAGS_DO_RESET = BIT(1),
+ GVE_PRIV_FLAGS_RESET_IN_PROGRESS = BIT(2),
+ GVE_PRIV_FLAGS_PROBE_IN_PROGRESS = BIT(3),
+};
+
enum gve_state_flags {
GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = BIT(1),
GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = BIT(2),
@@ -215,6 +224,53 @@ enum gve_state_flags {
GVE_PRIV_FLAGS_NAPI_ENABLED = BIT(4),
};
+static inline bool gve_get_do_reset(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
+}
+
+static inline void gve_set_do_reset(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
+}
+
+static inline void gve_clear_do_reset(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
+}
+
+static inline bool gve_get_reset_in_progress(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS,
+ &priv->service_task_flags);
+}
+
+static inline void gve_set_reset_in_progress(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
+}
+
+static inline void gve_clear_reset_in_progress(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
+}
+
+static inline bool gve_get_probe_in_progress(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS,
+ &priv->service_task_flags);
+}
+
+static inline void gve_set_probe_in_progress(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
+}
+
+static inline void gve_clear_probe_in_progress(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
+}
+
static inline bool gve_get_admin_queue_ok(struct gve_priv *priv)
{
return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
@@ -390,4 +446,10 @@ int gve_rx_alloc_rings(struct gve_priv *priv);
void gve_rx_free_rings(struct gve_priv *priv);
bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
netdev_features_t feat);
+/* Reset */
+void gve_schedule_reset(struct gve_priv *priv);
+int gve_reset(struct gve_priv *priv, bool attempt_teardown);
+int gve_adjust_queues(struct gve_priv *priv,
+ struct gve_queue_config new_rx_config,
+ struct gve_queue_config new_tx_config);
#endif /* _GVE_H_ */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 44462dce0254..126d6533b965 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -11,6 +11,7 @@
#include <linux/pci.h>
#include <linux/sched.h>
#include <linux/timer.h>
+#include <linux/workqueue.h>
#include <net/sch_generic.h>
#include "gve.h"
#include "gve_adminq.h"
@@ -77,6 +78,9 @@ static void gve_free_counter_array(struct gve_priv *priv)
static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
{
+ struct gve_priv *priv = arg;
+
+ queue_work(priv->gve_wq, &priv->service_task);
return IRQ_HANDLED;
}
@@ -284,6 +288,8 @@ static int gve_setup_device_resources(struct gve_priv *priv)
return err;
}
+static void gve_trigger_reset(struct gve_priv *priv);
+
static void gve_teardown_device_resources(struct gve_priv *priv)
{
int err;
@@ -295,7 +301,7 @@ static void gve_teardown_device_resources(struct gve_priv *priv)
dev_err(&priv->pdev->dev,
"Could not deconfigure device resources: err=%d\n",
err);
- return;
+ gve_trigger_reset(priv);
}
}
gve_free_counter_array(priv);
@@ -330,6 +336,9 @@ static int gve_register_qpls(struct gve_priv *priv)
netif_err(priv, drv, priv->dev,
"failed to register queue page list %d\n",
priv->qpls[i].id);
+ /* This failure will trigger a reset - no need to clean
+ * up
+ */
return err;
}
}
@@ -344,6 +353,7 @@ static int gve_unregister_qpls(struct gve_priv *priv)
for (i = 0; i < num_qpls; i++) {
err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
+ /* This failure will trigger a reset - no need to clean up */
if (err) {
netif_err(priv, drv, priv->dev,
"Failed to unregister queue page list %d\n",
@@ -364,6 +374,9 @@ static int gve_create_rings(struct gve_priv *priv)
if (err) {
netif_err(priv, drv, priv->dev, "failed to create tx queue %d\n",
i);
+ /* This failure will trigger a reset - no need to clean
+ * up
+ */
return err;
}
netif_dbg(priv, drv, priv->dev, "created tx queue %d\n", i);
@@ -373,6 +386,9 @@ static int gve_create_rings(struct gve_priv *priv)
if (err) {
netif_err(priv, drv, priv->dev, "failed to create rx queue %d\n",
i);
+ /* This failure will trigger a reset - no need to clean
+ * up
+ */
return err;
}
/* Rx data ring has been prefilled with packet buffers at
@@ -448,6 +464,9 @@ static int gve_destroy_rings(struct gve_priv *priv)
netif_err(priv, drv, priv->dev,
"failed to destroy tx queue %d\n",
i);
+ /* This failure will trigger a reset - no need to clean
+ * up
+ */
return err;
}
netif_dbg(priv, drv, priv->dev, "destroyed tx queue %d\n", i);
@@ -458,6 +477,9 @@ static int gve_destroy_rings(struct gve_priv *priv)
netif_err(priv, drv, priv->dev,
"failed to destroy rx queue %d\n",
i);
+ /* This failure will trigger a reset - no need to clean
+ * up
+ */
return err;
}
netif_dbg(priv, drv, priv->dev, "destroyed rx queue %d\n", i);
@@ -626,6 +648,18 @@ static void gve_free_qpls(struct gve_priv *priv)
kfree(priv->qpls);
}
+/* Use this to schedule a reset when the device is capable of continuing
+ * to handle other requests in its current state. If it is not, do a reset
+ * in thread instead.
+ */
+void gve_schedule_reset(struct gve_priv *priv)
+{
+ gve_set_do_reset(priv);
+ queue_work(priv->gve_wq, &priv->service_task);
+}
+
+static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
+static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
static void gve_turndown(struct gve_priv *priv);
static void gve_turnup(struct gve_priv *priv);
@@ -650,10 +684,10 @@ static int gve_open(struct net_device *dev)
err = gve_register_qpls(priv);
if (err)
- return err;
+ goto reset;
err = gve_create_rings(priv);
if (err)
- return err;
+ goto reset;
gve_set_device_rings_ok(priv);
gve_turnup(priv);
@@ -665,6 +699,19 @@ static int gve_open(struct net_device *dev)
free_qpls:
gve_free_qpls(priv);
return err;
+
+reset:
+ /* This must have been called from a reset due to the rtnl lock
+ * so just return at this point.
+ */
+ if (gve_get_reset_in_progress(priv))
+ return err;
+ /* Otherwise reset before returning */
+ gve_reset_and_teardown(priv, true);
+ /* if this fails there is nothing we can do so just ignore the return */
+ gve_reset_recovery(priv, false);
+ /* return the original error */
+ return err;
}
static int gve_close(struct net_device *dev)
@@ -677,16 +724,26 @@ static int gve_close(struct net_device *dev)
gve_turndown(priv);
err = gve_destroy_rings(priv);
if (err)
- return err;
+ goto err;
err = gve_unregister_qpls(priv);
if (err)
- return err;
+ goto err;
gve_clear_device_rings_ok(priv);
}
gve_free_rings(priv);
gve_free_qpls(priv);
return 0;
+
+err:
+ /* This must have been called from a reset due to the rtnl lock
+ * so just return at this point.
+ */
+ if (gve_get_reset_in_progress(priv))
+ return err;
+ /* Otherwise reset before returning */
+ gve_reset_and_teardown(priv, true);
+ return gve_reset_recovery(priv, false);
}
static void gve_turndown(struct gve_priv *priv)
@@ -749,6 +806,7 @@ static void gve_tx_timeout(struct net_device *dev)
{
struct gve_priv *priv = netdev_priv(dev);
+ gve_schedule_reset(priv);
priv->tx_timeo_cnt++;
}
@@ -760,6 +818,42 @@ static const struct net_device_ops gve_netdev_ops = {
.ndo_tx_timeout = gve_tx_timeout,
};
+static void gve_handle_status(struct gve_priv *priv, u32 status)
+{
+ if (GVE_DEVICE_STATUS_RESET_MASK & status) {
+ dev_info(&priv->pdev->dev, "Device requested reset.\n");
+ gve_set_do_reset(priv);
+ }
+}
+
+static void gve_handle_reset(struct gve_priv *priv)
+{
+ /* A service task will be scheduled at the end of probe to catch any
+ * resets that need to happen, and we don't want to reset until
+ * probe is done.
+ */
+ if (gve_get_probe_in_progress(priv))
+ return;
+
+ if (gve_get_do_reset(priv)) {
+ rtnl_lock();
+ gve_reset(priv, false);
+ rtnl_unlock();
+ }
+}
+
+/* Handle NIC status register changes and reset requests */
+static void gve_service_task(struct work_struct *work)
+{
+ struct gve_priv *priv = container_of(work, struct gve_priv,
+ service_task);
+
+ gve_handle_status(priv,
+ ioread32be(&priv->reg_bar0->device_status));
+
+ gve_handle_reset(priv);
+}
+
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
{
int num_ntfy;
@@ -847,6 +941,72 @@ static void gve_teardown_priv_resources(struct gve_priv *priv)
gve_adminq_free(&priv->pdev->dev, priv);
}
+static void gve_trigger_reset(struct gve_priv *priv)
+{
+ /* Reset the device by releasing the AQ */
+ gve_adminq_release(priv);
+}
+
+static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
+{
+ gve_trigger_reset(priv);
+ /* With the reset having already happened, close cannot fail */
+ if (was_up)
+ gve_close(priv->dev);
+ gve_teardown_priv_resources(priv);
+}
+
+static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
+{
+ int err;
+
+ err = gve_init_priv(priv, true);
+ if (err)
+ goto err;
+ if (was_up) {
+ err = gve_open(priv->dev);
+ if (err)
+ goto err;
+ }
+ return 0;
+err:
+ dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
+ gve_turndown(priv);
+ return err;
+}
+
+int gve_reset(struct gve_priv *priv, bool attempt_teardown)
+{
+ bool was_up = netif_carrier_ok(priv->dev);
+ int err;
+
+ dev_info(&priv->pdev->dev, "Performing reset\n");
+ gve_clear_do_reset(priv);
+ gve_set_reset_in_progress(priv);
+ /* If we aren't attempting to teardown normally, just go turndown and
+ * reset right away.
+ */
+ if (!attempt_teardown) {
+ gve_turndown(priv);
+ gve_reset_and_teardown(priv, was_up);
+ } else {
+ /* Otherwise attempt to close normally */
+ if (was_up) {
+ err = gve_close(priv->dev);
+ /* If that fails reset as we did above */
+ if (err)
+ gve_reset_and_teardown(priv, was_up);
+ }
+ /* Clean up any remaining resources */
+ gve_teardown_priv_resources(priv);
+ }
+
+ /* Set it all back up */
+ err = gve_reset_recovery(priv, was_up);
+ gve_clear_reset_in_progress(priv);
+ return err;
+}
+
static void gve_write_version(u8 __iomem *driver_version_register)
{
const char *c = gve_version_prefix;
@@ -943,21 +1103,36 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
priv->msg_enable = DEFAULT_MSG_LEVEL;
priv->reg_bar0 = reg_bar;
priv->db_bar2 = db_bar;
+ priv->service_task_flags = 0x0;
priv->state_flags = 0x0;
+
+ gve_set_probe_in_progress(priv);
+ priv->gve_wq = alloc_ordered_workqueue("gve", 0);
+ if (!priv->gve_wq) {
+ dev_err(&pdev->dev, "Could not allocate workqueue");
+ err = -ENOMEM;
+ goto abort_with_netdev;
+ }
+ INIT_WORK(&priv->service_task, gve_service_task);
priv->tx_cfg.max_queues = max_tx_queues;
priv->rx_cfg.max_queues = max_rx_queues;
err = gve_init_priv(priv, false);
if (err)
- goto abort_with_netdev;
+ goto abort_with_wq;
err = register_netdev(dev);
if (err)
- goto abort_with_netdev;
+ goto abort_with_wq;
dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
+ gve_clear_probe_in_progress(priv);
+ queue_work(priv->gve_wq, &priv->service_task);
return 0;
+abort_with_wq:
+ destroy_workqueue(priv->gve_wq);
+
abort_with_netdev:
free_netdev(dev);
@@ -985,6 +1160,7 @@ static void gve_remove(struct pci_dev *pdev)
unregister_netdev(netdev);
gve_teardown_priv_resources(priv);
+ destroy_workqueue(priv->gve_wq);
free_netdev(netdev);
pci_iounmap(pdev, db_bar);
pci_iounmap(pdev, reg_bar);
diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c
index 2254857eac50..84e0ecce14c4 100644
--- a/drivers/net/ethernet/google/gve/gve_rx.c
+++ b/drivers/net/ethernet/google/gve/gve_rx.c
@@ -334,6 +334,9 @@ static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
#endif
have_skb:
+ /* We didn't manage to allocate an skb but we haven't had any
+ * reset worthy failures.
+ */
if (!skb)
return true;
@@ -399,7 +402,7 @@ bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
rx->desc.seqno);
bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
if (!gve_rx(rx, desc, feat))
- return false;
+ gve_schedule_reset(priv);
cnt++;
idx = cnt & rx->desc.mask;
desc = rx->desc.desc_ring + idx;
--
2.22.0.410.gd8fdbe21b5-goog
^ permalink raw reply related
* [PATCH net-next v4 2/4] gve: Add transmit and receive support
From: Catherine Sullivan @ 2019-07-01 22:57 UTC (permalink / raw)
To: netdev
Cc: Catherine Sullivan, Sagi Shahar, Jon Olson, Willem de Bruijn,
Luigi Rizzo
In-Reply-To: <20190701225755.209250-1-csully@google.com>
Add support for passing traffic.
Signed-off-by: Catherine Sullivan <csully@google.com>
Signed-off-by: Sagi Shahar <sagis@google.com>
Signed-off-by: Jon Olson <jonolson@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Luigi Rizzo <lrizzo@google.com>
---
.../networking/device_drivers/google/gve.rst | 30 +
drivers/net/ethernet/google/gve/Makefile | 2 +-
drivers/net/ethernet/google/gve/gve.h | 260 +++++++-
drivers/net/ethernet/google/gve/gve_adminq.c | 138 +++++
drivers/net/ethernet/google/gve/gve_adminq.h | 83 +++
drivers/net/ethernet/google/gve/gve_desc.h | 113 ++++
drivers/net/ethernet/google/gve/gve_main.c | 573 ++++++++++++++++-
drivers/net/ethernet/google/gve/gve_rx.c | 443 +++++++++++++
drivers/net/ethernet/google/gve/gve_tx.c | 584 ++++++++++++++++++
9 files changed, 2221 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/ethernet/google/gve/gve_desc.h
create mode 100644 drivers/net/ethernet/google/gve/gve_rx.c
create mode 100644 drivers/net/ethernet/google/gve/gve_tx.c
diff --git a/Documentation/networking/device_drivers/google/gve.rst b/Documentation/networking/device_drivers/google/gve.rst
index 7397c82f4c8f..df8974fb3270 100644
--- a/Documentation/networking/device_drivers/google/gve.rst
+++ b/Documentation/networking/device_drivers/google/gve.rst
@@ -42,6 +42,8 @@ The driver interacts with the device in the following ways:
- See description below
- Interrupts
- See supported interrupts below
+ - Transmit and Receive Queues
+ - See description below
Registers
---------
@@ -80,3 +82,31 @@ Notification Block Interrupts
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The notification block interrupts are used to tell the driver to poll
the queues associated with that interrupt.
+
+The handler for these irqs schedule the napi for that block to run
+and poll the queues.
+
+Traffic Queues
+--------------
+gVNIC's queues are composed of a descriptor ring and a buffer and are
+assigned to a notification block.
+
+The descriptor rings are power-of-two-sized ring buffers consisting of
+fixed-size descriptors. They advance their head pointer using a __be32
+doorbell located in Bar2. The tail pointers are advanced by consuming
+descriptors in-order and updating a __be32 counter. Both the doorbell
+and the counter overflow to zero.
+
+Each queue's buffers must be registered in advance with the device as a
+queue page list, and packet data can only be put in those pages.
+
+Transmit
+~~~~~~~~
+gve maps the buffers for transmit rings into a FIFO and copies the packets
+into the FIFO before sending them to the NIC.
+
+Receive
+~~~~~~~
+The buffers for receive rings are put into a data ring that is the same
+length as the descriptor ring and the head and tail pointers advance over
+the rings together.
diff --git a/drivers/net/ethernet/google/gve/Makefile b/drivers/net/ethernet/google/gve/Makefile
index cec03ee6d931..a1890c93705b 100644
--- a/drivers/net/ethernet/google/gve/Makefile
+++ b/drivers/net/ethernet/google/gve/Makefile
@@ -1,4 +1,4 @@
# Makefile for the Google virtual Ethernet (gve) driver
obj-$(CONFIG_GVE) += gve.o
-gve-objs := gve_main.o gve_adminq.o
+gve-objs := gve_main.o gve_tx.o gve_rx.o gve_adminq.o
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 47fb86e5aeff..110ae46adc69 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -10,6 +10,8 @@
#include <linux/dma-mapping.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
+#include <linux/u64_stats_sync.h>
+#include "gve_desc.h"
#ifndef PCI_VENDOR_ID_GOOGLE
#define PCI_VENDOR_ID_GOOGLE 0x1ae0
@@ -20,18 +22,152 @@
#define GVE_REGISTER_BAR 0
#define GVE_DOORBELL_BAR 2
-/* 1 for management */
+/* Driver can alloc up to 2 segments for the header and 2 for the payload. */
+#define GVE_TX_MAX_IOVEC 4
+/* 1 for management, 1 for rx, 1 for tx */
#define GVE_MIN_MSIX 3
+/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
+struct gve_rx_desc_queue {
+ struct gve_rx_desc *desc_ring; /* the descriptor ring */
+ dma_addr_t bus; /* the bus for the desc_ring */
+ u32 cnt; /* free-running total number of completed packets */
+ u32 fill_cnt; /* free-running total number of descriptors posted */
+ u32 mask; /* masks the cnt to the size of the ring */
+ u8 seqno; /* the next expected seqno for this desc*/
+};
+
+/* The page info for a single slot in the RX data queue */
+struct gve_rx_slot_page_info {
+ struct page *page;
+ void *page_address;
+ u32 page_offset; /* offset to write to in page */
+};
+
+/* A list of pages registered with the device during setup and used by a queue
+ * as buffers
+ */
+struct gve_queue_page_list {
+ u32 id; /* unique id */
+ u32 num_entries;
+ struct page **pages; /* list of num_entries pages */
+ dma_addr_t *page_buses; /* the dma addrs of the pages */
+};
+
+/* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */
+struct gve_rx_data_queue {
+ struct gve_rx_data_slot *data_ring; /* read by NIC */
+ dma_addr_t data_bus; /* dma mapping of the slots */
+ struct gve_rx_slot_page_info *page_info; /* page info of the buffers */
+ struct gve_queue_page_list *qpl; /* qpl assigned to this queue */
+ u32 mask; /* masks the cnt to the size of the ring */
+ u32 cnt; /* free-running total number of completed packets */
+};
+
+struct gve_priv;
+
+/* An RX ring that contains a power-of-two sized desc and data ring. */
+struct gve_rx_ring {
+ struct gve_priv *gve;
+ struct gve_rx_desc_queue desc;
+ struct gve_rx_data_queue data;
+ u64 rbytes; /* free-running bytes received */
+ u64 rpackets; /* free-running packets received */
+ u32 q_num; /* queue index */
+ u32 ntfy_id; /* notification block index */
+ struct gve_queue_resources *q_resources; /* head and tail pointer idx */
+ dma_addr_t q_resources_bus; /* dma address for the queue resources */
+ struct u64_stats_sync statss; /* sync stats for 32bit archs */
+};
+
+/* A TX desc ring entry */
+union gve_tx_desc {
+ struct gve_tx_pkt_desc pkt; /* first desc for a packet */
+ struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
+};
+
+/* Tracks the memory in the fifo occupied by a segment of a packet */
+struct gve_tx_iovec {
+ u32 iov_offset; /* offset into this segment */
+ u32 iov_len; /* length */
+ u32 iov_padding; /* padding associated with this segment */
+};
+
+/* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc
+ * ring entry but only used for a pkt_desc not a seg_desc
+ */
+struct gve_tx_buffer_state {
+ struct sk_buff *skb; /* skb for this pkt */
+ struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
+};
+
+/* A TX buffer - each queue has one */
+struct gve_tx_fifo {
+ void *base; /* address of base of FIFO */
+ u32 size; /* total size */
+ atomic_t available; /* how much space is still available */
+ u32 head; /* offset to write at */
+ struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */
+};
+
+/* A TX ring that contains a power-of-two sized desc ring and a FIFO buffer */
+struct gve_tx_ring {
+ /* Cacheline 0 -- Accessed & dirtied during transmit */
+ struct gve_tx_fifo tx_fifo;
+ u32 req; /* driver tracked head pointer */
+ u32 done; /* driver tracked tail pointer */
+
+ /* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
+ __be32 last_nic_done ____cacheline_aligned; /* NIC tail pointer */
+ u64 pkt_done; /* free-running - total packets completed */
+ u64 bytes_done; /* free-running - total bytes completed */
+
+ /* Cacheline 2 -- Read-mostly fields */
+ union gve_tx_desc *desc ____cacheline_aligned;
+ struct gve_tx_buffer_state *info; /* Maps 1:1 to a desc */
+ struct netdev_queue *netdev_txq;
+ struct gve_queue_resources *q_resources; /* head and tail pointer idx */
+ u32 mask; /* masks req and done down to queue size */
+
+ /* Slow-path fields */
+ u32 q_num ____cacheline_aligned; /* queue idx */
+ u32 stop_queue; /* count of queue stops */
+ u32 wake_queue; /* count of queue wakes */
+ u32 ntfy_id; /* notification block index */
+ dma_addr_t bus; /* dma address of the descr ring */
+ dma_addr_t q_resources_bus; /* dma address of the queue resources */
+ struct u64_stats_sync statss; /* sync stats for 32bit archs */
+} ____cacheline_aligned;
+
+/* Wraps the info for one irq including the napi struct and the queues
+ * associated with that irq.
+ */
struct gve_notify_block {
__be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
char name[IFNAMSIZ + 16]; /* name registered with the kernel */
struct napi_struct napi; /* kernel napi struct for this block */
struct gve_priv *priv;
+ struct gve_tx_ring *tx; /* tx rings on this block */
+ struct gve_rx_ring *rx; /* rx rings on this block */
} ____cacheline_aligned;
+/* Tracks allowed and current queue settings */
+struct gve_queue_config {
+ u16 max_queues;
+ u16 num_queues; /* current */
+};
+
+/* Tracks the available and used qpl IDs */
+struct gve_qpl_config {
+ u32 qpl_map_size; /* map memory size */
+ unsigned long *qpl_id_map; /* bitmap of used qpl ids */
+};
+
struct gve_priv {
struct net_device *dev;
+ struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
+ struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */
+ struct gve_queue_page_list *qpls; /* array of num qpls */
struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */
dma_addr_t ntfy_block_bus;
struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */
@@ -41,7 +177,18 @@ struct gve_priv {
dma_addr_t counter_array_bus;
u16 num_event_counters;
+ u16 tx_desc_cnt; /* num desc per ring */
+ u16 rx_desc_cnt; /* num desc per ring */
+ u16 tx_pages_per_qpl; /* tx buffer length */
+ u16 rx_pages_per_qpl; /* rx buffer length */
+ u64 max_registered_pages;
+ u64 num_registered_pages; /* num pages registered with NIC */
+ u32 rx_copybreak; /* copy packets smaller than this */
+ u16 default_num_queues; /* default num queues to set up */
+ struct gve_queue_config tx_cfg;
+ struct gve_queue_config rx_cfg;
+ struct gve_qpl_config qpl_cfg; /* map used QPL ids */
u32 num_ntfy_blks; /* spilt between TX and RX so must be even */
struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
@@ -49,6 +196,9 @@ struct gve_priv {
u32 msg_enable; /* level for netif* netdev print macros */
struct pci_dev *pdev;
+ /* metrics */
+ u32 tx_timeo_cnt;
+
/* Admin queue - see gve_adminq.h*/
union gve_adminq_command *adminq;
dma_addr_t adminq_bus_addr;
@@ -132,4 +282,112 @@ static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv,
{
return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];
}
+
+/* Returns the index into ntfy_blocks of the given tx ring's block
+ */
+static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
+{
+ return queue_idx;
+}
+
+/* Returns the index into ntfy_blocks of the given rx ring's block
+ */
+static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
+{
+ return (priv->num_ntfy_blks / 2) + queue_idx;
+}
+
+/* Returns the number of tx queue page lists
+ */
+static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
+{
+ return priv->tx_cfg.num_queues;
+}
+
+/* Returns the number of rx queue page lists
+ */
+static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
+{
+ return priv->rx_cfg.num_queues;
+}
+
+/* Returns a pointer to the next available tx qpl in the list of qpls
+ */
+static inline
+struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv)
+{
+ int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,
+ priv->qpl_cfg.qpl_map_size);
+
+ /* we are out of tx qpls */
+ if (id >= gve_num_tx_qpls(priv))
+ return NULL;
+
+ set_bit(id, priv->qpl_cfg.qpl_id_map);
+ return &priv->qpls[id];
+}
+
+/* Returns a pointer to the next available rx qpl in the list of qpls
+ */
+static inline
+struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv)
+{
+ int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,
+ priv->qpl_cfg.qpl_map_size,
+ gve_num_tx_qpls(priv));
+
+ /* we are out of rx qpls */
+ if (id == priv->qpl_cfg.qpl_map_size)
+ return NULL;
+
+ set_bit(id, priv->qpl_cfg.qpl_id_map);
+ return &priv->qpls[id];
+}
+
+/* Unassigns the qpl with the given id
+ */
+static inline void gve_unassign_qpl(struct gve_priv *priv, int id)
+{
+ clear_bit(id, priv->qpl_cfg.qpl_id_map);
+}
+
+/* Returns the correct dma direction for tx and rx qpls
+ */
+static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv,
+ int id)
+{
+ if (id < gve_num_tx_qpls(priv))
+ return DMA_TO_DEVICE;
+ else
+ return DMA_FROM_DEVICE;
+}
+
+/* Returns true if the max mtu allows page recycling */
+static inline bool gve_can_recycle_pages(struct net_device *dev)
+{
+ /* We can't recycle the pages if we can't fit a packet into half a
+ * page.
+ */
+ return dev->max_mtu <= PAGE_SIZE / 2;
+}
+
+/* buffers */
+int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma,
+ enum dma_data_direction);
+void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
+ enum dma_data_direction);
+/* tx handling */
+netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev);
+bool gve_tx_poll(struct gve_notify_block *block, int budget);
+int gve_tx_alloc_rings(struct gve_priv *priv);
+void gve_tx_free_rings(struct gve_priv *priv);
+__be32 gve_tx_load_event_counter(struct gve_priv *priv,
+ struct gve_tx_ring *tx);
+/* rx handling */
+void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx);
+bool gve_rx_poll(struct gve_notify_block *block, int budget);
+int gve_rx_alloc_rings(struct gve_priv *priv);
+void gve_rx_free_rings(struct gve_priv *priv);
+bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
+ netdev_features_t feat);
#endif /* _GVE_H_ */
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
index 4bf32e4e2810..c3ba7baf0107 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.c
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -190,6 +190,72 @@ int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
return gve_adminq_execute_cmd(priv, &cmd);
}
+int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
+{
+ struct gve_tx_ring *tx = &priv->tx[queue_index];
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
+ cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
+ .queue_id = cpu_to_be32(queue_index),
+ .reserved = 0,
+ .queue_resources_addr = cpu_to_be64(tx->q_resources_bus),
+ .tx_ring_addr = cpu_to_be64(tx->bus),
+ .queue_page_list_id = cpu_to_be32(tx->tx_fifo.qpl->id),
+ .ntfy_id = cpu_to_be32(tx->ntfy_id),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
+int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
+{
+ struct gve_rx_ring *rx = &priv->rx[queue_index];
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
+ cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
+ .queue_id = cpu_to_be32(queue_index),
+ .index = cpu_to_be32(queue_index),
+ .reserved = 0,
+ .ntfy_id = cpu_to_be32(rx->ntfy_id),
+ .queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
+ .rx_desc_ring_addr = cpu_to_be64(rx->desc.bus),
+ .rx_data_ring_addr = cpu_to_be64(rx->data.data_bus),
+ .queue_page_list_id = cpu_to_be32(rx->data.qpl->id),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
+int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
+ cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
+ .queue_id = cpu_to_be32(queue_index),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
+int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
+ cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
+ .queue_id = cpu_to_be32(queue_index),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
int gve_adminq_describe_device(struct gve_priv *priv)
{
struct gve_device_descriptor *descriptor;
@@ -215,6 +281,25 @@ int gve_adminq_describe_device(struct gve_priv *priv)
if (err)
goto free_device_descriptor;
+ priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
+ if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
+ netif_err(priv, drv, priv->dev, "Tx desc count %d too low\n",
+ priv->tx_desc_cnt);
+ err = -EINVAL;
+ goto free_device_descriptor;
+ }
+ priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
+ if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
+ < PAGE_SIZE ||
+ priv->rx_desc_cnt * sizeof(priv->rx->data.data_ring[0])
+ < PAGE_SIZE) {
+ netif_err(priv, drv, priv->dev, "Rx desc count %d too low\n",
+ priv->rx_desc_cnt);
+ err = -EINVAL;
+ goto free_device_descriptor;
+ }
+ priv->max_registered_pages =
+ be64_to_cpu(descriptor->max_registered_pages);
mtu = be16_to_cpu(descriptor->mtu);
if (mtu < ETH_MIN_MTU) {
netif_err(priv, drv, priv->dev, "MTU %d below minimum MTU\n",
@@ -227,6 +312,14 @@ int gve_adminq_describe_device(struct gve_priv *priv)
ether_addr_copy(priv->dev->dev_addr, descriptor->mac);
mac = descriptor->mac;
netif_info(priv, drv, priv->dev, "MAC addr: %pM\n", mac);
+ priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
+ priv->rx_pages_per_qpl = be16_to_cpu(descriptor->rx_pages_per_qpl);
+ if (priv->rx_pages_per_qpl < priv->rx_desc_cnt) {
+ netif_err(priv, drv, priv->dev, "rx_pages_per_qpl cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
+ priv->rx_pages_per_qpl);
+ priv->rx_desc_cnt = priv->rx_pages_per_qpl;
+ }
+ priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
free_device_descriptor:
dma_free_coherent(&priv->pdev->dev, sizeof(*descriptor), descriptor,
@@ -234,6 +327,51 @@ int gve_adminq_describe_device(struct gve_priv *priv)
return err;
}
+int gve_adminq_register_page_list(struct gve_priv *priv,
+ struct gve_queue_page_list *qpl)
+{
+ struct device *hdev = &priv->pdev->dev;
+ u32 num_entries = qpl->num_entries;
+ u32 size = num_entries * sizeof(qpl->page_buses[0]);
+ union gve_adminq_command cmd;
+ dma_addr_t page_list_bus;
+ __be64 *page_list;
+ int err;
+ int i;
+
+ memset(&cmd, 0, sizeof(cmd));
+ page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
+ if (!page_list)
+ return -ENOMEM;
+
+ for (i = 0; i < num_entries; i++)
+ page_list[i] = cpu_to_be64(qpl->page_buses[i]);
+
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
+ cmd.reg_page_list = (struct gve_adminq_register_page_list) {
+ .page_list_id = cpu_to_be32(qpl->id),
+ .num_pages = cpu_to_be32(num_entries),
+ .page_address_list_addr = cpu_to_be64(page_list_bus),
+ };
+
+ err = gve_adminq_execute_cmd(priv, &cmd);
+ dma_free_coherent(hdev, size, page_list, page_list_bus);
+ return err;
+}
+
+int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
+ cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
+ .page_list_id = cpu_to_be32(page_list_id),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
{
union gve_adminq_command cmd;
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
index b860b0d69393..4dfa06edc0f8 100644
--- a/drivers/net/ethernet/google/gve/gve_adminq.h
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -13,6 +13,12 @@
enum gve_adminq_opcodes {
GVE_ADMINQ_DESCRIBE_DEVICE = 0x1,
GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES = 0x2,
+ GVE_ADMINQ_REGISTER_PAGE_LIST = 0x3,
+ GVE_ADMINQ_UNREGISTER_PAGE_LIST = 0x4,
+ GVE_ADMINQ_CREATE_TX_QUEUE = 0x5,
+ GVE_ADMINQ_CREATE_RX_QUEUE = 0x6,
+ GVE_ADMINQ_DESTROY_TX_QUEUE = 0x7,
+ GVE_ADMINQ_DESTROY_RX_QUEUE = 0x8,
GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES = 0x9,
GVE_ADMINQ_SET_DRIVER_PARAMETER = 0xB,
};
@@ -89,6 +95,70 @@ struct gve_adminq_configure_device_resources {
static_assert(sizeof(struct gve_adminq_configure_device_resources) == 32);
+struct gve_adminq_register_page_list {
+ __be32 page_list_id;
+ __be32 num_pages;
+ __be64 page_address_list_addr;
+};
+
+static_assert(sizeof(struct gve_adminq_register_page_list) == 16);
+
+struct gve_adminq_unregister_page_list {
+ __be32 page_list_id;
+};
+
+static_assert(sizeof(struct gve_adminq_unregister_page_list) == 4);
+
+struct gve_adminq_create_tx_queue {
+ __be32 queue_id;
+ __be32 reserved;
+ __be64 queue_resources_addr;
+ __be64 tx_ring_addr;
+ __be32 queue_page_list_id;
+ __be32 ntfy_id;
+};
+
+static_assert(sizeof(struct gve_adminq_create_tx_queue) == 32);
+
+struct gve_adminq_create_rx_queue {
+ __be32 queue_id;
+ __be32 index;
+ __be32 reserved;
+ __be32 ntfy_id;
+ __be64 queue_resources_addr;
+ __be64 rx_desc_ring_addr;
+ __be64 rx_data_ring_addr;
+ __be32 queue_page_list_id;
+ u8 padding[4];
+};
+
+static_assert(sizeof(struct gve_adminq_create_rx_queue) == 48);
+
+/* Queue resources that are shared with the device */
+struct gve_queue_resources {
+ union {
+ struct {
+ __be32 db_index; /* Device -> Guest */
+ __be32 counter_index; /* Device -> Guest */
+ };
+ u8 reserved[64];
+ };
+};
+
+static_assert(sizeof(struct gve_queue_resources) == 64);
+
+struct gve_adminq_destroy_tx_queue {
+ __be32 queue_id;
+};
+
+static_assert(sizeof(struct gve_adminq_destroy_tx_queue) == 4);
+
+struct gve_adminq_destroy_rx_queue {
+ __be32 queue_id;
+};
+
+static_assert(sizeof(struct gve_adminq_destroy_rx_queue) == 4);
+
/* GVE Set Driver Parameter Types */
enum gve_set_driver_param_types {
GVE_SET_PARAM_MTU = 0x1,
@@ -109,7 +179,13 @@ union gve_adminq_command {
union {
struct gve_adminq_configure_device_resources
configure_device_resources;
+ struct gve_adminq_create_tx_queue create_tx_queue;
+ struct gve_adminq_create_rx_queue create_rx_queue;
+ struct gve_adminq_destroy_tx_queue destroy_tx_queue;
+ struct gve_adminq_destroy_rx_queue destroy_rx_queue;
struct gve_adminq_describe_device describe_device;
+ struct gve_adminq_register_page_list reg_page_list;
+ struct gve_adminq_unregister_page_list unreg_page_list;
struct gve_adminq_set_driver_parameter set_driver_param;
};
};
@@ -130,5 +206,12 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv,
dma_addr_t db_array_bus_addr,
u32 num_ntfy_blks);
int gve_adminq_deconfigure_device_resources(struct gve_priv *priv);
+int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_id);
+int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_id);
+int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_id);
+int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_id);
+int gve_adminq_register_page_list(struct gve_priv *priv,
+ struct gve_queue_page_list *qpl);
+int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id);
int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu);
#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_desc.h b/drivers/net/ethernet/google/gve/gve_desc.h
new file mode 100644
index 000000000000..54779871d52e
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_desc.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT)
+ * Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+/* GVE Transmit Descriptor formats */
+
+#ifndef _GVE_DESC_H_
+#define _GVE_DESC_H_
+
+#include <linux/build_bug.h>
+
+/* A note on seg_addrs
+ *
+ * Base addresses encoded in seg_addr are not assumed to be physical
+ * addresses. The ring format assumes these come from some linear address
+ * space. This could be physical memory, kernel virtual memory, user virtual
+ * memory. gVNIC uses lists of registered pages. Each queue is assumed
+ * to be associated with a single such linear address space to ensure a
+ * consistent meaning for seg_addrs posted to its rings.
+ */
+
+struct gve_tx_pkt_desc {
+ u8 type_flags; /* desc type is lower 4 bits, flags upper */
+ u8 l4_csum_offset; /* relative offset of L4 csum word */
+ u8 l4_hdr_offset; /* Offset of start of L4 headers in packet */
+ u8 desc_cnt; /* Total descriptors for this packet */
+ __be16 len; /* Total length of this packet (in bytes) */
+ __be16 seg_len; /* Length of this descriptor's segment */
+ __be64 seg_addr; /* Base address (see note) of this segment */
+} __packed;
+
+struct gve_tx_seg_desc {
+ u8 type_flags; /* type is lower 4 bits, flags upper */
+ u8 l3_offset; /* TSO: 2 byte units to start of IPH */
+ __be16 reserved;
+ __be16 mss; /* TSO MSS */
+ __be16 seg_len;
+ __be64 seg_addr;
+} __packed;
+
+/* GVE Transmit Descriptor Types */
+#define GVE_TXD_STD (0x0 << 4) /* Std with Host Address */
+#define GVE_TXD_TSO (0x1 << 4) /* TSO with Host Address */
+#define GVE_TXD_SEG (0x2 << 4) /* Seg with Host Address */
+
+/* GVE Transmit Descriptor Flags for Std Pkts */
+#define GVE_TXF_L4CSUM BIT(0) /* Need csum offload */
+#define GVE_TXF_TSTAMP BIT(2) /* Timestamp required */
+
+/* GVE Transmit Descriptor Flags for TSO Segs */
+#define GVE_TXSF_IPV6 BIT(1) /* IPv6 TSO */
+
+/* GVE Receive Packet Descriptor */
+/* The start of an ethernet packet comes 2 bytes into the rx buffer.
+ * gVNIC adds this padding so that both the DMA and the L3/4 protocol header
+ * access is aligned.
+ */
+#define GVE_RX_PAD 2
+
+struct gve_rx_desc {
+ u8 padding[48];
+ __be32 rss_hash; /* Receive-side scaling hash (Toeplitz for gVNIC) */
+ __be16 mss;
+ __be16 reserved; /* Reserved to zero */
+ u8 hdr_len; /* Header length (L2-L4) including padding */
+ u8 hdr_off; /* 64-byte-scaled offset into RX_DATA entry */
+ __sum16 csum; /* 1's-complement partial checksum of L3+ bytes */
+ __be16 len; /* Length of the received packet */
+ __be16 flags_seq; /* Flags [15:3] and sequence number [2:0] (1-7) */
+} __packed;
+static_assert(sizeof(struct gve_rx_desc) == 64);
+
+/* As with the Tx ring format, the qpl_offset entries below are offsets into an
+ * ordered list of registered pages.
+ */
+struct gve_rx_data_slot {
+ /* byte offset into the rx registered segment of this slot */
+ __be64 qpl_offset;
+};
+
+/* GVE Recive Packet Descriptor Seq No */
+#define GVE_SEQNO(x) (be16_to_cpu(x) & 0x7)
+
+/* GVE Recive Packet Descriptor Flags */
+#define GVE_RXFLG(x) cpu_to_be16(1 << (3 + (x)))
+#define GVE_RXF_FRAG GVE_RXFLG(3) /* IP Fragment */
+#define GVE_RXF_IPV4 GVE_RXFLG(4) /* IPv4 */
+#define GVE_RXF_IPV6 GVE_RXFLG(5) /* IPv6 */
+#define GVE_RXF_TCP GVE_RXFLG(6) /* TCP Packet */
+#define GVE_RXF_UDP GVE_RXFLG(7) /* UDP Packet */
+#define GVE_RXF_ERR GVE_RXFLG(8) /* Packet Error Detected */
+
+/* GVE IRQ */
+#define GVE_IRQ_ACK BIT(31)
+#define GVE_IRQ_MASK BIT(30)
+#define GVE_IRQ_EVENT BIT(29)
+
+static inline bool gve_needs_rss(__be16 flag)
+{
+ if (flag & GVE_RXF_FRAG)
+ return false;
+ if (flag & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
+ return true;
+ return false;
+}
+
+static inline u8 gve_next_seqno(u8 seq)
+{
+ return (seq + 1) == 8 ? 1 : seq + 1;
+}
+#endif /* _GVE_DESC_H_ */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 41d8cfa8f998..44462dce0254 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -16,6 +16,8 @@
#include "gve_adminq.h"
#include "gve_register.h"
+#define GVE_DEFAULT_RX_COPYBREAK (256)
+
#define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK)
#define GVE_VERSION "1.0.0"
#define GVE_VERSION_PREFIX "GVE-"
@@ -23,6 +25,34 @@
static const char gve_version_str[] = GVE_VERSION;
static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
+static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
+{
+ struct gve_priv *priv = netdev_priv(dev);
+ unsigned int start;
+ int ring;
+
+ if (priv->rx) {
+ for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
+ do {
+ u64_stats_fetch_begin(&priv->rx[ring].statss);
+ s->rx_packets += priv->rx[ring].rpackets;
+ s->rx_bytes += priv->rx[ring].rbytes;
+ } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
+ start));
+ }
+ }
+ if (priv->tx) {
+ for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
+ do {
+ u64_stats_fetch_begin(&priv->tx[ring].statss);
+ s->tx_packets += priv->tx[ring].pkt_done;
+ s->tx_bytes += priv->tx[ring].bytes_done;
+ } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
+ start));
+ }
+ }
+}
+
static int gve_alloc_counter_array(struct gve_priv *priv)
{
priv->counter_array =
@@ -52,9 +82,50 @@ static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
static irqreturn_t gve_intr(int irq, void *arg)
{
+ struct gve_notify_block *block = arg;
+ struct gve_priv *priv = block->priv;
+
+ iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
+ napi_schedule_irqoff(&block->napi);
return IRQ_HANDLED;
}
+static int gve_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct gve_notify_block *block;
+ __be32 __iomem *irq_doorbell;
+ bool reschedule = false;
+ struct gve_priv *priv;
+
+ block = container_of(napi, struct gve_notify_block, napi);
+ priv = block->priv;
+
+ if (block->tx)
+ reschedule |= gve_tx_poll(block, budget);
+ if (block->rx)
+ reschedule |= gve_rx_poll(block, budget);
+
+ if (reschedule)
+ return budget;
+
+ napi_complete(napi);
+ irq_doorbell = gve_irq_doorbell(priv, block);
+ iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell);
+
+ /* Double check we have no extra work.
+ * Ensure unmask synchronizes with checking for work.
+ */
+ dma_rmb();
+ if (block->tx)
+ reschedule |= gve_tx_poll(block, -1);
+ if (block->rx)
+ reschedule |= gve_rx_poll(block, -1);
+ if (reschedule && napi_reschedule(napi))
+ iowrite32be(GVE_IRQ_MASK, irq_doorbell);
+
+ return 0;
+}
+
static int gve_alloc_notify_blocks(struct gve_priv *priv)
{
int num_vecs_requested = priv->num_ntfy_blks + 1;
@@ -79,10 +150,23 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv)
goto abort_with_msix_vectors;
}
if (vecs_enabled != num_vecs_requested) {
- priv->num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
+ int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
+ int vecs_per_type = new_num_ntfy_blks / 2;
+ int vecs_left = new_num_ntfy_blks % 2;
+
+ priv->num_ntfy_blks = new_num_ntfy_blks;
+ priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues,
+ vecs_per_type);
+ priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues,
+ vecs_per_type + vecs_left);
dev_err(&priv->pdev->dev,
- "Only received %d msix. Lowering number of notification blocks to %d\n",
- vecs_enabled, priv->num_ntfy_blks);
+ "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n",
+ vecs_enabled, priv->tx_cfg.max_queues,
+ priv->rx_cfg.max_queues);
+ if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)
+ priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
+ if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues)
+ priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
}
/* Half the notification blocks go to TX and half to RX */
active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus());
@@ -219,6 +303,463 @@ static void gve_teardown_device_resources(struct gve_priv *priv)
gve_clear_device_resources_ok(priv);
}
+static void gve_add_napi(struct gve_priv *priv, int ntfy_idx)
+{
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+
+ netif_napi_add(priv->dev, &block->napi, gve_napi_poll,
+ NAPI_POLL_WEIGHT);
+}
+
+static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
+{
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+
+ netif_napi_del(&block->napi);
+}
+
+static int gve_register_qpls(struct gve_priv *priv)
+{
+ int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+ int err;
+ int i;
+
+ for (i = 0; i < num_qpls; i++) {
+ err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to register queue page list %d\n",
+ priv->qpls[i].id);
+ return err;
+ }
+ }
+ return 0;
+}
+
+static int gve_unregister_qpls(struct gve_priv *priv)
+{
+ int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+ int err;
+ int i;
+
+ for (i = 0; i < num_qpls; i++) {
+ err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "Failed to unregister queue page list %d\n",
+ priv->qpls[i].id);
+ return err;
+ }
+ }
+ return 0;
+}
+
+static int gve_create_rings(struct gve_priv *priv)
+{
+ int err;
+ int i;
+
+ for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+ err = gve_adminq_create_tx_queue(priv, i);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "failed to create tx queue %d\n",
+ i);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "created tx queue %d\n", i);
+ }
+ for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+ err = gve_adminq_create_rx_queue(priv, i);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "failed to create rx queue %d\n",
+ i);
+ return err;
+ }
+ /* Rx data ring has been prefilled with packet buffers at
+ * queue allocation time.
+ * Write the doorbell to provide descriptor slots and packet
+ * buffers to the NIC.
+ */
+ gve_rx_write_doorbell(priv, &priv->rx[i]);
+ netif_dbg(priv, drv, priv->dev, "created rx queue %d\n", i);
+ }
+
+ return 0;
+}
+
+static int gve_alloc_rings(struct gve_priv *priv)
+{
+ int ntfy_idx;
+ int err;
+ int i;
+
+ /* Setup tx rings */
+ priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx),
+ GFP_KERNEL);
+ if (!priv->tx)
+ return -ENOMEM;
+ err = gve_tx_alloc_rings(priv);
+ if (err)
+ goto free_tx;
+ /* Setup rx rings */
+ priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx),
+ GFP_KERNEL);
+ if (!priv->rx) {
+ err = -ENOMEM;
+ goto free_tx_queue;
+ }
+ err = gve_rx_alloc_rings(priv);
+ if (err)
+ goto free_rx;
+ /* Add tx napi & init sync stats*/
+ for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+ u64_stats_init(&priv->tx[i].statss);
+ ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
+ gve_add_napi(priv, ntfy_idx);
+ }
+ /* Add rx napi & init sync stats*/
+ for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+ u64_stats_init(&priv->rx[i].statss);
+ ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
+ gve_add_napi(priv, ntfy_idx);
+ }
+
+ return 0;
+
+free_rx:
+ kfree(priv->rx);
+ priv->rx = NULL;
+free_tx_queue:
+ gve_tx_free_rings(priv);
+free_tx:
+ kfree(priv->tx);
+ priv->tx = NULL;
+ return err;
+}
+
+static int gve_destroy_rings(struct gve_priv *priv)
+{
+ int err;
+ int i;
+
+ for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+ err = gve_adminq_destroy_tx_queue(priv, i);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to destroy tx queue %d\n",
+ i);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "destroyed tx queue %d\n", i);
+ }
+ for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+ err = gve_adminq_destroy_rx_queue(priv, i);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "failed to destroy rx queue %d\n",
+ i);
+ return err;
+ }
+ netif_dbg(priv, drv, priv->dev, "destroyed rx queue %d\n", i);
+ }
+ return 0;
+}
+
+static void gve_free_rings(struct gve_priv *priv)
+{
+ int ntfy_idx;
+ int i;
+
+ if (priv->tx) {
+ for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+ ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
+ gve_remove_napi(priv, ntfy_idx);
+ }
+ gve_tx_free_rings(priv);
+ kfree(priv->tx);
+ priv->tx = NULL;
+ }
+ if (priv->rx) {
+ for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+ ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
+ gve_remove_napi(priv, ntfy_idx);
+ }
+ gve_rx_free_rings(priv);
+ kfree(priv->rx);
+ priv->rx = NULL;
+ }
+}
+
+int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma,
+ enum dma_data_direction dir)
+{
+ *page = alloc_page(GFP_KERNEL);
+ if (!page)
+ return -ENOMEM;
+ *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir);
+ if (dma_mapping_error(dev, *dma)) {
+ put_page(*page);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id,
+ int pages)
+{
+ struct gve_queue_page_list *qpl = &priv->qpls[id];
+ int err;
+ int i;
+
+ if (pages + priv->num_registered_pages > priv->max_registered_pages) {
+ netif_err(priv, drv, priv->dev,
+ "Reached max number of registered pages %llu > %llu\n",
+ pages + priv->num_registered_pages,
+ priv->max_registered_pages);
+ return -EINVAL;
+ }
+
+ qpl->id = id;
+ qpl->num_entries = pages;
+ qpl->pages = kvzalloc(pages * sizeof(*qpl->pages), GFP_KERNEL);
+ /* caller handles clean up */
+ if (!qpl->pages)
+ return -ENOMEM;
+ qpl->page_buses = kvzalloc(pages * sizeof(*qpl->page_buses),
+ GFP_KERNEL);
+ /* caller handles clean up */
+ if (!qpl->page_buses)
+ return -ENOMEM;
+
+ for (i = 0; i < pages; i++) {
+ err = gve_alloc_page(&priv->pdev->dev, &qpl->pages[i],
+ &qpl->page_buses[i],
+ gve_qpl_dma_dir(priv, id));
+ /* caller handles clean up */
+ if (err)
+ return -ENOMEM;
+ }
+ priv->num_registered_pages += pages;
+
+ return 0;
+}
+
+void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
+ enum dma_data_direction dir)
+{
+ if (!dma_mapping_error(dev, dma))
+ dma_unmap_page(dev, dma, PAGE_SIZE, dir);
+ if (page)
+ put_page(page);
+}
+
+static void gve_free_queue_page_list(struct gve_priv *priv,
+ int id)
+{
+ struct gve_queue_page_list *qpl = &priv->qpls[id];
+ int i;
+
+ if (!qpl->pages)
+ return;
+ if (!qpl->page_buses)
+ goto free_pages;
+
+ for (i = 0; i < qpl->num_entries; i++)
+ gve_free_page(&priv->pdev->dev, qpl->pages[i],
+ qpl->page_buses[i], gve_qpl_dma_dir(priv, id));
+
+ kfree(qpl->page_buses);
+free_pages:
+ kfree(qpl->pages);
+ priv->num_registered_pages -= qpl->num_entries;
+}
+
+static int gve_alloc_qpls(struct gve_priv *priv)
+{
+ int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+ int i, j;
+ int err;
+
+ priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL);
+ if (!priv->qpls)
+ return -ENOMEM;
+
+ for (i = 0; i < gve_num_tx_qpls(priv); i++) {
+ err = gve_alloc_queue_page_list(priv, i,
+ priv->tx_pages_per_qpl);
+ if (err)
+ goto free_qpls;
+ }
+ for (; i < num_qpls; i++) {
+ err = gve_alloc_queue_page_list(priv, i,
+ priv->rx_pages_per_qpl);
+ if (err)
+ goto free_qpls;
+ }
+
+ priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) *
+ sizeof(unsigned long) * BITS_PER_BYTE;
+ priv->qpl_cfg.qpl_id_map = kvzalloc(BITS_TO_LONGS(num_qpls) *
+ sizeof(unsigned long), GFP_KERNEL);
+ if (!priv->qpl_cfg.qpl_id_map)
+ goto free_qpls;
+
+ return 0;
+
+free_qpls:
+ for (j = 0; j <= i; j++)
+ gve_free_queue_page_list(priv, j);
+ kfree(priv->qpls);
+ return err;
+}
+
+static void gve_free_qpls(struct gve_priv *priv)
+{
+ int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
+ int i;
+
+ kfree(priv->qpl_cfg.qpl_id_map);
+
+ for (i = 0; i < num_qpls; i++)
+ gve_free_queue_page_list(priv, i);
+
+ kfree(priv->qpls);
+}
+
+static void gve_turndown(struct gve_priv *priv);
+static void gve_turnup(struct gve_priv *priv);
+
+static int gve_open(struct net_device *dev)
+{
+ struct gve_priv *priv = netdev_priv(dev);
+ int err;
+
+ err = gve_alloc_qpls(priv);
+ if (err)
+ return err;
+ err = gve_alloc_rings(priv);
+ if (err)
+ goto free_qpls;
+
+ err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues);
+ if (err)
+ goto free_rings;
+ err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues);
+ if (err)
+ goto free_rings;
+
+ err = gve_register_qpls(priv);
+ if (err)
+ return err;
+ err = gve_create_rings(priv);
+ if (err)
+ return err;
+ gve_set_device_rings_ok(priv);
+
+ gve_turnup(priv);
+ netif_carrier_on(dev);
+ return 0;
+
+free_rings:
+ gve_free_rings(priv);
+free_qpls:
+ gve_free_qpls(priv);
+ return err;
+}
+
+static int gve_close(struct net_device *dev)
+{
+ struct gve_priv *priv = netdev_priv(dev);
+ int err;
+
+ netif_carrier_off(dev);
+ if (gve_get_device_rings_ok(priv)) {
+ gve_turndown(priv);
+ err = gve_destroy_rings(priv);
+ if (err)
+ return err;
+ err = gve_unregister_qpls(priv);
+ if (err)
+ return err;
+ gve_clear_device_rings_ok(priv);
+ }
+
+ gve_free_rings(priv);
+ gve_free_qpls(priv);
+ return 0;
+}
+
+static void gve_turndown(struct gve_priv *priv)
+{
+ int idx;
+
+ if (netif_carrier_ok(priv->dev))
+ netif_carrier_off(priv->dev);
+
+ if (!gve_get_napi_enabled(priv))
+ return;
+
+ /* Disable napi to prevent more work from coming in */
+ for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
+ int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+
+ napi_disable(&block->napi);
+ }
+ for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
+ int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+
+ napi_disable(&block->napi);
+ }
+
+ /* Stop tx queues */
+ netif_tx_disable(priv->dev);
+
+ gve_clear_napi_enabled(priv);
+}
+
+static void gve_turnup(struct gve_priv *priv)
+{
+ int idx;
+
+ /* Start the tx queues */
+ netif_tx_start_all_queues(priv->dev);
+
+ /* Enable napi and unmask interrupts for all queues */
+ for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
+ int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+
+ napi_enable(&block->napi);
+ iowrite32be(0, gve_irq_doorbell(priv, block));
+ }
+ for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
+ int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+
+ napi_enable(&block->napi);
+ iowrite32be(0, gve_irq_doorbell(priv, block));
+ }
+
+ gve_set_napi_enabled(priv);
+}
+
+static void gve_tx_timeout(struct net_device *dev)
+{
+ struct gve_priv *priv = netdev_priv(dev);
+
+ priv->tx_timeo_cnt++;
+}
+
+static const struct net_device_ops gve_netdev_ops = {
+ .ndo_start_xmit = gve_tx,
+ .ndo_open = gve_open,
+ .ndo_stop = gve_close,
+ .ndo_get_stats64 = gve_get_stats,
+ .ndo_tx_timeout = gve_tx_timeout,
+};
+
static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
{
int num_ntfy;
@@ -264,12 +805,33 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
goto err;
}
+ priv->num_registered_pages = 0;
+ priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
/* gvnic has one Notification Block per MSI-x vector, except for the
* management vector
*/
priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
priv->mgmt_msix_idx = priv->num_ntfy_blks;
+ priv->tx_cfg.max_queues =
+ min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
+ priv->rx_cfg.max_queues =
+ min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
+
+ priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
+ priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
+ if (priv->default_num_queues > 0) {
+ priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
+ priv->tx_cfg.num_queues);
+ priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
+ priv->rx_cfg.num_queues);
+ }
+
+ netif_info(priv, drv, priv->dev, "TX queues %d, RX queues %d\n",
+ priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
+ netif_info(priv, drv, priv->dev, "Max TX queues %d, Max RX queues %d\n",
+ priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
+
setup_device:
err = gve_setup_device_resources(priv);
if (!err)
@@ -336,6 +898,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0);
if (!reg_bar) {
+ dev_err(&pdev->dev, "Failed to map pci bar!\n");
err = -ENOMEM;
goto abort_with_pci_region;
}
@@ -359,6 +922,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
SET_NETDEV_DEV(dev, &pdev->dev);
pci_set_drvdata(pdev, dev);
+ dev->netdev_ops = &gve_netdev_ops;
/* advertise features */
dev->hw_features = NETIF_F_HIGHDMA;
dev->hw_features |= NETIF_F_SG;
@@ -369,6 +933,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
dev->hw_features |= NETIF_F_RXCSUM;
dev->hw_features |= NETIF_F_RXHASH;
dev->features = dev->hw_features;
+ dev->watchdog_timeo = 5 * HZ;
dev->min_mtu = ETH_MIN_MTU;
netif_carrier_off(dev);
@@ -379,6 +944,8 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
priv->reg_bar0 = reg_bar;
priv->db_bar2 = db_bar;
priv->state_flags = 0x0;
+ priv->tx_cfg.max_queues = max_tx_queues;
+ priv->rx_cfg.max_queues = max_rx_queues;
err = gve_init_priv(priv, false);
if (err)
diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c
new file mode 100644
index 000000000000..2254857eac50
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_rx.c
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#include "gve.h"
+#include "gve_adminq.h"
+#include <linux/etherdevice.h>
+
+static void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx)
+{
+ struct gve_notify_block *block =
+ &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)];
+
+ block->rx = NULL;
+}
+
+static void gve_rx_free_ring(struct gve_priv *priv, int idx)
+{
+ struct gve_rx_ring *rx = &priv->rx[idx];
+ struct device *dev = &priv->pdev->dev;
+ size_t bytes;
+ u32 slots;
+
+ gve_rx_remove_from_block(priv, idx);
+
+ bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
+ dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
+ rx->desc.desc_ring = NULL;
+
+ dma_free_coherent(dev, sizeof(*rx->q_resources),
+ rx->q_resources, rx->q_resources_bus);
+ rx->q_resources = NULL;
+
+ gve_unassign_qpl(priv, rx->data.qpl->id);
+ rx->data.qpl = NULL;
+ kfree(rx->data.page_info);
+
+ slots = rx->data.mask + 1;
+ bytes = sizeof(*rx->data.data_ring) * slots;
+ dma_free_coherent(dev, bytes, rx->data.data_ring,
+ rx->data.data_bus);
+ rx->data.data_ring = NULL;
+ netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
+}
+
+static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
+ struct gve_rx_data_slot *slot,
+ dma_addr_t addr, struct page *page)
+{
+ page_info->page = page;
+ page_info->page_offset = 0;
+ page_info->page_address = page_address(page);
+ slot->qpl_offset = cpu_to_be64(addr);
+}
+
+static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
+{
+ struct gve_priv *priv = rx->gve;
+ u32 slots;
+ int i;
+
+ /* Allocate one page per Rx queue slot. Each page is split into two
+ * packet buffers, when possible we "page flip" between the two.
+ */
+ slots = rx->data.mask + 1;
+
+ rx->data.page_info = kvzalloc(slots *
+ sizeof(*rx->data.page_info), GFP_KERNEL);
+ if (!rx->data.page_info)
+ return -ENOMEM;
+
+ rx->data.qpl = gve_assign_rx_qpl(priv);
+
+ for (i = 0; i < slots; i++) {
+ struct page *page = rx->data.qpl->pages[i];
+ dma_addr_t addr = i * PAGE_SIZE;
+
+ gve_setup_rx_buffer(&rx->data.page_info[i],
+ &rx->data.data_ring[i], addr, page);
+ }
+
+ return slots;
+}
+
+static void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)
+{
+ u32 ntfy_idx = gve_rx_idx_to_ntfy(priv, queue_idx);
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+ struct gve_rx_ring *rx = &priv->rx[queue_idx];
+
+ block->rx = rx;
+ rx->ntfy_id = ntfy_idx;
+}
+
+static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
+{
+ struct gve_rx_ring *rx = &priv->rx[idx];
+ struct device *hdev = &priv->pdev->dev;
+ u32 slots, npages;
+ int filled_pages;
+ size_t bytes;
+ int err;
+
+ netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
+ /* Make sure everything is zeroed to start with */
+ memset(rx, 0, sizeof(*rx));
+
+ rx->gve = priv;
+ rx->q_num = idx;
+
+ slots = priv->rx_pages_per_qpl;
+ rx->data.mask = slots - 1;
+
+ /* alloc rx data ring */
+ bytes = sizeof(*rx->data.data_ring) * slots;
+ rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
+ &rx->data.data_bus,
+ GFP_KERNEL);
+ if (!rx->data.data_ring)
+ return -ENOMEM;
+ filled_pages = gve_prefill_rx_pages(rx);
+ if (filled_pages < 0) {
+ err = -ENOMEM;
+ goto abort_with_slots;
+ }
+ rx->desc.fill_cnt = filled_pages;
+ /* Ensure data ring slots (packet buffers) are visible. */
+ dma_wmb();
+
+ /* Alloc gve_queue_resources */
+ rx->q_resources =
+ dma_alloc_coherent(hdev,
+ sizeof(*rx->q_resources),
+ &rx->q_resources_bus,
+ GFP_KERNEL);
+ if (!rx->q_resources) {
+ err = -ENOMEM;
+ goto abort_filled;
+ }
+ netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
+ (unsigned long)rx->data.data_bus);
+
+ /* alloc rx desc ring */
+ bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
+ npages = bytes / PAGE_SIZE;
+ if (npages * PAGE_SIZE != bytes) {
+ err = -EIO;
+ goto abort_with_q_resources;
+ }
+
+ rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
+ GFP_KERNEL);
+ if (!rx->desc.desc_ring) {
+ err = -ENOMEM;
+ goto abort_with_q_resources;
+ }
+ rx->desc.mask = slots - 1;
+ rx->desc.cnt = 0;
+ rx->desc.seqno = 1;
+ gve_rx_add_to_block(priv, idx);
+
+ return 0;
+
+abort_with_q_resources:
+ dma_free_coherent(hdev, sizeof(*rx->q_resources),
+ rx->q_resources, rx->q_resources_bus);
+ rx->q_resources = NULL;
+abort_filled:
+ kfree(rx->data.page_info);
+abort_with_slots:
+ bytes = sizeof(*rx->data.data_ring) * slots;
+ dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
+ rx->data.data_ring = NULL;
+
+ return err;
+}
+
+int gve_rx_alloc_rings(struct gve_priv *priv)
+{
+ int err = 0;
+ int i;
+
+ for (i = 0; i < priv->rx_cfg.num_queues; i++) {
+ err = gve_rx_alloc_ring(priv, i);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "Failed to alloc rx ring=%d: err=%d\n",
+ i, err);
+ break;
+ }
+ }
+ /* Unallocate if there was an error */
+ if (err) {
+ int j;
+
+ for (j = 0; j < i; j++)
+ gve_rx_free_ring(priv, j);
+ }
+ return err;
+}
+
+void gve_rx_free_rings(struct gve_priv *priv)
+{
+ int i;
+
+ for (i = 0; i < priv->rx_cfg.num_queues; i++)
+ gve_rx_free_ring(priv, i);
+}
+
+void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
+{
+ u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
+
+ iowrite32be(rx->desc.fill_cnt, &priv->db_bar2[db_idx]);
+}
+
+static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
+{
+ if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
+ return PKT_HASH_TYPE_L4;
+ if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
+ return PKT_HASH_TYPE_L3;
+ return PKT_HASH_TYPE_L2;
+}
+
+static struct sk_buff *gve_rx_copy(struct net_device *dev,
+ struct napi_struct *napi,
+ struct gve_rx_slot_page_info *page_info,
+ u16 len)
+{
+ struct sk_buff *skb = napi_alloc_skb(napi, len);
+ void *va = page_info->page_address + GVE_RX_PAD +
+ page_info->page_offset;
+
+ if (unlikely(!skb))
+ return NULL;
+
+ __skb_put(skb, len);
+
+ skb_copy_to_linear_data(skb, va, len);
+
+ skb->protocol = eth_type_trans(skb, dev);
+ return skb;
+}
+
+static struct sk_buff *gve_rx_add_frags(struct net_device *dev,
+ struct napi_struct *napi,
+ struct gve_rx_slot_page_info *page_info,
+ u16 len)
+{
+ struct sk_buff *skb = napi_get_frags(napi);
+
+ if (unlikely(!skb))
+ return NULL;
+
+ skb_add_rx_frag(skb, 0, page_info->page,
+ page_info->page_offset +
+ GVE_RX_PAD, len, PAGE_SIZE / 2);
+
+ return skb;
+}
+
+static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info,
+ struct gve_rx_data_slot *data_ring)
+{
+ u64 addr = be64_to_cpu(data_ring->qpl_offset);
+
+ page_info->page_offset ^= PAGE_SIZE / 2;
+ addr ^= PAGE_SIZE / 2;
+ data_ring->qpl_offset = cpu_to_be64(addr);
+}
+
+static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
+ netdev_features_t feat)
+{
+ struct gve_rx_slot_page_info *page_info;
+ struct gve_priv *priv = rx->gve;
+ struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
+ struct net_device *dev = priv->dev;
+ struct sk_buff *skb;
+ int pagecount;
+ u16 len;
+ u32 idx;
+
+ /* drop this packet */
+ if (unlikely(rx_desc->flags_seq & GVE_RXF_ERR))
+ return true;
+
+ len = be16_to_cpu(rx_desc->len) - GVE_RX_PAD;
+ idx = rx->data.cnt & rx->data.mask;
+ page_info = &rx->data.page_info[idx];
+
+ /* gvnic can only receive into registered segments. If the buffer
+ * can't be recycled, our only choice is to copy the data out of
+ * it so that we can return it to the device.
+ */
+
+#if PAGE_SIZE == 4096
+ if (len <= priv->rx_copybreak) {
+ /* Just copy small packets */
+ skb = gve_rx_copy(dev, napi, page_info, len);
+ goto have_skb;
+ }
+ if (unlikely(!gve_can_recycle_pages(dev))) {
+ skb = gve_rx_copy(dev, napi, page_info, len);
+ goto have_skb;
+ }
+ pagecount = page_count(page_info->page);
+ if (pagecount == 1) {
+ /* No part of this page is used by any SKBs; we attach
+ * the page fragment to a new SKB and pass it up the
+ * stack.
+ */
+ skb = gve_rx_add_frags(dev, napi, page_info, len);
+ if (!skb)
+ return true;
+ /* Make sure the kernel stack can't release the page */
+ get_page(page_info->page);
+ /* "flip" to other packet buffer on this page */
+ gve_rx_flip_buff(page_info, &rx->data.data_ring[idx]);
+ } else if (pagecount >= 2) {
+ /* We have previously passed the other half of this
+ * page up the stack, but it has not yet been freed.
+ */
+ skb = gve_rx_copy(dev, napi, page_info, len);
+ } else {
+ WARN(pagecount < 1, "Pagecount should never be < 1");
+ return false;
+ }
+#else
+ skb = gve_rx_copy(dev, napi, page_info, len);
+#endif
+
+have_skb:
+ if (!skb)
+ return true;
+
+ rx->data.cnt++;
+
+ if (likely(feat & NETIF_F_RXCSUM)) {
+ /* NIC passes up the partial sum */
+ if (rx_desc->csum)
+ skb->ip_summed = CHECKSUM_COMPLETE;
+ else
+ skb->ip_summed = CHECKSUM_NONE;
+ skb->csum = csum_unfold(rx_desc->csum);
+ }
+
+ /* parse flags & pass relevant info up */
+ if (likely(feat & NETIF_F_RXHASH) &&
+ gve_needs_rss(rx_desc->flags_seq))
+ skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
+ gve_rss_type(rx_desc->flags_seq));
+
+ if (skb_is_nonlinear(skb))
+ napi_gro_frags(napi);
+ else
+ napi_gro_receive(napi, skb);
+ return true;
+}
+
+static bool gve_rx_work_pending(struct gve_rx_ring *rx)
+{
+ struct gve_rx_desc *desc;
+ __be16 flags_seq;
+ u32 next_idx;
+
+ next_idx = rx->desc.cnt & rx->desc.mask;
+ desc = rx->desc.desc_ring + next_idx;
+
+ flags_seq = desc->flags_seq;
+ /* Make sure we have synchronized the seq no with the device */
+ smp_rmb();
+
+ return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
+}
+
+bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
+ netdev_features_t feat)
+{
+ struct gve_priv *priv = rx->gve;
+ struct gve_rx_desc *desc;
+ u32 cnt = rx->desc.cnt;
+ u32 idx = cnt & rx->desc.mask;
+ u32 work_done = 0;
+ u64 bytes = 0;
+
+ desc = rx->desc.desc_ring + idx;
+ while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
+ work_done < budget) {
+ netif_info(priv, rx_status, priv->dev,
+ "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
+ rx->q_num, idx, desc, desc->flags_seq);
+ netif_info(priv, rx_status, priv->dev,
+ "[%d] seqno=%d rx->desc.seqno=%d\n",
+ rx->q_num, GVE_SEQNO(desc->flags_seq),
+ rx->desc.seqno);
+ bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
+ if (!gve_rx(rx, desc, feat))
+ return false;
+ cnt++;
+ idx = cnt & rx->desc.mask;
+ desc = rx->desc.desc_ring + idx;
+ rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
+ work_done++;
+ }
+
+ if (!work_done)
+ return false;
+
+ u64_stats_update_begin(&rx->statss);
+ rx->rpackets += work_done;
+ rx->rbytes += bytes;
+ u64_stats_update_end(&rx->statss);
+ rx->desc.cnt = cnt;
+ rx->desc.fill_cnt += work_done;
+
+ /* restock desc ring slots */
+ dma_wmb(); /* Ensure descs are visible before ringing doorbell */
+ gve_rx_write_doorbell(priv, rx);
+ return gve_rx_work_pending(rx);
+}
+
+bool gve_rx_poll(struct gve_notify_block *block, int budget)
+{
+ struct gve_rx_ring *rx = block->rx;
+ netdev_features_t feat;
+ bool repoll = false;
+
+ feat = block->napi.dev->features;
+
+ /* If budget is 0, do all the work */
+ if (budget == 0)
+ budget = INT_MAX;
+
+ if (budget > 0)
+ repoll |= gve_clean_rx_done(rx, budget, feat);
+ else
+ repoll |= gve_rx_work_pending(rx);
+ return repoll;
+}
diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c
new file mode 100644
index 000000000000..778b87b5a06c
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_tx.c
@@ -0,0 +1,584 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#include "gve.h"
+#include "gve_adminq.h"
+#include <linux/ip.h>
+#include <linux/tcp.h>
+#include <linux/vmalloc.h>
+#include <linux/skbuff.h>
+
+static inline void gve_tx_put_doorbell(struct gve_priv *priv,
+ struct gve_queue_resources *q_resources,
+ u32 val)
+{
+ iowrite32be(val, &priv->db_bar2[be32_to_cpu(q_resources->db_index)]);
+}
+
+/* gvnic can only transmit from a Registered Segment.
+ * We copy skb payloads into the registered segment before writing Tx
+ * descriptors and ringing the Tx doorbell.
+ *
+ * gve_tx_fifo_* manages the Registered Segment as a FIFO - clients must
+ * free allocations in the order they were allocated.
+ */
+
+static int gve_tx_fifo_init(struct gve_priv *priv, struct gve_tx_fifo *fifo)
+{
+ fifo->base = vmap(fifo->qpl->pages, fifo->qpl->num_entries, VM_MAP,
+ PAGE_KERNEL);
+ if (unlikely(!fifo->base)) {
+ netif_err(priv, drv, priv->dev, "Failed to vmap fifo, qpl_id = %d\n",
+ fifo->qpl->id);
+ return -ENOMEM;
+ }
+
+ fifo->size = fifo->qpl->num_entries * PAGE_SIZE;
+ atomic_set(&fifo->available, fifo->size);
+ fifo->head = 0;
+ return 0;
+}
+
+static void gve_tx_fifo_release(struct gve_priv *priv, struct gve_tx_fifo *fifo)
+{
+ WARN(atomic_read(&fifo->available) != fifo->size,
+ "Releasing non-empty fifo");
+
+ vunmap(fifo->base);
+}
+
+static int gve_tx_fifo_pad_alloc_one_frag(struct gve_tx_fifo *fifo,
+ size_t bytes)
+{
+ return (fifo->head + bytes < fifo->size) ? 0 : fifo->size - fifo->head;
+}
+
+static bool gve_tx_fifo_can_alloc(struct gve_tx_fifo *fifo, size_t bytes)
+{
+ return (atomic_read(&fifo->available) <= bytes) ? false : true;
+}
+
+/* gve_tx_alloc_fifo - Allocate fragment(s) from Tx FIFO
+ * @fifo: FIFO to allocate from
+ * @bytes: Allocation size
+ * @iov: Scatter-gather elements to fill with allocation fragment base/len
+ *
+ * Returns number of valid elements in iov[] or negative on error.
+ *
+ * Allocations from a given FIFO must be externally synchronized but concurrent
+ * allocation and frees are allowed.
+ */
+static int gve_tx_alloc_fifo(struct gve_tx_fifo *fifo, size_t bytes,
+ struct gve_tx_iovec iov[2])
+{
+ size_t overflow, padding;
+ u32 aligned_head;
+ int nfrags = 0;
+
+ if (!bytes)
+ return 0;
+
+ /* This check happens before we know how much padding is needed to
+ * align to a cacheline boundary for the payload, but that is fine,
+ * because the FIFO head always start aligned, and the FIFO's boundaries
+ * are aligned, so if there is space for the data, there is space for
+ * the padding to the next alignment.
+ */
+ WARN(!gve_tx_fifo_can_alloc(fifo, bytes),
+ "Reached %s when there's not enough space in the fifo", __func__);
+
+ nfrags++;
+
+ iov[0].iov_offset = fifo->head;
+ iov[0].iov_len = bytes;
+ fifo->head += bytes;
+
+ if (fifo->head > fifo->size) {
+ /* If the allocation did not fit in the tail fragment of the
+ * FIFO, also use the head fragment.
+ */
+ nfrags++;
+ overflow = fifo->head - fifo->size;
+ iov[0].iov_len -= overflow;
+ iov[1].iov_offset = 0; /* Start of fifo*/
+ iov[1].iov_len = overflow;
+
+ fifo->head = overflow;
+ }
+
+ /* Re-align to a cacheline boundary */
+ aligned_head = L1_CACHE_ALIGN(fifo->head);
+ padding = aligned_head - fifo->head;
+ iov[nfrags - 1].iov_padding = padding;
+ atomic_sub(bytes + padding, &fifo->available);
+ fifo->head = aligned_head;
+
+ if (fifo->head == fifo->size)
+ fifo->head = 0;
+
+ return nfrags;
+}
+
+/* gve_tx_free_fifo - Return space to Tx FIFO
+ * @fifo: FIFO to return fragments to
+ * @bytes: Bytes to free
+ */
+static void gve_tx_free_fifo(struct gve_tx_fifo *fifo, size_t bytes)
+{
+ atomic_add(bytes, &fifo->available);
+}
+
+static void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx)
+{
+ struct gve_notify_block *block =
+ &priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)];
+
+ block->tx = NULL;
+}
+
+static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
+ u32 to_do, bool try_to_wake);
+
+static void gve_tx_free_ring(struct gve_priv *priv, int idx)
+{
+ struct gve_tx_ring *tx = &priv->tx[idx];
+ struct device *hdev = &priv->pdev->dev;
+ size_t bytes;
+ u32 slots;
+
+ gve_tx_remove_from_block(priv, idx);
+ slots = tx->mask + 1;
+ gve_clean_tx_done(priv, tx, tx->req, false);
+ netdev_tx_reset_queue(tx->netdev_txq);
+
+ dma_free_coherent(hdev, sizeof(*tx->q_resources),
+ tx->q_resources, tx->q_resources_bus);
+ tx->q_resources = NULL;
+
+ gve_tx_fifo_release(priv, &tx->tx_fifo);
+ gve_unassign_qpl(priv, tx->tx_fifo.qpl->id);
+ tx->tx_fifo.qpl = NULL;
+
+ bytes = sizeof(*tx->desc) * slots;
+ dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
+ tx->desc = NULL;
+
+ vfree(tx->info);
+ tx->info = NULL;
+
+ netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx);
+}
+
+static void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx)
+{
+ int ntfy_idx = gve_tx_idx_to_ntfy(priv, queue_idx);
+ struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
+ struct gve_tx_ring *tx = &priv->tx[queue_idx];
+
+ block->tx = tx;
+ tx->ntfy_id = ntfy_idx;
+}
+
+static int gve_tx_alloc_ring(struct gve_priv *priv, int idx)
+{
+ struct gve_tx_ring *tx = &priv->tx[idx];
+ struct device *hdev = &priv->pdev->dev;
+ u32 slots = priv->tx_desc_cnt;
+ size_t bytes;
+
+ /* Make sure everything is zeroed to start */
+ memset(tx, 0, sizeof(*tx));
+ tx->q_num = idx;
+
+ tx->mask = slots - 1;
+
+ /* alloc metadata */
+ tx->info = vzalloc(sizeof(*tx->info) * slots);
+ if (!tx->info)
+ return -ENOMEM;
+
+ /* alloc tx queue */
+ bytes = sizeof(*tx->desc) * slots;
+ tx->desc = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);
+ if (!tx->desc)
+ goto abort_with_info;
+
+ tx->tx_fifo.qpl = gve_assign_tx_qpl(priv);
+
+ /* map Tx FIFO */
+ if (gve_tx_fifo_init(priv, &tx->tx_fifo))
+ goto abort_with_desc;
+
+ tx->q_resources =
+ dma_alloc_coherent(hdev,
+ sizeof(*tx->q_resources),
+ &tx->q_resources_bus,
+ GFP_KERNEL);
+ if (!tx->q_resources)
+ goto abort_with_fifo;
+
+ netif_dbg(priv, drv, priv->dev, "tx[%d]->bus=%lx\n", idx,
+ (unsigned long)tx->bus);
+ tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
+ gve_tx_add_to_block(priv, idx);
+
+ return 0;
+
+abort_with_fifo:
+ gve_tx_fifo_release(priv, &tx->tx_fifo);
+abort_with_desc:
+ dma_free_coherent(hdev, bytes, tx->desc, tx->bus);
+ tx->desc = NULL;
+abort_with_info:
+ vfree(tx->info);
+ tx->info = NULL;
+ return -ENOMEM;
+}
+
+int gve_tx_alloc_rings(struct gve_priv *priv)
+{
+ int err = 0;
+ int i;
+
+ for (i = 0; i < priv->tx_cfg.num_queues; i++) {
+ err = gve_tx_alloc_ring(priv, i);
+ if (err) {
+ netif_err(priv, drv, priv->dev,
+ "Failed to alloc tx ring=%d: err=%d\n",
+ i, err);
+ break;
+ }
+ }
+ /* Unallocate if there was an error */
+ if (err) {
+ int j;
+
+ for (j = 0; j < i; j++)
+ gve_tx_free_ring(priv, j);
+ }
+ return err;
+}
+
+void gve_tx_free_rings(struct gve_priv *priv)
+{
+ int i;
+
+ for (i = 0; i < priv->tx_cfg.num_queues; i++)
+ gve_tx_free_ring(priv, i);
+}
+
+/* gve_tx_avail - Calculates the number of slots available in the ring
+ * @tx: tx ring to check
+ *
+ * Returns the number of slots available
+ *
+ * The capacity of the queue is mask + 1. We don't need to reserve an entry.
+ **/
+static inline u32 gve_tx_avail(struct gve_tx_ring *tx)
+{
+ return tx->mask + 1 - (tx->req - tx->done);
+}
+
+static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,
+ struct sk_buff *skb)
+{
+ int pad_bytes, align_hdr_pad;
+ int bytes;
+ int hlen;
+
+ hlen = skb_is_gso(skb) ? skb_checksum_start_offset(skb) +
+ tcp_hdrlen(skb) : skb_headlen(skb);
+
+ pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo,
+ hlen);
+ /* We need to take into account the header alignment padding. */
+ align_hdr_pad = L1_CACHE_ALIGN(hlen) - hlen;
+ bytes = align_hdr_pad + pad_bytes + skb->len;
+
+ return bytes;
+}
+
+/* The most descriptors we could need are 3 - 1 for the headers, 1 for
+ * the beginning of the payload at the end of the FIFO, and 1 if the
+ * payload wraps to the beginning of the FIFO.
+ */
+#define MAX_TX_DESC_NEEDED 3
+
+/* Check if sufficient resources (descriptor ring space, FIFO space) are
+ * available to transmit the given number of bytes.
+ */
+static inline bool gve_can_tx(struct gve_tx_ring *tx, int bytes_required)
+{
+ return (gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED &&
+ gve_tx_fifo_can_alloc(&tx->tx_fifo, bytes_required));
+}
+
+/* Stops the queue if the skb cannot be transmitted. */
+static int gve_maybe_stop_tx(struct gve_tx_ring *tx, struct sk_buff *skb)
+{
+ int bytes_required;
+
+ bytes_required = gve_skb_fifo_bytes_required(tx, skb);
+ if (likely(gve_can_tx(tx, bytes_required)))
+ return 0;
+
+ /* No space, so stop the queue */
+ tx->stop_queue++;
+ netif_tx_stop_queue(tx->netdev_txq);
+ smp_mb(); /* sync with restarting queue in gve_clean_tx_done() */
+
+ /* Now check for resources again, in case gve_clean_tx_done() freed
+ * resources after we checked and we stopped the queue after
+ * gve_clean_tx_done() checked.
+ *
+ * gve_maybe_stop_tx() gve_clean_tx_done()
+ * nsegs/can_alloc test failed
+ * gve_tx_free_fifo()
+ * if (tx queue stopped)
+ * netif_tx_queue_wake()
+ * netif_tx_stop_queue()
+ * Need to check again for space here!
+ */
+ if (likely(!gve_can_tx(tx, bytes_required)))
+ return -EBUSY;
+
+ netif_tx_start_queue(tx->netdev_txq);
+ tx->wake_queue++;
+ return 0;
+}
+
+static void gve_tx_fill_pkt_desc(union gve_tx_desc *pkt_desc,
+ struct sk_buff *skb, bool is_gso,
+ int l4_hdr_offset, u32 desc_cnt,
+ u16 hlen, u64 addr)
+{
+ /* l4_hdr_offset and csum_offset are in units of 16-bit words */
+ if (is_gso) {
+ pkt_desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;
+ pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
+ pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
+ } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+ pkt_desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;
+ pkt_desc->pkt.l4_csum_offset = skb->csum_offset >> 1;
+ pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
+ } else {
+ pkt_desc->pkt.type_flags = GVE_TXD_STD;
+ pkt_desc->pkt.l4_csum_offset = 0;
+ pkt_desc->pkt.l4_hdr_offset = 0;
+ }
+ pkt_desc->pkt.desc_cnt = desc_cnt;
+ pkt_desc->pkt.len = cpu_to_be16(skb->len);
+ pkt_desc->pkt.seg_len = cpu_to_be16(hlen);
+ pkt_desc->pkt.seg_addr = cpu_to_be64(addr);
+}
+
+static void gve_tx_fill_seg_desc(union gve_tx_desc *seg_desc,
+ struct sk_buff *skb, bool is_gso,
+ u16 len, u64 addr)
+{
+ seg_desc->seg.type_flags = GVE_TXD_SEG;
+ if (is_gso) {
+ if (skb_is_gso_v6(skb))
+ seg_desc->seg.type_flags |= GVE_TXSF_IPV6;
+ seg_desc->seg.l3_offset = skb_network_offset(skb) >> 1;
+ seg_desc->seg.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
+ }
+ seg_desc->seg.seg_len = cpu_to_be16(len);
+ seg_desc->seg.seg_addr = cpu_to_be64(addr);
+}
+
+static int gve_tx_add_skb(struct gve_tx_ring *tx, struct sk_buff *skb)
+{
+ int pad_bytes, hlen, hdr_nfrags, payload_nfrags, l4_hdr_offset;
+ union gve_tx_desc *pkt_desc, *seg_desc;
+ struct gve_tx_buffer_state *info;
+ bool is_gso = skb_is_gso(skb);
+ u32 idx = tx->req & tx->mask;
+ int payload_iov = 2;
+ int copy_offset;
+ u32 next_idx;
+ int i;
+
+ info = &tx->info[idx];
+ pkt_desc = &tx->desc[idx];
+
+ l4_hdr_offset = skb_checksum_start_offset(skb);
+ /* If the skb is gso, then we want the tcp header in the first segment
+ * otherwise we want the linear portion of the skb (which will contain
+ * the checksum because skb->csum_start and skb->csum_offset are given
+ * relative to skb->head) in the first segment.
+ */
+ hlen = is_gso ? l4_hdr_offset + tcp_hdrlen(skb) :
+ skb_headlen(skb);
+
+ info->skb = skb;
+ /* We don't want to split the header, so if necessary, pad to the end
+ * of the fifo and then put the header at the beginning of the fifo.
+ */
+ pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, hlen);
+ hdr_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, hlen + pad_bytes,
+ &info->iov[0]);
+ WARN(!hdr_nfrags, "hdr_nfrags should never be 0!");
+ payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen,
+ &info->iov[payload_iov]);
+
+ gve_tx_fill_pkt_desc(pkt_desc, skb, is_gso, l4_hdr_offset,
+ 1 + payload_nfrags, hlen,
+ info->iov[hdr_nfrags - 1].iov_offset);
+
+ skb_copy_bits(skb, 0,
+ tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset,
+ hlen);
+ copy_offset = hlen;
+
+ for (i = payload_iov; i < payload_nfrags + payload_iov; i++) {
+ next_idx = (tx->req + 1 + i - payload_iov) & tx->mask;
+ seg_desc = &tx->desc[next_idx];
+
+ gve_tx_fill_seg_desc(seg_desc, skb, is_gso,
+ info->iov[i].iov_len,
+ info->iov[i].iov_offset);
+
+ skb_copy_bits(skb, copy_offset,
+ tx->tx_fifo.base + info->iov[i].iov_offset,
+ info->iov[i].iov_len);
+ copy_offset += info->iov[i].iov_len;
+ }
+
+ return 1 + payload_nfrags;
+}
+
+netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev)
+{
+ struct gve_priv *priv = netdev_priv(dev);
+ struct gve_tx_ring *tx;
+ int nsegs;
+
+ WARN(skb_get_queue_mapping(skb) > priv->tx_cfg.num_queues,
+ "skb queue index out of range");
+ tx = &priv->tx[skb_get_queue_mapping(skb)];
+ if (unlikely(gve_maybe_stop_tx(tx, skb))) {
+ /* We need to ring the txq doorbell -- we have stopped the Tx
+ * queue for want of resources, but prior calls to gve_tx()
+ * may have added descriptors without ringing the doorbell.
+ */
+
+ /* Ensure tx descs from a prior gve_tx are visible before
+ * ringing doorbell.
+ */
+ dma_wmb();
+ gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
+ return NETDEV_TX_BUSY;
+ }
+ nsegs = gve_tx_add_skb(tx, skb);
+
+ netdev_tx_sent_queue(tx->netdev_txq, skb->len);
+ skb_tx_timestamp(skb);
+
+ /* give packets to NIC */
+ tx->req += nsegs;
+
+ if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())
+ return NETDEV_TX_OK;
+
+ /* Ensure tx descs are visible before ringing doorbell */
+ dma_wmb();
+ gve_tx_put_doorbell(priv, tx->q_resources, tx->req);
+ return NETDEV_TX_OK;
+}
+
+#define GVE_TX_START_THRESH PAGE_SIZE
+
+static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
+ u32 to_do, bool try_to_wake)
+{
+ struct gve_tx_buffer_state *info;
+ u64 pkts = 0, bytes = 0;
+ size_t space_freed = 0;
+ struct sk_buff *skb;
+ int i, j;
+ u32 idx;
+
+ for (j = 0; j < to_do; j++) {
+ idx = tx->done & tx->mask;
+ netif_info(priv, tx_done, priv->dev,
+ "[%d] %s: idx=%d (req=%u done=%u)\n",
+ tx->q_num, __func__, idx, tx->req, tx->done);
+ info = &tx->info[idx];
+ skb = info->skb;
+
+ /* Mark as free */
+ if (skb) {
+ info->skb = NULL;
+ bytes += skb->len;
+ pkts++;
+ dev_consume_skb_any(skb);
+ /* FIFO free */
+ for (i = 0; i < ARRAY_SIZE(info->iov); i++) {
+ space_freed += info->iov[i].iov_len +
+ info->iov[i].iov_padding;
+ info->iov[i].iov_len = 0;
+ info->iov[i].iov_padding = 0;
+ }
+ }
+ tx->done++;
+ }
+
+ gve_tx_free_fifo(&tx->tx_fifo, space_freed);
+ u64_stats_update_begin(&tx->statss);
+ tx->bytes_done += bytes;
+ tx->pkt_done += pkts;
+ u64_stats_update_end(&tx->statss);
+ netdev_tx_completed_queue(tx->netdev_txq, pkts, bytes);
+
+ /* start the queue if we've stopped it */
+#ifndef CONFIG_BQL
+ /* Make sure that the doorbells are synced */
+ smp_mb();
+#endif
+ if (try_to_wake && netif_tx_queue_stopped(tx->netdev_txq) &&
+ likely(gve_can_tx(tx, GVE_TX_START_THRESH))) {
+ tx->wake_queue++;
+ netif_tx_wake_queue(tx->netdev_txq);
+ }
+
+ return pkts;
+}
+
+__be32 gve_tx_load_event_counter(struct gve_priv *priv,
+ struct gve_tx_ring *tx)
+{
+ u32 counter_index = be32_to_cpu((tx->q_resources->counter_index));
+
+ return READ_ONCE(priv->counter_array[counter_index]);
+}
+
+bool gve_tx_poll(struct gve_notify_block *block, int budget)
+{
+ struct gve_priv *priv = block->priv;
+ struct gve_tx_ring *tx = block->tx;
+ bool repoll = false;
+ u32 nic_done;
+ u32 to_do;
+
+ /* If budget is 0, do all the work */
+ if (budget == 0)
+ budget = INT_MAX;
+
+ /* Find out how much work there is to be done */
+ tx->last_nic_done = gve_tx_load_event_counter(priv, tx);
+ nic_done = be32_to_cpu(tx->last_nic_done);
+ if (budget > 0) {
+ /* Do as much work as we have that the budget will
+ * allow
+ */
+ to_do = min_t(u32, (nic_done - tx->done), budget);
+ gve_clean_tx_done(priv, tx, to_do, true);
+ }
+ /* If we still have work we want to repoll */
+ repoll |= (nic_done != tx->done);
+ return repoll;
+}
--
2.22.0.410.gd8fdbe21b5-goog
^ permalink raw reply related
* [PATCH net-next v4 1/4] gve: Add basic driver framework for Compute Engine Virtual NIC
From: Catherine Sullivan @ 2019-07-01 22:57 UTC (permalink / raw)
To: netdev
Cc: Catherine Sullivan, Sagi Shahar, Jon Olson, Willem de Bruijn,
Luigi Rizzo
In-Reply-To: <20190701225755.209250-1-csully@google.com>
Add a driver framework for the Compute Engine Virtual NIC that will be
available in the future.
At this point the only functionality is loading the driver.
Signed-off-by: Catherine Sullivan <csully@google.com>
Signed-off-by: Sagi Shahar <sagis@google.com>
Signed-off-by: Jon Olson <jonolson@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Luigi Rizzo <lrizzo@google.com>
---
.../networking/device_drivers/google/gve.rst | 82 ++++
.../networking/device_drivers/index.rst | 1 +
MAINTAINERS | 9 +
drivers/net/ethernet/Kconfig | 1 +
drivers/net/ethernet/Makefile | 1 +
drivers/net/ethernet/google/Kconfig | 27 ++
drivers/net/ethernet/google/Makefile | 5 +
drivers/net/ethernet/google/gve/Makefile | 4 +
drivers/net/ethernet/google/gve/gve.h | 135 ++++++
drivers/net/ethernet/google/gve/gve_adminq.c | 249 ++++++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 134 ++++++
drivers/net/ethernet/google/gve/gve_main.c | 446 ++++++++++++++++++
.../net/ethernet/google/gve/gve_register.h | 27 ++
13 files changed, 1121 insertions(+)
create mode 100644 Documentation/networking/device_drivers/google/gve.rst
create mode 100644 drivers/net/ethernet/google/Kconfig
create mode 100644 drivers/net/ethernet/google/Makefile
create mode 100644 drivers/net/ethernet/google/gve/Makefile
create mode 100644 drivers/net/ethernet/google/gve/gve.h
create mode 100644 drivers/net/ethernet/google/gve/gve_adminq.c
create mode 100644 drivers/net/ethernet/google/gve/gve_adminq.h
create mode 100644 drivers/net/ethernet/google/gve/gve_main.c
create mode 100644 drivers/net/ethernet/google/gve/gve_register.h
diff --git a/Documentation/networking/device_drivers/google/gve.rst b/Documentation/networking/device_drivers/google/gve.rst
new file mode 100644
index 000000000000..7397c82f4c8f
--- /dev/null
+++ b/Documentation/networking/device_drivers/google/gve.rst
@@ -0,0 +1,82 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+==============================================================
+Linux kernel driver for Compute Engine Virtual Ethernet (gve):
+==============================================================
+
+Supported Hardware
+===================
+The GVE driver binds to a single PCI device id used by the virtual
+Ethernet device found in some Compute Engine VMs.
+
++--------------+----------+---------+
+|Field | Value | Comments|
++==============+==========+=========+
+|Vendor ID | `0x1AE0` | Google |
++--------------+----------+---------+
+|Device ID | `0x0042` | |
++--------------+----------+---------+
+|Sub-vendor ID | `0x1AE0` | Google |
++--------------+----------+---------+
+|Sub-device ID | `0x0058` | |
++--------------+----------+---------+
+|Revision ID | `0x0` | |
++--------------+----------+---------+
+|Device Class | `0x200` | Ethernet|
++--------------+----------+---------+
+
+PCI Bars
+========
+The gVNIC PCI device exposes three 32-bit memory BARS:
+- Bar0 - Device configuration and status registers.
+- Bar1 - MSI-X vector table
+- Bar2 - IRQ, RX and TX doorbells
+
+Device Interactions
+===================
+The driver interacts with the device in the following ways:
+ - Registers
+ - A block of MMIO registers
+ - See gve_register.h for more detail
+ - Admin Queue
+ - See description below
+ - Interrupts
+ - See supported interrupts below
+
+Registers
+---------
+All registers are MMIO and big endian.
+
+The registers are used for initializing and configuring the device as well as
+querying device status in response to management interrupts.
+
+Admin Queue (AQ)
+----------------
+The Admin Queue is a PAGE_SIZE memory block, treated as an array of AQ
+commands, used by the driver to issue commands to the device and set up
+resources.The driver and the device maintain a count of how many commands
+have been submitted and executed. To issue AQ commands, the driver must do
+the following (with proper locking):
+
+1) Copy new commands into next available slots in the AQ array
+2) Increment its counter by he number of new commands
+3) Write the counter into the GVE_ADMIN_QUEUE_DOORBELL register
+4) Poll the ADMIN_QUEUE_EVENT_COUNTER register until it equals
+ the value written to the doorbell, or until a timeout.
+
+The device will update the status field in each AQ command reported as
+executed through the ADMIN_QUEUE_EVENT_COUNTER register.
+
+Interrupts
+----------
+The following interrupts are supported by the driver:
+
+Management Interrupt
+~~~~~~~~~~~~~~~~~~~~
+The management interrupt is used by the device to tell the driver to
+look at the GVE_DEVICE_STATUS register.
+
+Notification Block Interrupts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The notification block interrupts are used to tell the driver to poll
+the queues associated with that interrupt.
diff --git a/Documentation/networking/device_drivers/index.rst b/Documentation/networking/device_drivers/index.rst
index 24598d5f8ffa..2b7fefe72351 100644
--- a/Documentation/networking/device_drivers/index.rst
+++ b/Documentation/networking/device_drivers/index.rst
@@ -21,6 +21,7 @@ Contents:
intel/i40e
intel/iavf
intel/ice
+ google/gve
mellanox/mlx5
.. only:: subproject
diff --git a/MAINTAINERS b/MAINTAINERS
index b4304d10f14e..449e7cdb3303 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6720,6 +6720,15 @@ L: linux-input@vger.kernel.org
S: Maintained
F: drivers/input/touchscreen/goodix.c
+GOOGLE ETHERNET DRIVERS
+M: Catherine Sullivan <csully@google.com>
+R: Sagi Shahar <sagis@google.com>
+R: Jon Olson <jonolson@google.com>
+L: netdev@vger.kernel.org
+S: Supported
+F: Documentation/networking/device_drivers/google/gve.txt
+F: drivers/net/ethernet/google
+
GPD POCKET FAN DRIVER
M: Hans de Goede <hdegoede@redhat.com>
L: platform-driver-x86@vger.kernel.org
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index fe115b7caba0..93a2d4deb27c 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -76,6 +76,7 @@ source "drivers/net/ethernet/ezchip/Kconfig"
source "drivers/net/ethernet/faraday/Kconfig"
source "drivers/net/ethernet/freescale/Kconfig"
source "drivers/net/ethernet/fujitsu/Kconfig"
+source "drivers/net/ethernet/google/Kconfig"
source "drivers/net/ethernet/hisilicon/Kconfig"
source "drivers/net/ethernet/hp/Kconfig"
source "drivers/net/ethernet/huawei/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index 7b5bf9682066..fb9155cffcff 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -39,6 +39,7 @@ obj-$(CONFIG_NET_VENDOR_EZCHIP) += ezchip/
obj-$(CONFIG_NET_VENDOR_FARADAY) += faraday/
obj-$(CONFIG_NET_VENDOR_FREESCALE) += freescale/
obj-$(CONFIG_NET_VENDOR_FUJITSU) += fujitsu/
+obj-$(CONFIG_NET_VENDOR_GOOGLE) += google/
obj-$(CONFIG_NET_VENDOR_HISILICON) += hisilicon/
obj-$(CONFIG_NET_VENDOR_HP) += hp/
obj-$(CONFIG_NET_VENDOR_HUAWEI) += huawei/
diff --git a/drivers/net/ethernet/google/Kconfig b/drivers/net/ethernet/google/Kconfig
new file mode 100644
index 000000000000..b8f04d052fda
--- /dev/null
+++ b/drivers/net/ethernet/google/Kconfig
@@ -0,0 +1,27 @@
+#
+# Google network device configuration
+#
+
+config NET_VENDOR_GOOGLE
+ bool "Google Devices"
+ default y
+ help
+ If you have a network (Ethernet) device belonging to this class, say Y.
+
+ Note that the answer to this question doesn't directly affect the
+ kernel: saying N will just cause the configurator to skip all
+ the questions about Google devices. If you say Y, you will be asked
+ for your specific device in the following questions.
+
+if NET_VENDOR_GOOGLE
+
+config GVE
+ tristate "Google Virtual NIC (gVNIC) support"
+ depends on PCI_MSI
+ help
+ This driver supports Google Virtual NIC (gVNIC)"
+
+ To compile this driver as a module, choose M here.
+ The module will be called gve.
+
+endif #NET_VENDOR_GOOGLE
diff --git a/drivers/net/ethernet/google/Makefile b/drivers/net/ethernet/google/Makefile
new file mode 100644
index 000000000000..402cc3ba1639
--- /dev/null
+++ b/drivers/net/ethernet/google/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the Google network device drivers.
+#
+
+obj-$(CONFIG_GVE) += gve/
diff --git a/drivers/net/ethernet/google/gve/Makefile b/drivers/net/ethernet/google/gve/Makefile
new file mode 100644
index 000000000000..cec03ee6d931
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/Makefile
@@ -0,0 +1,4 @@
+# Makefile for the Google virtual Ethernet (gve) driver
+
+obj-$(CONFIG_GVE) += gve.o
+gve-objs := gve_main.o gve_adminq.o
diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
new file mode 100644
index 000000000000..47fb86e5aeff
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT)
+ * Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#ifndef _GVE_H_
+#define _GVE_H_
+
+#include <linux/dma-mapping.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+
+#ifndef PCI_VENDOR_ID_GOOGLE
+#define PCI_VENDOR_ID_GOOGLE 0x1ae0
+#endif
+
+#define PCI_DEV_ID_GVNIC 0x0042
+
+#define GVE_REGISTER_BAR 0
+#define GVE_DOORBELL_BAR 2
+
+/* 1 for management */
+#define GVE_MIN_MSIX 3
+
+struct gve_notify_block {
+ __be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
+ char name[IFNAMSIZ + 16]; /* name registered with the kernel */
+ struct napi_struct napi; /* kernel napi struct for this block */
+ struct gve_priv *priv;
+} ____cacheline_aligned;
+
+struct gve_priv {
+ struct net_device *dev;
+ struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */
+ dma_addr_t ntfy_block_bus;
+ struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */
+ char mgmt_msix_name[IFNAMSIZ + 16];
+ u32 mgmt_msix_idx;
+ __be32 *counter_array; /* array of num_event_counters */
+ dma_addr_t counter_array_bus;
+
+ u16 num_event_counters;
+
+ u32 num_ntfy_blks; /* spilt between TX and RX so must be even */
+
+ struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
+ __be32 __iomem *db_bar2; /* "array" of doorbells */
+ u32 msg_enable; /* level for netif* netdev print macros */
+ struct pci_dev *pdev;
+
+ /* Admin queue - see gve_adminq.h*/
+ union gve_adminq_command *adminq;
+ dma_addr_t adminq_bus_addr;
+ u32 adminq_mask; /* masks prod_cnt to adminq size */
+ u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
+
+ unsigned long state_flags;
+};
+
+enum gve_state_flags {
+ GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = BIT(1),
+ GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = BIT(2),
+ GVE_PRIV_FLAGS_DEVICE_RINGS_OK = BIT(3),
+ GVE_PRIV_FLAGS_NAPI_ENABLED = BIT(4),
+};
+
+static inline bool gve_get_admin_queue_ok(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
+}
+
+static inline void gve_set_admin_queue_ok(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
+}
+
+static inline void gve_clear_admin_queue_ok(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
+}
+
+static inline bool gve_get_device_resources_ok(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
+}
+
+static inline void gve_set_device_resources_ok(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
+}
+
+static inline void gve_clear_device_resources_ok(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
+}
+
+static inline bool gve_get_device_rings_ok(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
+}
+
+static inline void gve_set_device_rings_ok(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
+}
+
+static inline void gve_clear_device_rings_ok(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
+}
+
+static inline bool gve_get_napi_enabled(struct gve_priv *priv)
+{
+ return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
+}
+
+static inline void gve_set_napi_enabled(struct gve_priv *priv)
+{
+ set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
+}
+
+static inline void gve_clear_napi_enabled(struct gve_priv *priv)
+{
+ clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
+}
+
+/* Returns the address of the ntfy_blocks irq doorbell
+ */
+static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv,
+ struct gve_notify_block *block)
+{
+ return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];
+}
+#endif /* _GVE_H_ */
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c
new file mode 100644
index 000000000000..4bf32e4e2810
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_adminq.c
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/pci.h>
+#include "gve.h"
+#include "gve_adminq.h"
+#include "gve_register.h"
+
+#define GVE_MAX_ADMINQ_RELEASE_CHECK 500
+#define GVE_ADMINQ_SLEEP_LEN 20
+#define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100
+
+int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
+{
+ priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
+ &priv->adminq_bus_addr, GFP_KERNEL);
+ if (unlikely(!priv->adminq))
+ return -ENOMEM;
+
+ priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
+ priv->adminq_prod_cnt = 0;
+
+ /* Setup Admin queue with the device */
+ iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
+ &priv->reg_bar0->adminq_pfn);
+
+ gve_set_admin_queue_ok(priv);
+ return 0;
+}
+
+void gve_adminq_release(struct gve_priv *priv)
+{
+ int i = 0;
+
+ /* Tell the device the adminq is leaving */
+ iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
+ while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
+ /* If this is reached the device is unrecoverable and still
+ * holding memory. Continue looping to avoid memory corruption,
+ * but WARN so it is visible what is going on.
+ */
+ if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
+ WARN(1, "Unrecoverable platform error!");
+ i++;
+ msleep(GVE_ADMINQ_SLEEP_LEN);
+ }
+ gve_clear_device_rings_ok(priv);
+ gve_clear_device_resources_ok(priv);
+ gve_clear_admin_queue_ok(priv);
+}
+
+void gve_adminq_free(struct device *dev, struct gve_priv *priv)
+{
+ if (!gve_get_admin_queue_ok(priv))
+ return;
+ gve_adminq_release(priv);
+ dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
+ gve_clear_admin_queue_ok(priv);
+}
+
+static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
+{
+ iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
+}
+
+static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
+{
+ int i;
+
+ for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
+ if (ioread32be(&priv->reg_bar0->adminq_event_counter)
+ == prod_cnt)
+ return true;
+ msleep(GVE_ADMINQ_SLEEP_LEN);
+ }
+
+ return false;
+}
+
+static int gve_adminq_parse_err(struct device *dev, u32 status)
+{
+ if (status != GVE_ADMINQ_COMMAND_PASSED &&
+ status != GVE_ADMINQ_COMMAND_UNSET)
+ dev_err(dev, "AQ command failed with status %d\n", status);
+
+ switch (status) {
+ case GVE_ADMINQ_COMMAND_PASSED:
+ return 0;
+ case GVE_ADMINQ_COMMAND_UNSET:
+ dev_err(dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
+ return -EINVAL;
+ case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
+ case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
+ case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
+ case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
+ case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
+ return -EAGAIN;
+ case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
+ case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
+ case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
+ case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
+ case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
+ case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
+ return -EINVAL;
+ case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
+ return -ETIME;
+ case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
+ case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
+ return -EACCES;
+ case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
+ return -ENOMEM;
+ case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
+ return -ENOTSUPP;
+ default:
+ dev_err(dev, "parse_aq_err: unknown status code %d\n", status);
+ return -EINVAL;
+ }
+}
+
+/* This function is not threadsafe - the caller is responsible for any
+ * necessary locks.
+ */
+int gve_adminq_execute_cmd(struct gve_priv *priv,
+ union gve_adminq_command *cmd_orig)
+{
+ union gve_adminq_command *cmd;
+ u32 status = 0;
+ u32 prod_cnt;
+
+ cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
+ priv->adminq_prod_cnt++;
+ prod_cnt = priv->adminq_prod_cnt;
+
+ memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
+
+ gve_adminq_kick_cmd(priv, prod_cnt);
+ if (!gve_adminq_wait_for_cmd(priv, prod_cnt)) {
+ dev_err(&priv->pdev->dev, "AQ command timed out, need to reset AQ\n");
+ return -ENOTRECOVERABLE;
+ }
+
+ memcpy(cmd_orig, cmd, sizeof(*cmd));
+ status = be32_to_cpu(READ_ONCE(cmd->status));
+ return gve_adminq_parse_err(&priv->pdev->dev, status);
+}
+
+/* The device specifies that the management vector can either be the first irq
+ * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
+ * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
+ * the management vector is first.
+ *
+ * gve arranges the msix vectors so that the management vector is last.
+ */
+#define GVE_NTFY_BLK_BASE_MSIX_IDX 0
+int gve_adminq_configure_device_resources(struct gve_priv *priv,
+ dma_addr_t counter_array_bus_addr,
+ u32 num_counters,
+ dma_addr_t db_array_bus_addr,
+ u32 num_ntfy_blks)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
+ cmd.configure_device_resources =
+ (struct gve_adminq_configure_device_resources) {
+ .counter_array = cpu_to_be64(counter_array_bus_addr),
+ .num_counters = cpu_to_be32(num_counters),
+ .irq_db_addr = cpu_to_be64(db_array_bus_addr),
+ .num_irq_dbs = cpu_to_be32(num_ntfy_blks),
+ .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])),
+ .ntfy_blk_msix_base_idx =
+ cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
+int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
+
+int gve_adminq_describe_device(struct gve_priv *priv)
+{
+ struct gve_device_descriptor *descriptor;
+ union gve_adminq_command cmd;
+ dma_addr_t descriptor_bus;
+ int err = 0;
+ u8 *mac;
+ u16 mtu;
+
+ memset(&cmd, 0, sizeof(cmd));
+ descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
+ &descriptor_bus, GFP_KERNEL);
+ if (!descriptor)
+ return -ENOMEM;
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
+ cmd.describe_device.device_descriptor_addr =
+ cpu_to_be64(descriptor_bus);
+ cmd.describe_device.device_descriptor_version =
+ cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
+ cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
+
+ err = gve_adminq_execute_cmd(priv, &cmd);
+ if (err)
+ goto free_device_descriptor;
+
+ mtu = be16_to_cpu(descriptor->mtu);
+ if (mtu < ETH_MIN_MTU) {
+ netif_err(priv, drv, priv->dev, "MTU %d below minimum MTU\n",
+ mtu);
+ err = -EINVAL;
+ goto free_device_descriptor;
+ }
+ priv->dev->max_mtu = mtu;
+ priv->num_event_counters = be16_to_cpu(descriptor->counters);
+ ether_addr_copy(priv->dev->dev_addr, descriptor->mac);
+ mac = descriptor->mac;
+ netif_info(priv, drv, priv->dev, "MAC addr: %pM\n", mac);
+
+free_device_descriptor:
+ dma_free_coherent(&priv->pdev->dev, sizeof(*descriptor), descriptor,
+ descriptor_bus);
+ return err;
+}
+
+int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
+{
+ union gve_adminq_command cmd;
+
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
+ cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
+ .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
+ .parameter_value = cpu_to_be64(mtu),
+ };
+
+ return gve_adminq_execute_cmd(priv, &cmd);
+}
diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h
new file mode 100644
index 000000000000..b860b0d69393
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_adminq.h
@@ -0,0 +1,134 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT)
+ * Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#ifndef _GVE_ADMINQ_H
+#define _GVE_ADMINQ_H
+
+#include <linux/build_bug.h>
+
+/* Admin queue opcodes */
+enum gve_adminq_opcodes {
+ GVE_ADMINQ_DESCRIBE_DEVICE = 0x1,
+ GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES = 0x2,
+ GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES = 0x9,
+ GVE_ADMINQ_SET_DRIVER_PARAMETER = 0xB,
+};
+
+/* Admin queue status codes */
+enum gve_adminq_statuses {
+ GVE_ADMINQ_COMMAND_UNSET = 0x0,
+ GVE_ADMINQ_COMMAND_PASSED = 0x1,
+ GVE_ADMINQ_COMMAND_ERROR_ABORTED = 0xFFFFFFF0,
+ GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS = 0xFFFFFFF1,
+ GVE_ADMINQ_COMMAND_ERROR_CANCELLED = 0xFFFFFFF2,
+ GVE_ADMINQ_COMMAND_ERROR_DATALOSS = 0xFFFFFFF3,
+ GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED = 0xFFFFFFF4,
+ GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION = 0xFFFFFFF5,
+ GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR = 0xFFFFFFF6,
+ GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT = 0xFFFFFFF7,
+ GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND = 0xFFFFFFF8,
+ GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE = 0xFFFFFFF9,
+ GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED = 0xFFFFFFFA,
+ GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED = 0xFFFFFFFB,
+ GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED = 0xFFFFFFFC,
+ GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE = 0xFFFFFFFD,
+ GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED = 0xFFFFFFFE,
+ GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR = 0xFFFFFFFF,
+};
+
+#define GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION 1
+
+/* All AdminQ command structs should be naturally packed. The static_assert
+ * calls make sure this is the case at compile time.
+ */
+
+struct gve_adminq_describe_device {
+ __be64 device_descriptor_addr;
+ __be32 device_descriptor_version;
+ __be32 available_length;
+};
+
+static_assert(sizeof(struct gve_adminq_describe_device) == 16);
+
+struct gve_device_descriptor {
+ __be64 max_registered_pages;
+ __be16 reserved1;
+ __be16 tx_queue_entries;
+ __be16 rx_queue_entries;
+ __be16 default_num_queues;
+ __be16 mtu;
+ __be16 counters;
+ __be16 tx_pages_per_qpl;
+ __be16 rx_pages_per_qpl;
+ u8 mac[ETH_ALEN];
+ __be16 num_device_options;
+ __be16 total_length;
+ u8 reserved2[6];
+};
+
+static_assert(sizeof(struct gve_device_descriptor) == 40);
+
+struct device_option {
+ __be32 option_id;
+ __be32 option_length;
+};
+
+static_assert(sizeof(struct device_option) == 8);
+
+struct gve_adminq_configure_device_resources {
+ __be64 counter_array;
+ __be64 irq_db_addr;
+ __be32 num_counters;
+ __be32 num_irq_dbs;
+ __be32 irq_db_stride;
+ __be32 ntfy_blk_msix_base_idx;
+};
+
+static_assert(sizeof(struct gve_adminq_configure_device_resources) == 32);
+
+/* GVE Set Driver Parameter Types */
+enum gve_set_driver_param_types {
+ GVE_SET_PARAM_MTU = 0x1,
+};
+
+struct gve_adminq_set_driver_parameter {
+ __be32 parameter_type;
+ u8 reserved[4];
+ __be64 parameter_value;
+};
+
+static_assert(sizeof(struct gve_adminq_set_driver_parameter) == 16);
+
+union gve_adminq_command {
+ struct {
+ __be32 opcode;
+ __be32 status;
+ union {
+ struct gve_adminq_configure_device_resources
+ configure_device_resources;
+ struct gve_adminq_describe_device describe_device;
+ struct gve_adminq_set_driver_parameter set_driver_param;
+ };
+ };
+ u8 reserved[64];
+};
+
+static_assert(sizeof(union gve_adminq_command) == 64);
+
+int gve_adminq_alloc(struct device *dev, struct gve_priv *priv);
+void gve_adminq_free(struct device *dev, struct gve_priv *priv);
+void gve_adminq_release(struct gve_priv *priv);
+int gve_adminq_execute_cmd(struct gve_priv *priv,
+ union gve_adminq_command *cmd_orig);
+int gve_adminq_describe_device(struct gve_priv *priv);
+int gve_adminq_configure_device_resources(struct gve_priv *priv,
+ dma_addr_t counter_array_bus_addr,
+ u32 num_counters,
+ dma_addr_t db_array_bus_addr,
+ u32 num_ntfy_blks);
+int gve_adminq_deconfigure_device_resources(struct gve_priv *priv);
+int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu);
+#endif /* _GVE_ADMINQ_H */
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
new file mode 100644
index 000000000000..41d8cfa8f998
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -0,0 +1,446 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#include <linux/cpumask.h>
+#include <linux/etherdevice.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/sched.h>
+#include <linux/timer.h>
+#include <net/sch_generic.h>
+#include "gve.h"
+#include "gve_adminq.h"
+#include "gve_register.h"
+
+#define DEFAULT_MSG_LEVEL (NETIF_MSG_DRV | NETIF_MSG_LINK)
+#define GVE_VERSION "1.0.0"
+#define GVE_VERSION_PREFIX "GVE-"
+
+static const char gve_version_str[] = GVE_VERSION;
+static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
+
+static int gve_alloc_counter_array(struct gve_priv *priv)
+{
+ priv->counter_array =
+ dma_alloc_coherent(&priv->pdev->dev,
+ priv->num_event_counters *
+ sizeof(*priv->counter_array),
+ &priv->counter_array_bus, GFP_KERNEL);
+ if (!priv->counter_array)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void gve_free_counter_array(struct gve_priv *priv)
+{
+ dma_free_coherent(&priv->pdev->dev,
+ priv->num_event_counters *
+ sizeof(*priv->counter_array),
+ priv->counter_array, priv->counter_array_bus);
+ priv->counter_array = NULL;
+}
+
+static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
+{
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t gve_intr(int irq, void *arg)
+{
+ return IRQ_HANDLED;
+}
+
+static int gve_alloc_notify_blocks(struct gve_priv *priv)
+{
+ int num_vecs_requested = priv->num_ntfy_blks + 1;
+ char *name = priv->dev->name;
+ unsigned int active_cpus;
+ int vecs_enabled;
+ int i, j;
+ int err;
+
+ priv->msix_vectors = kvzalloc(num_vecs_requested *
+ sizeof(*priv->msix_vectors), GFP_KERNEL);
+ if (!priv->msix_vectors)
+ return -ENOMEM;
+ for (i = 0; i < num_vecs_requested; i++)
+ priv->msix_vectors[i].entry = i;
+ vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors,
+ GVE_MIN_MSIX, num_vecs_requested);
+ if (vecs_enabled < 0) {
+ dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
+ GVE_MIN_MSIX, vecs_enabled);
+ err = vecs_enabled;
+ goto abort_with_msix_vectors;
+ }
+ if (vecs_enabled != num_vecs_requested) {
+ priv->num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
+ dev_err(&priv->pdev->dev,
+ "Only received %d msix. Lowering number of notification blocks to %d\n",
+ vecs_enabled, priv->num_ntfy_blks);
+ }
+ /* Half the notification blocks go to TX and half to RX */
+ active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus());
+
+ /* Setup Management Vector - the last vector */
+ snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt",
+ name);
+ err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
+ gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
+ if (err) {
+ dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
+ goto abort_with_msix_enabled;
+ }
+ priv->ntfy_blocks =
+ dma_alloc_coherent(&priv->pdev->dev,
+ priv->num_ntfy_blks *
+ sizeof(*priv->ntfy_blocks),
+ &priv->ntfy_block_bus, GFP_KERNEL);
+ if (!priv->ntfy_blocks) {
+ err = -ENOMEM;
+ goto abort_with_mgmt_vector;
+ }
+ /* Setup the other blocks - the first n-1 vectors */
+ for (i = 0; i < priv->num_ntfy_blks; i++) {
+ struct gve_notify_block *block = &priv->ntfy_blocks[i];
+ int msix_idx = i;
+
+ snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d",
+ name, i);
+ block->priv = priv;
+ err = request_irq(priv->msix_vectors[msix_idx].vector,
+ gve_intr, 0, block->name, block);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to receive msix vector %d\n", i);
+ goto abort_with_some_ntfy_blocks;
+ }
+ irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
+ get_cpu_mask(i % active_cpus));
+ }
+ return 0;
+abort_with_some_ntfy_blocks:
+ for (j = 0; j < i; j++) {
+ struct gve_notify_block *block = &priv->ntfy_blocks[j];
+ int msix_idx = j;
+
+ irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
+ NULL);
+ free_irq(priv->msix_vectors[msix_idx].vector, block);
+ }
+ dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
+ sizeof(*priv->ntfy_blocks),
+ priv->ntfy_blocks, priv->ntfy_block_bus);
+ priv->ntfy_blocks = NULL;
+abort_with_mgmt_vector:
+ free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
+abort_with_msix_enabled:
+ pci_disable_msix(priv->pdev);
+abort_with_msix_vectors:
+ kfree(priv->msix_vectors);
+ priv->msix_vectors = NULL;
+ return err;
+}
+
+static void gve_free_notify_blocks(struct gve_priv *priv)
+{
+ int i;
+
+ /* Free the irqs */
+ for (i = 0; i < priv->num_ntfy_blks; i++) {
+ struct gve_notify_block *block = &priv->ntfy_blocks[i];
+ int msix_idx = i;
+
+ irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
+ NULL);
+ free_irq(priv->msix_vectors[msix_idx].vector, block);
+ }
+ dma_free_coherent(&priv->pdev->dev,
+ priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks),
+ priv->ntfy_blocks, priv->ntfy_block_bus);
+ priv->ntfy_blocks = NULL;
+ free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
+ pci_disable_msix(priv->pdev);
+ kfree(priv->msix_vectors);
+ priv->msix_vectors = NULL;
+}
+
+static int gve_setup_device_resources(struct gve_priv *priv)
+{
+ int err;
+
+ err = gve_alloc_counter_array(priv);
+ if (err)
+ return err;
+ err = gve_alloc_notify_blocks(priv);
+ if (err)
+ goto abort_with_counter;
+ err = gve_adminq_configure_device_resources(priv,
+ priv->counter_array_bus,
+ priv->num_event_counters,
+ priv->ntfy_block_bus,
+ priv->num_ntfy_blks);
+ if (unlikely(err)) {
+ dev_err(&priv->pdev->dev,
+ "could not setup device_resources: err=%d\n", err);
+ err = -ENXIO;
+ goto abort_with_ntfy_blocks;
+ }
+ gve_set_device_resources_ok(priv);
+ return 0;
+abort_with_ntfy_blocks:
+ gve_free_notify_blocks(priv);
+abort_with_counter:
+ gve_free_counter_array(priv);
+ return err;
+}
+
+static void gve_teardown_device_resources(struct gve_priv *priv)
+{
+ int err;
+
+ /* Tell device its resources are being freed */
+ if (gve_get_device_resources_ok(priv)) {
+ err = gve_adminq_deconfigure_device_resources(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not deconfigure device resources: err=%d\n",
+ err);
+ return;
+ }
+ }
+ gve_free_counter_array(priv);
+ gve_free_notify_blocks(priv);
+ gve_clear_device_resources_ok(priv);
+}
+
+static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
+{
+ int num_ntfy;
+ int err;
+
+ /* Set up the adminq */
+ err = gve_adminq_alloc(&priv->pdev->dev, priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Failed to alloc admin queue: err=%d\n", err);
+ return err;
+ }
+
+ if (skip_describe_device)
+ goto setup_device;
+
+ /* Get the initial information we need from the device */
+ err = gve_adminq_describe_device(priv);
+ if (err) {
+ dev_err(&priv->pdev->dev,
+ "Could not get device information: err=%d\n", err);
+ goto err;
+ }
+ if (priv->dev->max_mtu > PAGE_SIZE) {
+ priv->dev->max_mtu = PAGE_SIZE;
+ err = gve_adminq_set_mtu(priv, priv->dev->mtu);
+ if (err) {
+ netif_err(priv, drv, priv->dev, "Could not set mtu");
+ goto err;
+ }
+ }
+ priv->dev->mtu = priv->dev->max_mtu;
+ num_ntfy = pci_msix_vec_count(priv->pdev);
+ if (num_ntfy <= 0) {
+ dev_err(&priv->pdev->dev,
+ "could not count MSI-x vectors: err=%d\n", num_ntfy);
+ err = num_ntfy;
+ goto err;
+ } else if (num_ntfy < GVE_MIN_MSIX) {
+ dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
+ GVE_MIN_MSIX, num_ntfy);
+ err = -EINVAL;
+ goto err;
+ }
+
+ /* gvnic has one Notification Block per MSI-x vector, except for the
+ * management vector
+ */
+ priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
+ priv->mgmt_msix_idx = priv->num_ntfy_blks;
+
+setup_device:
+ err = gve_setup_device_resources(priv);
+ if (!err)
+ return 0;
+err:
+ gve_adminq_free(&priv->pdev->dev, priv);
+ return err;
+}
+
+static void gve_teardown_priv_resources(struct gve_priv *priv)
+{
+ gve_teardown_device_resources(priv);
+ gve_adminq_free(&priv->pdev->dev, priv);
+}
+
+static void gve_write_version(u8 __iomem *driver_version_register)
+{
+ const char *c = gve_version_prefix;
+
+ while (*c) {
+ writeb(*c, driver_version_register);
+ c++;
+ }
+
+ c = gve_version_str;
+ while (*c) {
+ writeb(*c, driver_version_register);
+ c++;
+ }
+ writeb('\n', driver_version_register);
+}
+
+static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ int max_tx_queues, max_rx_queues;
+ struct net_device *dev;
+ __be32 __iomem *db_bar;
+ struct gve_registers __iomem *reg_bar;
+ struct gve_priv *priv;
+ int err;
+
+ err = pci_enable_device(pdev);
+ if (err)
+ return -ENXIO;
+
+ err = pci_request_regions(pdev, "gvnic-cfg");
+ if (err)
+ goto abort_with_enabled;
+
+ pci_set_master(pdev);
+
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+ if (err) {
+ dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err);
+ goto abort_with_pci_region;
+ }
+
+ err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+ if (err) {
+ dev_err(&pdev->dev,
+ "Failed to set consistent dma mask: err=%d\n", err);
+ goto abort_with_pci_region;
+ }
+
+ reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0);
+ if (!reg_bar) {
+ err = -ENOMEM;
+ goto abort_with_pci_region;
+ }
+
+ db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
+ if (!db_bar) {
+ dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
+ err = -ENOMEM;
+ goto abort_with_reg_bar;
+ }
+
+ gve_write_version(®_bar->driver_version);
+ /* Get max queues to alloc etherdev */
+ max_rx_queues = ioread32be(®_bar->max_tx_queues);
+ max_tx_queues = ioread32be(®_bar->max_rx_queues);
+ /* Alloc and setup the netdev and priv */
+ dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues);
+ if (!dev) {
+ dev_err(&pdev->dev, "could not allocate netdev\n");
+ goto abort_with_db_bar;
+ }
+ SET_NETDEV_DEV(dev, &pdev->dev);
+ pci_set_drvdata(pdev, dev);
+ /* advertise features */
+ dev->hw_features = NETIF_F_HIGHDMA;
+ dev->hw_features |= NETIF_F_SG;
+ dev->hw_features |= NETIF_F_HW_CSUM;
+ dev->hw_features |= NETIF_F_TSO;
+ dev->hw_features |= NETIF_F_TSO6;
+ dev->hw_features |= NETIF_F_TSO_ECN;
+ dev->hw_features |= NETIF_F_RXCSUM;
+ dev->hw_features |= NETIF_F_RXHASH;
+ dev->features = dev->hw_features;
+ dev->min_mtu = ETH_MIN_MTU;
+ netif_carrier_off(dev);
+
+ priv = netdev_priv(dev);
+ priv->dev = dev;
+ priv->pdev = pdev;
+ priv->msg_enable = DEFAULT_MSG_LEVEL;
+ priv->reg_bar0 = reg_bar;
+ priv->db_bar2 = db_bar;
+ priv->state_flags = 0x0;
+
+ err = gve_init_priv(priv, false);
+ if (err)
+ goto abort_with_netdev;
+
+ err = register_netdev(dev);
+ if (err)
+ goto abort_with_netdev;
+
+ dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
+ return 0;
+
+abort_with_netdev:
+ free_netdev(dev);
+
+abort_with_db_bar:
+ pci_iounmap(pdev, db_bar);
+
+abort_with_reg_bar:
+ pci_iounmap(pdev, reg_bar);
+
+abort_with_pci_region:
+ pci_release_regions(pdev);
+
+abort_with_enabled:
+ pci_disable_device(pdev);
+ return -ENXIO;
+}
+EXPORT_SYMBOL(gve_probe);
+
+static void gve_remove(struct pci_dev *pdev)
+{
+ struct net_device *netdev = pci_get_drvdata(pdev);
+ struct gve_priv *priv = netdev_priv(netdev);
+ __be32 __iomem *db_bar = priv->db_bar2;
+ void __iomem *reg_bar = priv->reg_bar0;
+
+ unregister_netdev(netdev);
+ gve_teardown_priv_resources(priv);
+ free_netdev(netdev);
+ pci_iounmap(pdev, db_bar);
+ pci_iounmap(pdev, reg_bar);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+}
+
+static const struct pci_device_id gve_id_table[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) },
+ { }
+};
+
+static struct pci_driver gvnic_driver = {
+ .name = "gvnic",
+ .id_table = gve_id_table,
+ .probe = gve_probe,
+ .remove = gve_remove,
+};
+
+module_pci_driver(gvnic_driver);
+
+MODULE_DEVICE_TABLE(pci, gve_id_table);
+MODULE_AUTHOR("Google, Inc.");
+MODULE_DESCRIPTION("gVNIC Driver");
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_VERSION(GVE_VERSION);
diff --git a/drivers/net/ethernet/google/gve/gve_register.h b/drivers/net/ethernet/google/gve/gve_register.h
new file mode 100644
index 000000000000..84ab8893aadd
--- /dev/null
+++ b/drivers/net/ethernet/google/gve/gve_register.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT)
+ * Google virtual Ethernet (gve) driver
+ *
+ * Copyright (C) 2015-2019 Google, Inc.
+ */
+
+#ifndef _GVE_REGISTER_H_
+#define _GVE_REGISTER_H_
+
+/* Fixed Configuration Registers */
+struct gve_registers {
+ __be32 device_status;
+ __be32 driver_status;
+ __be32 max_tx_queues;
+ __be32 max_rx_queues;
+ __be32 adminq_pfn;
+ __be32 adminq_doorbell;
+ __be32 adminq_event_counter;
+ u8 reserved[3];
+ u8 driver_version;
+};
+
+enum gve_device_status_flags {
+ GVE_DEVICE_STATUS_RESET_MASK = BIT(1),
+ GVE_DEVICE_STATUS_LINK_STATUS_MASK = BIT(2),
+};
+#endif /* _GVE_REGISTER_H_ */
--
2.22.0.410.gd8fdbe21b5-goog
^ permalink raw reply related
* [PATCH net-next v4 0/4] Add gve driver
From: Catherine Sullivan @ 2019-07-01 22:57 UTC (permalink / raw)
To: netdev; +Cc: Catherine Sullivan, kbuild test robot, Julia Lawall
This patch series adds the gve driver which will support the
Compute Engine Virtual NIC that will be available in the future.
v2:
- Patch 1:
- Remove gve_size_assert.h and use static_assert instead.
- Loop forever instead of bugging if the device won't reset
- Use module_pci_driver
- Patch 2:
- Use be16_to_cpu in the RX Seq No define
- Remove unneeded ndo_change_mtu
- Patch 3:
- No Changes
- Patch 4:
- Instead of checking netif_carrier_ok in ethtool stats, just make sure
v3:
- Patch 1:
- Remove X86 dep
- Patch 2:
- No changes
- Patch 3:
- No changes
- Patch 4:
- Remove unneeded memsets in ethtool stats
v4:
- Patch 1:
- Use io[read|write]32be instead of [read|write]l(cpu_to_be32())
- Explicitly add padding to gve_adminq_set_driver_parameter
- Use static where appropriate
- Patch 2:
- Use u64_stats_sync
- Explicity add padding to gve_adminq_create_rx_queue
- Fix some enianness typing issues found by kbuild
- Use static where appropriate
- Remove unused variables
- Patch 3:
- Use io[read|write]32be instead of [read|write]l(cpu_to_be32())
- Patch 4:
- Use u64_stats_sync
- Use static where appropriate
Warnings reported by:
Reported-by: kbuild test robot <lkp@intel.com>
Reported-by: Julia Lawall <julia.lawall@lip6.fr>
Catherine Sullivan (4):
gve: Add basic driver framework for Compute Engine Virtual NIC
gve: Add transmit and receive support
gve: Add workqueue and reset support
gve: Add ethtool support
.../networking/device_drivers/google/gve.rst | 123 ++
.../networking/device_drivers/index.rst | 1 +
MAINTAINERS | 9 +
drivers/net/ethernet/Kconfig | 1 +
drivers/net/ethernet/Makefile | 1 +
drivers/net/ethernet/google/Kconfig | 27 +
drivers/net/ethernet/google/Makefile | 5 +
drivers/net/ethernet/google/gve/Makefile | 4 +
drivers/net/ethernet/google/gve/gve.h | 459 ++++++
drivers/net/ethernet/google/gve/gve_adminq.c | 387 ++++++
drivers/net/ethernet/google/gve/gve_adminq.h | 217 +++
drivers/net/ethernet/google/gve/gve_desc.h | 113 ++
drivers/net/ethernet/google/gve/gve_ethtool.c | 243 ++++
drivers/net/ethernet/google/gve/gve_main.c | 1228 +++++++++++++++++
.../net/ethernet/google/gve/gve_register.h | 27 +
drivers/net/ethernet/google/gve/gve_rx.c | 446 ++++++
drivers/net/ethernet/google/gve/gve_tx.c | 584 ++++++++
17 files changed, 3875 insertions(+)
create mode 100644 Documentation/networking/device_drivers/google/gve.rst
create mode 100644 drivers/net/ethernet/google/Kconfig
create mode 100644 drivers/net/ethernet/google/Makefile
create mode 100644 drivers/net/ethernet/google/gve/Makefile
create mode 100644 drivers/net/ethernet/google/gve/gve.h
create mode 100644 drivers/net/ethernet/google/gve/gve_adminq.c
create mode 100644 drivers/net/ethernet/google/gve/gve_adminq.h
create mode 100644 drivers/net/ethernet/google/gve/gve_desc.h
create mode 100644 drivers/net/ethernet/google/gve/gve_ethtool.c
create mode 100644 drivers/net/ethernet/google/gve/gve_main.c
create mode 100644 drivers/net/ethernet/google/gve/gve_register.h
create mode 100644 drivers/net/ethernet/google/gve/gve_rx.c
create mode 100644 drivers/net/ethernet/google/gve/gve_tx.c
--
2.22.0.410.gd8fdbe21b5-goog
^ permalink raw reply
* Re: [PATCH] net: stmmac: add sanity check to device_property_read_u32_array call
From: Martin Blumenstingl @ 2019-07-01 22:43 UTC (permalink / raw)
To: Colin Ian King
Cc: alexandre.torgue, davem, joabreu, kernel-janitors,
linux-arm-kernel, linux-kernel, linux-stm32, mcoquelin.stm32,
netdev, peppe.cavallaro
In-Reply-To: <CAFBinCAx5qrPK1z68bF-tGKpJQfKLnee65qBOxMS4nj8t381+Q@mail.gmail.com>
On Fri, Jun 28, 2019 at 6:05 PM Martin Blumenstingl
<martin.blumenstingl@googlemail.com> wrote:
>
> Hi Colin,
>
> On Fri, Jun 28, 2019 at 10:32 AM Colin Ian King
> <colin.king@canonical.com> wrote:
> >
> > On 28/06/2019 05:15, Martin Blumenstingl wrote:
> > > On Tue, Jun 25, 2019 at 9:58 AM Colin Ian King <colin.king@canonical.com> wrote:
> > >>
> > >> On 25/06/2019 05:44, Martin Blumenstingl wrote:
> > >>> Hi Colin,
> > >>>
> > >>> On Thu, Jun 20, 2019 at 3:34 AM Martin Blumenstingl
> > >>> <martin.blumenstingl@googlemail.com> wrote:
> > >>>>
> > >>>> Hi Colin,
> > >>>>
> > >>>> On Wed, Jun 19, 2019 at 8:55 AM Colin Ian King <colin.king@canonical.com> wrote:
> > >>>>>
> > >>>>> On 19/06/2019 06:13, Martin Blumenstingl wrote:
> > >>>>>> Hi Colin,
> > >>>>>>
> > >>>>>>> Currently the call to device_property_read_u32_array is not error checked
> > >>>>>>> leading to potential garbage values in the delays array that are then used
> > >>>>>>> in msleep delays. Add a sanity check to the property fetching.
> > >>>>>>>
> > >>>>>>> Addresses-Coverity: ("Uninitialized scalar variable")
> > >>>>>>> Signed-off-by: Colin Ian King <colin.king@canonical.com>
> > >>>>>> I have also sent a patch [0] to fix initialize the array.
> > >>>>>> can you please look at my patch so we can work out which one to use?
> > >>>>>>
> > >>>>>> my concern is that the "snps,reset-delays-us" property is optional,
> > >>>>>> the current dt-bindings documentation states that it's a required
> > >>>>>> property. in reality it isn't, there are boards (two examples are
> > >>>>>> mentioned in my patch: [0]) without it.
> > >>>>>>
> > >>>>>> so I believe that the resulting behavior has to be:
> > >>>>>> 1. don't delay if this property is missing (instead of delaying for
> > >>>>>> <garbage value> ms)
> > >>>>>> 2. don't error out if this property is missing
> > >>>>>>
> > >>>>>> your patch covers #1, can you please check whether #2 is also covered?
> > >>>>>> I tested case #2 when submitting my patch and it worked fine (even
> > >>>>>> though I could not reproduce the garbage values which are being read
> > >>>>>> on some boards)
> > >>> in the meantime I have tested your patch.
> > >>> when I don't set the "snps,reset-delays-us" property then I get the
> > >>> following error:
> > >>> invalid property snps,reset-delays-us
> > >>>
> > >>> my patch has landed in the meantime: [0]
> > >>> how should we proceed with your patch?
> >
> > Your fix is good, so I think we should just drop/forget about my fix.
> thank you for looking at the situation
>
> as far I understand the -net/-net-next tree all commits are immutable
> so if we want to remove your patch we need to send a revert
> do you want me to do that (I can do it on Monday) or will you take care of that?
I just sent the patch: [0]
[0] https://patchwork.ozlabs.org/patch/1125686/
^ permalink raw reply
* [PATCH] net: stmmac: make "snps,reset-delays-us" optional again
From: Martin Blumenstingl @ 2019-07-01 22:42 UTC (permalink / raw)
To: davem, netdev, colin.king
Cc: peppe.cavallaro, alexandre.torgue, joabreu, linux-arm-kernel,
linux-kernel, Martin Blumenstingl
Commit 760f1dc2958022 ("net: stmmac: add sanity check to
device_property_read_u32_array call") introduced error checking of the
device_property_read_u32_array() call in stmmac_mdio_reset().
This results in the following error when the "snps,reset-delays-us"
property is not defined in devicetree:
invalid property snps,reset-delays-us
This sanity check made sense until commit 84ce4d0f9f55b4 ("net: stmmac:
initialize the reset delay array") ensured that there are fallback
values for the reset delay if the "snps,reset-delays-us" property is
absent. That was at the cost of making that property mandatory though.
Drop the sanity check for device_property_read_u32_array() and thus make
the "snps,reset-delays-us" property optional again (avoiding the error
message while loading the stmmac driver with a .dtb where the property
is absent).
Fixes: 760f1dc2958022 ("net: stmmac: add sanity check to device_property_read_u32_array call")
Signed-off-by: Martin Blumenstingl <martin.blumenstingl@googlemail.com>
---
This is a fix for a patch in net-next and should either go into net-next
or 5.3-rcX.
drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c | 13 +++----------
1 file changed, 3 insertions(+), 10 deletions(-)
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
index f8061e34122f..18cadf0b0d66 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
@@ -242,7 +242,6 @@ int stmmac_mdio_reset(struct mii_bus *bus)
if (priv->device->of_node) {
struct gpio_desc *reset_gpio;
u32 delays[3] = { 0, 0, 0 };
- int ret;
reset_gpio = devm_gpiod_get_optional(priv->device,
"snps,reset",
@@ -250,15 +249,9 @@ int stmmac_mdio_reset(struct mii_bus *bus)
if (IS_ERR(reset_gpio))
return PTR_ERR(reset_gpio);
- ret = device_property_read_u32_array(priv->device,
- "snps,reset-delays-us",
- delays,
- ARRAY_SIZE(delays));
- if (ret) {
- dev_err(ndev->dev.parent,
- "invalid property snps,reset-delays-us\n");
- return -EINVAL;
- }
+ device_property_read_u32_array(priv->device,
+ "snps,reset-delays-us",
+ delays, ARRAY_SIZE(delays));
if (delays[0])
msleep(DIV_ROUND_UP(delays[0], 1000));
--
2.22.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox