* [PATCH net-next v2 05/12] net: homa: create homa_rpc.h and homa_rpc.c
From: John Ousterhout @ 2024-11-11 23:39 UTC (permalink / raw)
To: netdev, linux-api; +Cc: John Ousterhout
In-Reply-To: <20241111234006.5942-1-ouster@cs.stanford.edu>
These files provide basic functions for managing remote procedure calls,
which are the fundamental entities managed by Homa. Each RPC consists
of a request message from a client to a server, followed by a response
message returned from the server to the client.
Signed-off-by: John Ousterhout <ouster@cs.stanford.edu>
---
net/homa/homa_rpc.c | 488 ++++++++++++++++++++++++++++++++++++++++++++
net/homa/homa_rpc.h | 446 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 934 insertions(+)
create mode 100644 net/homa/homa_rpc.c
create mode 100644 net/homa/homa_rpc.h
diff --git a/net/homa/homa_rpc.c b/net/homa/homa_rpc.c
new file mode 100644
index 000000000000..a44433b1e745
--- /dev/null
+++ b/net/homa/homa_rpc.c
@@ -0,0 +1,488 @@
+// SPDX-License-Identifier: BSD-2-Clause
+
+/* This file contains functions for managing homa_rpc structs. */
+
+#include "homa_impl.h"
+#include "homa_peer.h"
+#include "homa_pool.h"
+#include "homa_stub.h"
+
+/**
+ * homa_rpc_new_client() - Allocate and construct a client RPC (one that is used
+ * to issue an outgoing request). Doesn't send any packets. Invoked with no
+ * locks held.
+ * @hsk: Socket to which the RPC belongs.
+ * @dest: Address of host (ip and port) to which the RPC will be sent.
+ *
+ * Return: A printer to the newly allocated object, or a negative
+ * errno if an error occurred. The RPC will be locked; the
+ * caller must eventually unlock it.
+ */
+struct homa_rpc *homa_rpc_new_client(struct homa_sock *hsk,
+ const union sockaddr_in_union *dest)
+{
+ struct in6_addr dest_addr_as_ipv6 = canonical_ipv6_addr(dest);
+ struct homa_rpc_bucket *bucket;
+ struct homa_rpc *crpc;
+ int err;
+
+ crpc = kmalloc(sizeof(*crpc), GFP_KERNEL);
+ if (unlikely(!crpc))
+ return ERR_PTR(-ENOMEM);
+
+ /* Initialize fields that don't require the socket lock. */
+ crpc->hsk = hsk;
+ crpc->id = atomic64_fetch_add(2, &hsk->homa->next_outgoing_id);
+ bucket = homa_client_rpc_bucket(hsk, crpc->id);
+ crpc->bucket = bucket;
+ crpc->state = RPC_OUTGOING;
+ atomic_set(&crpc->flags, 0);
+ crpc->peer = homa_peer_find(hsk->homa->peers, &dest_addr_as_ipv6,
+ &hsk->inet);
+ if (IS_ERR(crpc->peer)) {
+ err = PTR_ERR(crpc->peer);
+ goto error;
+ }
+ crpc->dport = ntohs(dest->in6.sin6_port);
+ crpc->completion_cookie = 0;
+ crpc->error = 0;
+ crpc->msgin.length = -1;
+ crpc->msgin.num_bpages = 0;
+ memset(&crpc->msgout, 0, sizeof(crpc->msgout));
+ crpc->msgout.length = -1;
+ INIT_LIST_HEAD(&crpc->ready_links);
+ INIT_LIST_HEAD(&crpc->buf_links);
+ INIT_LIST_HEAD(&crpc->dead_links);
+ crpc->interest = NULL;
+ INIT_LIST_HEAD(&crpc->throttled_links);
+ crpc->silent_ticks = 0;
+ crpc->resend_timer_ticks = hsk->homa->timer_ticks;
+ crpc->done_timer_ticks = 0;
+ crpc->magic = HOMA_RPC_MAGIC;
+ crpc->start_ns = sched_clock();
+
+ /* Initialize fields that require locking. This allows the most
+ * expensive work, such as copying in the message from user space,
+ * to be performed without holding locks. Also, can't hold spin
+ * locks while doing things that could block, such as memory allocation.
+ */
+ homa_bucket_lock(bucket, crpc->id, "homa_rpc_new_client");
+ homa_sock_lock(hsk, "homa_rpc_new_client");
+ if (hsk->shutdown) {
+ homa_sock_unlock(hsk);
+ homa_rpc_unlock(crpc);
+ err = -ESHUTDOWN;
+ goto error;
+ }
+ hlist_add_head(&crpc->hash_links, &bucket->rpcs);
+ list_add_tail_rcu(&crpc->active_links, &hsk->active_rpcs);
+ homa_sock_unlock(hsk);
+
+ __acquire(&crpc->bucket->lock);
+ return crpc;
+
+error:
+ kfree(crpc);
+ return ERR_PTR(err);
+}
+
+/**
+ * homa_rpc_new_server() - Allocate and construct a server RPC (one that is
+ * used to manage an incoming request). If appropriate, the RPC will also
+ * be handed off (we do it here, while we have the socket locked, to avoid
+ * acquiring the socket lock a second time later for the handoff).
+ * @hsk: Socket that owns this RPC.
+ * @source: IP address (network byte order) of the RPC's client.
+ * @h: Header for the first data packet received for this RPC; used
+ * to initialize the RPC.
+ * @created: Will be set to 1 if a new RPC was created and 0 if an
+ * existing RPC was found.
+ *
+ * Return: A pointer to a new RPC, which is locked, or a negative errno
+ * if an error occurred. If there is already an RPC corresponding
+ * to h, then it is returned instead of creating a new RPC.
+ */
+struct homa_rpc *homa_rpc_new_server(struct homa_sock *hsk,
+ const struct in6_addr *source,
+ struct data_header *h, int *created)
+{
+ __u64 id = homa_local_id(h->common.sender_id);
+ struct homa_rpc_bucket *bucket;
+ struct homa_rpc *srpc = NULL;
+ int err;
+
+ /* Lock the bucket, and make sure no-one else has already created
+ * the desired RPC.
+ */
+ bucket = homa_server_rpc_bucket(hsk, id);
+ homa_bucket_lock(bucket, id, "homa_rpc_new_server");
+ hlist_for_each_entry_rcu(srpc, &bucket->rpcs, hash_links) {
+ if (srpc->id == id &&
+ srpc->dport == ntohs(h->common.sport) &&
+ ipv6_addr_equal(&srpc->peer->addr, source)) {
+ /* RPC already exists; just return it instead
+ * of creating a new RPC.
+ */
+ *created = 0;
+ __acquire(&srpc->bucket->lock);
+ return srpc;
+ }
+ }
+
+ /* Initialize fields that don't require the socket lock. */
+ srpc = kmalloc(sizeof(*srpc), GFP_KERNEL);
+ if (!srpc) {
+ err = -ENOMEM;
+ goto error;
+ }
+ srpc->hsk = hsk;
+ srpc->bucket = bucket;
+ srpc->state = RPC_INCOMING;
+ atomic_set(&srpc->flags, 0);
+ srpc->peer = homa_peer_find(hsk->homa->peers, source, &hsk->inet);
+ if (IS_ERR(srpc->peer)) {
+ err = PTR_ERR(srpc->peer);
+ goto error;
+ }
+ srpc->dport = ntohs(h->common.sport);
+ srpc->id = id;
+ srpc->completion_cookie = 0;
+ srpc->error = 0;
+ srpc->msgin.length = -1;
+ srpc->msgin.num_bpages = 0;
+ memset(&srpc->msgout, 0, sizeof(srpc->msgout));
+ srpc->msgout.length = -1;
+ INIT_LIST_HEAD(&srpc->ready_links);
+ INIT_LIST_HEAD(&srpc->buf_links);
+ INIT_LIST_HEAD(&srpc->dead_links);
+ srpc->interest = NULL;
+ INIT_LIST_HEAD(&srpc->throttled_links);
+ srpc->silent_ticks = 0;
+ srpc->resend_timer_ticks = hsk->homa->timer_ticks;
+ srpc->done_timer_ticks = 0;
+ srpc->magic = HOMA_RPC_MAGIC;
+ srpc->start_ns = sched_clock();
+ err = homa_message_in_init(srpc, ntohl(h->message_length));
+ if (err != 0)
+ goto error;
+
+ /* Initialize fields that require socket to be locked. */
+ homa_sock_lock(hsk, "homa_rpc_new_server");
+ if (hsk->shutdown) {
+ homa_sock_unlock(hsk);
+ err = -ESHUTDOWN;
+ goto error;
+ }
+ hlist_add_head(&srpc->hash_links, &bucket->rpcs);
+ list_add_tail_rcu(&srpc->active_links, &hsk->active_rpcs);
+ if (ntohl(h->seg.offset) == 0 && srpc->msgin.num_bpages > 0) {
+ atomic_or(RPC_PKTS_READY, &srpc->flags);
+ homa_rpc_handoff(srpc);
+ }
+ homa_sock_unlock(hsk);
+ *created = 1;
+ __acquire(&srpc->bucket->lock);
+ return srpc;
+
+error:
+ homa_bucket_unlock(bucket, id);
+ kfree(srpc);
+ return ERR_PTR(err);
+}
+
+/**
+ * homa_rpc_acked() - This function is invoked when an ack is received
+ * for an RPC; if the RPC still exists, is freed.
+ * @hsk: Socket on which the ack was received. May or may not correspond
+ * to the RPC, but can sometimes be used to avoid a socket lookup.
+ * @saddr: Source address from which the act was received (the client
+ * note for the RPC)
+ * @ack: Information about an RPC from @saddr that may now be deleted safely.
+ */
+void homa_rpc_acked(struct homa_sock *hsk, const struct in6_addr *saddr,
+ struct homa_ack *ack)
+{
+ __u16 client_port = ntohs(ack->client_port);
+ __u16 server_port = ntohs(ack->server_port);
+ __u64 id = homa_local_id(ack->client_id);
+ struct homa_sock *hsk2 = hsk;
+ struct homa_rpc *rpc;
+
+ if (hsk2->port != server_port) {
+ /* Without RCU, sockets other than hsk can be deleted
+ * out from under us.
+ */
+ rcu_read_lock();
+ hsk2 = homa_sock_find(hsk->homa->port_map, server_port);
+ if (!hsk2)
+ goto done;
+ }
+ rpc = homa_find_server_rpc(hsk2, saddr, client_port, id);
+ if (rpc) {
+ homa_rpc_free(rpc);
+ homa_rpc_unlock(rpc);
+ }
+
+done:
+ if (hsk->port != server_port)
+ rcu_read_unlock();
+}
+
+/**
+ * homa_rpc_free() - Destructor for homa_rpc; will arrange for all resources
+ * associated with the RPC to be released (eventually).
+ * @rpc: Structure to clean up, or NULL. Must be locked. Its socket must
+ * not be locked.
+ */
+void homa_rpc_free(struct homa_rpc *rpc)
+ __acquires(&rpc->hsk->lock)
+ __releases(&rpc->hsk->lock)
+{
+ /* The goal for this function is to make the RPC inaccessible,
+ * so that no other code will ever access it again. However, don't
+ * actually release resources; leave that to homa_rpc_reap, which
+ * runs later. There are two reasons for this. First, releasing
+ * resources may be expensive, so we don't want to keep the caller
+ * waiting; homa_rpc_reap will run in situations where there is time
+ * to spare. Second, there may be other code that currently has
+ * pointers to this RPC but temporarily released the lock (e.g. to
+ * copy data to/from user space). It isn't safe to clean up until
+ * that code has finished its work and released any pointers to the
+ * RPC (homa_rpc_reap will ensure that this has happened). So, this
+ * function should only make changes needed to make the RPC
+ * inaccessible.
+ */
+ if (!rpc || rpc->state == RPC_DEAD)
+ return;
+ rpc->state = RPC_DEAD;
+
+ /* Unlink from all lists, so no-one will ever find this RPC again. */
+ homa_sock_lock(rpc->hsk, "homa_rpc_free");
+ __hlist_del(&rpc->hash_links);
+ list_del_rcu(&rpc->active_links);
+ list_add_tail_rcu(&rpc->dead_links, &rpc->hsk->dead_rpcs);
+ __list_del_entry(&rpc->ready_links);
+ __list_del_entry(&rpc->buf_links);
+ if (rpc->interest) {
+ rpc->interest->reg_rpc = NULL;
+ wake_up_process(rpc->interest->thread);
+ rpc->interest = NULL;
+ }
+
+ if (rpc->msgin.length >= 0) {
+ rpc->hsk->dead_skbs += skb_queue_len(&rpc->msgin.packets);
+ while (1) {
+ struct homa_gap *gap = list_first_entry_or_null(&rpc->msgin.gaps,
+ struct homa_gap, links);
+ if (!gap)
+ break;
+ list_del(&gap->links);
+ kfree(gap);
+ }
+ }
+ rpc->hsk->dead_skbs += rpc->msgout.num_skbs;
+ if (rpc->hsk->dead_skbs > rpc->hsk->homa->max_dead_buffs)
+ /* This update isn't thread-safe; it's just a
+ * statistic so it's OK if updates occasionally get
+ * missed.
+ */
+ rpc->hsk->homa->max_dead_buffs = rpc->hsk->dead_skbs;
+
+ homa_sock_unlock(rpc->hsk);
+ homa_remove_from_throttled(rpc);
+}
+
+/**
+ * homa_rpc_reap() - Invoked to release resources associated with dead
+ * RPCs for a given socket. For a large RPC, it can take a long time to
+ * free all of its packet buffers, so we try to perform this work
+ * off the critical path where it won't delay applications. Each call to
+ * this function does a small chunk of work. See the file reap.txt for
+ * more information.
+ * @hsk: Homa socket that may contain dead RPCs. Must not be locked by the
+ * caller; this function will lock and release.
+ * @count: Number of buffers to free during this call.
+ *
+ * Return: A return value of 0 means that we ran out of work to do; calling
+ * again will do no work (there could be unreaped RPCs, but if so,
+ * reaping has been disabled for them). A value greater than
+ * zero means there is still more reaping work to be done.
+ */
+int homa_rpc_reap(struct homa_sock *hsk, int count)
+{
+#define BATCH_MAX 20
+ struct homa_rpc *rpcs[BATCH_MAX];
+ struct sk_buff *skbs[BATCH_MAX];
+ int num_skbs, num_rpcs;
+ struct homa_rpc *rpc;
+ int i, batch_size;
+ int rx_frees = 0;
+ int result;
+
+ /* Each iteration through the following loop will reap
+ * BATCH_MAX skbs.
+ */
+ while (count > 0) {
+ batch_size = count;
+ if (batch_size > BATCH_MAX)
+ batch_size = BATCH_MAX;
+ count -= batch_size;
+ num_skbs = 0;
+ num_rpcs = 0;
+
+ homa_sock_lock(hsk, "homa_rpc_reap");
+ if (atomic_read(&hsk->protect_count)) {
+ homa_sock_unlock(hsk);
+ return 0;
+ }
+
+ /* Collect buffers and freeable RPCs. */
+ list_for_each_entry_rcu(rpc, &hsk->dead_rpcs, dead_links) {
+ if ((atomic_read(&rpc->flags) & RPC_CANT_REAP) ||
+ atomic_read(&rpc->msgout.active_xmits) != 0) {
+ continue;
+ }
+ rpc->magic = 0;
+
+ /* For Tx sk_buffs, collect them here but defer
+ * freeing until after releasing the socket lock.
+ */
+ if (rpc->msgout.length >= 0) {
+ while (rpc->msgout.packets) {
+ skbs[num_skbs] = rpc->msgout.packets;
+ rpc->msgout.packets = homa_get_skb_info(rpc
+ ->msgout.packets)->next_skb;
+ num_skbs++;
+ rpc->msgout.num_skbs--;
+ if (num_skbs >= batch_size)
+ goto release;
+ }
+ }
+
+ /* In the normal case rx sk_buffs will already have been
+ * freed before we got here. Thus it's OK to free
+ * immediately in rare situations where there are
+ * buffers left.
+ */
+ if (rpc->msgin.length >= 0) {
+ while (1) {
+ struct sk_buff *skb;
+
+ skb = skb_dequeue(&rpc->msgin.packets);
+ if (!skb)
+ break;
+ kfree_skb(skb);
+ rx_frees++;
+ }
+ }
+
+ /* If we get here, it means all packets have been
+ * removed from the RPC.
+ */
+ rpcs[num_rpcs] = rpc;
+ num_rpcs++;
+ list_del_rcu(&rpc->dead_links);
+ if (num_rpcs >= batch_size)
+ goto release;
+ }
+
+ /* Free all of the collected resources; release the socket
+ * lock while doing this.
+ */
+release:
+ hsk->dead_skbs -= num_skbs + rx_frees;
+ result = !list_empty(&hsk->dead_rpcs) &&
+ (num_skbs + num_rpcs) != 0;
+ homa_sock_unlock(hsk);
+ homa_skb_free_many_tx(hsk->homa, skbs, num_skbs);
+ for (i = 0; i < num_rpcs; i++) {
+ rpc = rpcs[i];
+ /* Lock and unlock the RPC before freeing it. This
+ * is needed to deal with races where the code
+ * that invoked homa_rpc_free hasn't unlocked the
+ * RPC yet.
+ */
+ homa_rpc_lock(rpc, "homa_rpc_reap");
+ homa_rpc_unlock(rpc);
+
+ if (unlikely(rpc->msgin.num_bpages))
+ homa_pool_release_buffers(rpc->hsk->buffer_pool,
+ rpc->msgin.num_bpages,
+ rpc->msgin.bpage_offsets);
+ if (rpc->msgin.length >= 0) {
+ while (1) {
+ struct homa_gap *gap = list_first_entry_or_null(&rpc
+ ->msgin.gaps,
+ struct homa_gap, links);
+ if (!gap)
+ break;
+ list_del(&gap->links);
+ kfree(gap);
+ }
+ }
+ rpc->state = 0;
+ kfree(rpc);
+ }
+ if (!result)
+ break;
+ }
+ homa_pool_check_waiting(hsk->buffer_pool);
+ return result;
+}
+
+/**
+ * homa_find_client_rpc() - Locate client-side information about the RPC that
+ * a packet belongs to, if there is any. Thread-safe without socket lock.
+ * @hsk: Socket via which packet was received.
+ * @id: Unique identifier for the RPC.
+ *
+ * Return: A pointer to the homa_rpc for this id, or NULL if none.
+ * The RPC will be locked; the caller must eventually unlock it
+ * by invoking homa_rpc_unlock.
+ */
+struct homa_rpc *homa_find_client_rpc(struct homa_sock *hsk, __u64 id)
+{
+ struct homa_rpc_bucket *bucket = homa_client_rpc_bucket(hsk, id);
+ struct homa_rpc *crpc;
+
+ homa_bucket_lock(bucket, id, __func__);
+ hlist_for_each_entry_rcu(crpc, &bucket->rpcs, hash_links) {
+ if (crpc->id == id) {
+ __acquire(&crpc->bucket->lock);
+ return crpc;
+ }
+ }
+ homa_bucket_unlock(bucket, id);
+ return NULL;
+}
+
+/**
+ * homa_find_server_rpc() - Locate server-side information about the RPC that
+ * a packet belongs to, if there is any. Thread-safe without socket lock.
+ * @hsk: Socket via which packet was received.
+ * @saddr: Address from which the packet was sent.
+ * @sport: Port at @saddr from which the packet was sent.
+ * @id: Unique identifier for the RPC (must have server bit set).
+ *
+ * Return: A pointer to the homa_rpc matching the arguments, or NULL
+ * if none. The RPC will be locked; the caller must eventually
+ * unlock it by invoking homa_rpc_unlock.
+ */
+struct homa_rpc *homa_find_server_rpc(struct homa_sock *hsk,
+ const struct in6_addr *saddr, __u16 sport,
+ __u64 id)
+{
+ struct homa_rpc_bucket *bucket = homa_server_rpc_bucket(hsk, id);
+ struct homa_rpc *srpc;
+
+ homa_bucket_lock(bucket, id, __func__);
+ hlist_for_each_entry_rcu(srpc, &bucket->rpcs, hash_links) {
+ if (srpc->id == id && srpc->dport == sport &&
+ ipv6_addr_equal(&srpc->peer->addr, saddr)) {
+ __acquire(&srpc->bucket->lock);
+ return srpc;
+ }
+ }
+ homa_bucket_unlock(bucket, id);
+ return NULL;
+}
diff --git a/net/homa/homa_rpc.h b/net/homa/homa_rpc.h
new file mode 100644
index 000000000000..19c262f56039
--- /dev/null
+++ b/net/homa/homa_rpc.h
@@ -0,0 +1,446 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+
+/* This file defines homa_rpc and related structs. */
+
+#ifndef _HOMA_RPC_H
+#define _HOMA_RPC_H
+
+#include <linux/percpu-defs.h>
+#include <linux/skbuff.h>
+#include <linux/types.h>
+
+#include "homa_sock.h"
+#include "homa_wire.h"
+
+/* Forward references. */
+struct homa_ack;
+
+/**
+ * struct homa_message_out - Describes a message (either request or response)
+ * for which this machine is the sender.
+ */
+struct homa_message_out {
+ /**
+ * @length: Total bytes in message (excluding headers). A value
+ * less than 0 means this structure is uninitialized and therefore
+ * not in use (all other fields will be zero in this case).
+ */
+ int length;
+
+ /** @num_skbs: Total number of buffers currently in @packets. */
+ int num_skbs;
+
+ /**
+ * @copied_from_user: Number of bytes of the message that have
+ * been copied from user space into skbs in @packets.
+ */
+ int copied_from_user;
+
+ /**
+ * @packets: Singly-linked list of all packets in message, linked
+ * using homa_next_skb. The list is in order of offset in the message
+ * (offset 0 first); each sk_buff can potentially contain multiple
+ * data_segments, which will be split into separate packets by GSO.
+ * This list grows gradually as data is copied in from user space,
+ * so it may not be complete.
+ */
+ struct sk_buff *packets;
+
+ /**
+ * @next_xmit: Pointer to pointer to next packet to transmit (will
+ * either refer to @packets or homa_next_skb(skb) for some skb
+ * in @packets).
+ */
+ struct sk_buff **next_xmit;
+
+ /**
+ * @next_xmit_offset: All bytes in the message, up to but not
+ * including this one, have been transmitted.
+ */
+ int next_xmit_offset;
+
+ /**
+ * @active_xmits: The number of threads that are currently
+ * transmitting data packets for this RPC; can't reap the RPC
+ * until this count becomes zero.
+ */
+ atomic_t active_xmits;
+
+ /**
+ * @init_ns: Time in sched_clock units when this structure was
+ * initialized. Used to find the oldest outgoing message.
+ */
+ __u64 init_ns;
+};
+
+/**
+ * struct homa_gap - Represents a range of bytes within a message that have
+ * not yet been received.
+ */
+struct homa_gap {
+ /** @start: offset of first byte in this gap. */
+ int start;
+
+ /** @end: offset of byte just after last one in this gap. */
+ int end;
+
+ /**
+ * @time: time (in sched_clock units) when the gap was first detected.
+ * As of 7/2024 this isn't used for anything.
+ */
+ __u64 time;
+
+ /** @links: for linking into list in homa_message_in. */
+ struct list_head links;
+};
+
+/**
+ * struct homa_message_in - Holds the state of a message received by
+ * this machine; used for both requests and responses.
+ */
+struct homa_message_in {
+ /**
+ * @length: Payload size in bytes. A value less than 0 means this
+ * structure is uninitialized and therefore not in use.
+ */
+ int length;
+
+ /**
+ * @packets: DATA packets for this message that have been received but
+ * not yet copied to user space (no particular order).
+ */
+ struct sk_buff_head packets;
+
+ /**
+ * @recv_end: Offset of the byte just after the highest one that
+ * has been received so far.
+ */
+ int recv_end;
+
+ /**
+ * @gaps: List of homa_gaps describing all of the bytes with
+ * offsets less than @recv_end that have not yet been received.
+ */
+ struct list_head gaps;
+
+ /**
+ * @bytes_remaining: Amount of data for this message that has
+ * not yet been received; will determine the message's priority.
+ */
+ int bytes_remaining;
+
+ /** @resend_all: if nonzero, set resend_all in the next grant packet. */
+ __u8 resend_all;
+
+ /**
+ * @num_bpages: The number of entries in @bpage_offsets used for this
+ * message (0 means buffers not allocated yet).
+ */
+ __u32 num_bpages;
+
+ /** @bpage_offsets: Describes buffer space allocated for this message.
+ * Each entry is an offset from the start of the buffer region.
+ * All but the last pointer refer to areas of size HOMA_BPAGE_SIZE.
+ */
+ __u32 bpage_offsets[HOMA_MAX_BPAGES];
+};
+
+/**
+ * struct homa_rpc - One of these structures exists for each active
+ * RPC. The same structure is used to manage both outgoing RPCs on
+ * clients and incoming RPCs on servers.
+ */
+struct homa_rpc {
+ /** @hsk: Socket that owns the RPC. */
+ struct homa_sock *hsk;
+
+ /** @bucket: Pointer to the bucket in hsk->client_rpc_buckets or
+ * hsk->server_rpc_buckets where this RPC is linked. Used primarily
+ * for locking the RPC (which is done by locking its bucket).
+ */
+ struct homa_rpc_bucket *bucket;
+
+ /**
+ * @state: The current state of this RPC:
+ *
+ * @RPC_OUTGOING: The RPC is waiting for @msgout to be transmitted
+ * to the peer.
+ * @RPC_INCOMING: The RPC is waiting for data @msgin to be received
+ * from the peer; at least one packet has already
+ * been received.
+ * @RPC_IN_SERVICE: Used only for server RPCs: the request message
+ * has been read from the socket, but the response
+ * message has not yet been presented to the kernel.
+ * @RPC_DEAD: RPC has been deleted and is waiting to be
+ * reaped. In some cases, information in the RPC
+ * structure may be accessed in this state.
+ *
+ * Client RPCs pass through states in the following order:
+ * RPC_OUTGOING, RPC_INCOMING, RPC_DEAD.
+ *
+ * Server RPCs pass through states in the following order:
+ * RPC_INCOMING, RPC_IN_SERVICE, RPC_OUTGOING, RPC_DEAD.
+ */
+ enum {
+ RPC_OUTGOING = 5,
+ RPC_INCOMING = 6,
+ RPC_IN_SERVICE = 8,
+ RPC_DEAD = 9
+ } state;
+
+ /**
+ * @flags: Additional state information: an OR'ed combination of
+ * various single-bit flags. See below for definitions. Must be
+ * manipulated with atomic operations because some of the manipulations
+ * occur without holding the RPC lock.
+ */
+ atomic_t flags;
+
+ /* Valid bits for @flags:
+ * RPC_PKTS_READY - The RPC has input packets ready to be
+ * copied to user space.
+ * RPC_COPYING_FROM_USER - Data is being copied from user space into
+ * the RPC; the RPC must not be reaped.
+ * RPC_COPYING_TO_USER - Data is being copied from this RPC to
+ * user space; the RPC must not be reaped.
+ * RPC_HANDING_OFF - This RPC is in the process of being
+ * handed off to a waiting thread; it must
+ * not be reaped.
+ * APP_NEEDS_LOCK - Means that code in the application thread
+ * needs the RPC lock (e.g. so it can start
+ * copying data to user space) so others
+ * (e.g. SoftIRQ processing) should relinquish
+ * the lock ASAP. Without this, SoftIRQ can
+ * lock out the application for a long time,
+ * preventing data copies to user space from
+ * starting (and they limit throughput at
+ * high network speeds).
+ */
+#define RPC_PKTS_READY 1
+#define RPC_COPYING_FROM_USER 2
+#define RPC_COPYING_TO_USER 4
+#define RPC_HANDING_OFF 8
+#define APP_NEEDS_LOCK 16
+
+#define RPC_CANT_REAP (RPC_COPYING_FROM_USER | RPC_COPYING_TO_USER \
+ | RPC_HANDING_OFF)
+
+ /**
+ * @peer: Information about the other machine (the server, if
+ * this is a client RPC, or the client, if this is a server RPC).
+ */
+ struct homa_peer *peer;
+
+ /** @dport: Port number on @peer that will handle packets. */
+ __u16 dport;
+
+ /**
+ * @id: Unique identifier for the RPC among all those issued
+ * from its port. The low-order bit indicates whether we are
+ * server (1) or client (0) for this RPC.
+ */
+ __u64 id;
+
+ /**
+ * @completion_cookie: Only used on clients. Contains identifying
+ * information about the RPC provided by the application; returned to
+ * the application with the RPC's result.
+ */
+ __u64 completion_cookie;
+
+ /**
+ * @error: Only used on clients. If nonzero, then the RPC has
+ * failed and the value is a negative errno that describes the
+ * problem.
+ */
+ int error;
+
+ /**
+ * @msgin: Information about the message we receive for this RPC
+ * (for server RPCs this is the request, for client RPCs this is the
+ * response).
+ */
+ struct homa_message_in msgin;
+
+ /**
+ * @msgout: Information about the message we send for this RPC
+ * (for client RPCs this is the request, for server RPCs this is the
+ * response).
+ */
+ struct homa_message_out msgout;
+
+ /**
+ * @hash_links: Used to link this object into a hash bucket for
+ * either @hsk->client_rpc_buckets (for a client RPC), or
+ * @hsk->server_rpc_buckets (for a server RPC).
+ */
+ struct hlist_node hash_links;
+
+ /**
+ * @ready_links: Used to link this object into
+ * @hsk->ready_requests or @hsk->ready_responses.
+ */
+ struct list_head ready_links;
+
+ /**
+ * @buf_links: Used to link this RPC into @hsk->waiting_for_bufs.
+ * If the RPC isn't on @hsk->waiting_for_bufs, this is an empty
+ * list pointing to itself.
+ */
+ struct list_head buf_links;
+
+ /**
+ * @active_links: For linking this object into @hsk->active_rpcs.
+ * The next field will be LIST_POISON1 if this RPC hasn't yet been
+ * linked into @hsk->active_rpcs. Access with RCU.
+ */
+ struct list_head active_links;
+
+ /** @dead_links: For linking this object into @hsk->dead_rpcs. */
+ struct list_head dead_links;
+
+ /**
+ * @interest: Describes a thread that wants to be notified when
+ * msgin is complete, or NULL if none.
+ */
+ struct homa_interest *interest;
+
+ /**
+ * @throttled_links: Used to link this RPC into homa->throttled_rpcs.
+ * If this RPC isn't in homa->throttled_rpcs, this is an empty
+ * list pointing to itself.
+ */
+ struct list_head throttled_links;
+
+ /**
+ * @silent_ticks: Number of times homa_timer has been invoked
+ * since the last time a packet indicating progress was received
+ * for this RPC, so we don't need to send a resend for a while.
+ */
+ int silent_ticks;
+
+ /**
+ * @resend_timer_ticks: Value of homa->timer_ticks the last time
+ * we sent a RESEND for this RPC.
+ */
+ __u32 resend_timer_ticks;
+
+ /**
+ * @done_timer_ticks: The value of homa->timer_ticks the first
+ * time we noticed that this (server) RPC is done (all response
+ * packets have been transmitted), so we're ready for an ack.
+ * Zero means we haven't reached that point yet.
+ */
+ __u32 done_timer_ticks;
+
+ /**
+ * @magic: when the RPC is alive, this holds a distinct value that
+ * is unlikely to occur naturally. The value is cleared when the
+ * RPC is reaped, so we can detect accidental use of an RPC after
+ * it has been reaped.
+ */
+#define HOMA_RPC_MAGIC 0xdeadbeef
+ int magic;
+
+ /**
+ * @start_ns: time (from sched_clock()) when this RPC was created.
+ * Used (sometimes) for testing.
+ */
+ u64 start_ns;
+};
+
+void homa_check_rpc(struct homa_rpc *rpc);
+struct homa_rpc *homa_find_client_rpc(struct homa_sock *hsk, __u64 id);
+struct homa_rpc *homa_find_server_rpc(struct homa_sock *hsk,
+ const struct in6_addr *saddr, __u16 sport,
+ __u64 id);
+void homa_rpc_acked(struct homa_sock *hsk,
+ const struct in6_addr *saddr,
+ struct homa_ack *ack);
+void homa_rpc_free(struct homa_rpc *rpc);
+struct homa_rpc *homa_rpc_new_client(struct homa_sock *hsk,
+ const union sockaddr_in_union *dest);
+struct homa_rpc *homa_rpc_new_server(struct homa_sock *hsk,
+ const struct in6_addr *source,
+ struct data_header *h,
+ int *created);
+int homa_rpc_reap(struct homa_sock *hsk, int count);
+
+/**
+ * homa_rpc_lock() - Acquire the lock for an RPC.
+ * @rpc: RPC to lock. Note: this function is only safe under
+ * limited conditions (in most cases homa_bucket_lock should be
+ * used). The caller must ensure that the RPC cannot be reaped
+ * before the lock is acquired. It cannot do that by acquirin
+ * the socket lock, since that violates lock ordering constraints.
+ * One approach is to use homa_protect_rpcs. Don't use this function
+ * unless you are very sure what you are doing! See sync.txt for
+ * more info on locking.
+ * @locker: Static string identifying the locking code. Normally ignored,
+ * but used occasionally for diagnostics and debugging.
+ */
+// static inline void homa_rpc_lock(struct homa_rpc *rpc, const char *locker)
+// __acquires(&rpc->bucket->lock)
+// {
+// homa_bucket_lock(rpc->bucket, rpc->id, locker);
+// }
+
+#define homa_rpc_lock(rpc, locker) do { \
+ struct homa_rpc *_rpc = rpc; \
+ homa_bucket_lock(_rpc->bucket, _rpc->id, locker); \
+} while (0)
+
+/**
+ * homa_rpc_unlock() - Release the lock for an RPC.
+ * @rpc: RPC to unlock.
+ */
+static inline void homa_rpc_unlock(struct homa_rpc *rpc)
+ __releases(&rpc->bucket->lock)
+{
+ homa_bucket_unlock(rpc->bucket, rpc->id);
+}
+
+/**
+ * homa_protect_rpcs() - Ensures that no RPCs will be reaped for a given
+ * socket until homa_sock_unprotect is called. Typically used by functions
+ * that want to scan the active RPCs for a socket without holding the socket
+ * lock. Multiple calls to this function may be in effect at once.
+ * @hsk: Socket whose RPCs should be protected. Must not be locked
+ * by the caller; will be locked here.
+ *
+ * Return: 1 for success, 0 if the socket has been shutdown, in which
+ * case its RPCs cannot be protected.
+ */
+static inline int homa_protect_rpcs(struct homa_sock *hsk)
+{
+ int result;
+
+ homa_sock_lock(hsk, __func__);
+ result = !hsk->shutdown;
+ if (result)
+ atomic_inc(&hsk->protect_count);
+ homa_sock_unlock(hsk);
+ return result;
+}
+
+/**
+ * homa_unprotect_rpcs() - Cancel the effect of a previous call to
+ * homa_sock_protect(), so that RPCs can once again be reaped.
+ * @hsk: Socket whose RPCs should be unprotected.
+ */
+static inline void homa_unprotect_rpcs(struct homa_sock *hsk)
+{
+ atomic_dec(&hsk->protect_count);
+}
+
+/**
+ * homa_is_client(): returns true if we are the client for a particular RPC,
+ * false if we are the server.
+ * @id: Id of the RPC in question.
+ */
+static inline bool homa_is_client(__u64 id)
+{
+ return (id & 1) == 0;
+}
+
+#endif /* _HOMA_RPC_H */
--
2.34.1
^ permalink raw reply related
* [PATCH net-next v2 02/12] net: homa: define Homa packet formats
From: John Ousterhout @ 2024-11-11 23:39 UTC (permalink / raw)
To: netdev, linux-api; +Cc: John Ousterhout
In-Reply-To: <20241111234006.5942-1-ouster@cs.stanford.edu>
Signed-off-by: John Ousterhout <ouster@cs.stanford.edu>
---
net/homa/homa_wire.h | 378 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 378 insertions(+)
create mode 100644 net/homa/homa_wire.h
diff --git a/net/homa/homa_wire.h b/net/homa/homa_wire.h
new file mode 100644
index 000000000000..d7b87dc88cc1
--- /dev/null
+++ b/net/homa/homa_wire.h
@@ -0,0 +1,378 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+
+/* This file defines the on-the-wire format of Homa packets. */
+
+#ifndef _HOMA_WIRE_H
+#define _HOMA_WIRE_H
+
+#include <linux/skbuff.h>
+
+/**
+ * enum homa_packet_type - Defines the possible types of Homa packets.
+ *
+ * See the xxx_header structs below for more information about each type.
+ */
+enum homa_packet_type {
+ DATA = 0x10,
+ RESEND = 0x12,
+ UNKNOWN = 0x13,
+ BUSY = 0x14,
+ NEED_ACK = 0x17,
+ ACK = 0x18,
+ BOGUS = 0x19, /* Used only in unit tests. */
+ /* If you add a new type here, you must also do the following:
+ * 1. Change BOGUS so it is the highest opcode
+ * 2. Add support for the new opcode in homa_print_packet,
+ * homa_print_packet_short, homa_symbol_for_type, and mock_skb_new.
+ * 3. Add the header length to header_lengths in homa_plumbing.c.
+ */
+};
+
+/** define HOMA_IPV6_HEADER_LENGTH - Size of IP header (V6). */
+#define HOMA_IPV6_HEADER_LENGTH 40
+
+/** define HOMA_IPV4_HEADER_LENGTH - Size of IP header (V4). */
+#define HOMA_IPV4_HEADER_LENGTH 20
+
+/**
+ * define HOMA_SKB_EXTRA - How many bytes of additional space to allow at the
+ * beginning of each sk_buff, before the IP header. This includes room for a
+ * VLAN header and also includes some extra space, "just to be safe" (not
+ * really sure if this is needed).
+ */
+#define HOMA_SKB_EXTRA 40
+
+/**
+ * define HOMA_ETH_OVERHEAD - Number of bytes per Ethernet packet for Ethernet
+ * header, CRC, preamble, and inter-packet gap.
+ */
+#define HOMA_ETH_OVERHEAD 42
+
+/**
+ * define HOMA_MIN_PKT_LENGTH - Every Homa packet must be padded to at least
+ * this length to meet Ethernet frame size limitations. This number includes
+ * Homa headers and data, but not IP or Ethernet headers.
+ */
+#define HOMA_MIN_PKT_LENGTH 26
+
+/**
+ * define HOMA_MAX_HEADER - Number of bytes in the largest Homa header.
+ */
+#define HOMA_MAX_HEADER 90
+
+/**
+ * define ETHERNET_MAX_PAYLOAD - Maximum length of an Ethernet packet,
+ * excluding preamble, frame delimeter, VLAN header, CRC, and interpacket gap;
+ * i.e. all of this space is available for Homa.
+ */
+#define ETHERNET_MAX_PAYLOAD 1500
+
+/**
+ * define HOMA_MAX_PRIORITIES - The maximum number of priority levels that
+ * Homa can use (the actual number can be restricted to less than this at
+ * runtime). Changing this value will affect packet formats.
+ */
+#define HOMA_MAX_PRIORITIES 8
+
+/**
+ * struct common_header - Wire format for the first bytes in every Homa
+ * packet. This must (mostly) match the format of a TCP header to enable
+ * Homa packets to actually be transmitted as TCP packets (and thereby
+ * take advantage of TSO and other features).
+ */
+struct common_header {
+ /**
+ * @sport: Port on source machine from which packet was sent.
+ * Must be in the same position as in a TCP header.
+ */
+ __be16 sport;
+
+ /**
+ * @dport: Port on destination that is to receive packet. Must be
+ * in the same position as in a TCP header.
+ */
+ __be16 dport;
+
+ /**
+ * @sequence: corresponds to the sequence number field in TCP headers;
+ * used in DATA packets to hold the offset in the message of the first
+ * byte of data. This value will only be correct in the first segment
+ * of a GSO packet.
+ */
+ __be32 sequence;
+
+ /**
+ * The fields below correspond to the acknowledgment field in TCP
+ * headers; not used by Homa, except for the low-order 8 bits, which
+ * specify the Homa packet type (one of the values in the
+ * homa_packet_type enum).
+ */
+ __be16 ack1;
+ __u8 ack2;
+ __u8 type;
+
+ /**
+ * @doff: High order 4 bits holds the number of 4-byte chunks in a
+ * data_header (low-order bits unused). Used only for DATA packets;
+ * must be in the same position as the data offset in a TCP header.
+ * Used by TSO to determine where the replicated header portion ends.
+ */
+ __u8 doff;
+
+ __u8 dummy1;
+
+ /**
+ * @window: Corresponds to the window field in TCP headers. Not used
+ * by HOMA.
+ */
+ __be16 window;
+
+ /**
+ * @checksum: not used by Homa, but must occupy the same bytes as
+ * the checksum in a TCP header (TSO may modify this?).
+ */
+ __be16 checksum;
+
+ __be16 dummy2;
+
+ /**
+ * @sender_id: the identifier of this RPC as used on the sender (i.e.,
+ * if the low-order bit is set, then the sender is the server for
+ * this RPC).
+ */
+ __be64 sender_id;
+} __packed;
+
+/**
+ * struct homa_ack - Identifies an RPC that can be safely deleted by its
+ * server. After sending the response for an RPC, the server must retain its
+ * state for the RPC until it knows that the client has successfully
+ * received the entire response. An ack indicates this. Clients will
+ * piggyback acks on future data packets, but if a client doesn't send
+ * any data to the server, the server will eventually request an ack
+ * explicitly with a NEED_ACK packet, in which case the client will
+ * return an explicit ACK.
+ */
+struct homa_ack {
+ /**
+ * @id: The client's identifier for the RPC. 0 means this ack
+ * is invalid.
+ */
+ __be64 client_id;
+
+ /** @client_port: The client-side port for the RPC. */
+ __be16 client_port;
+
+ /** @server_port: The server-side port for the RPC. */
+ __be16 server_port;
+} __packed;
+
+/* struct data_header - Contains data for part or all of a Homa message.
+ * An incoming packet consists of a data_header followed by message data.
+ * An outgoing packet can have this simple format as well, or it can be
+ * structured as a GSO packet. GSO packets look like this:
+ *
+ * No hijacking:
+ *
+ * |-----------------------|
+ * | |
+ * | data_header |
+ * | |
+ * |---------------------- |
+ * | |
+ * | |
+ * | segment data |
+ * | |
+ * | |
+ * |-----------------------|
+ * | seg_header |
+ * |-----------------------|
+ * | |
+ * | |
+ * | segment data |
+ * | |
+ * | |
+ * |-----------------------|
+ * | seg_header |
+ * |-----------------------|
+ * | |
+ * | |
+ * | segment data |
+ * | |
+ * | |
+ * |-----------------------|
+ *
+ * TSO will not adjust @common.sequence in the segments, so Homa sprinkles
+ * correct offsets (in seg_headers) throughout the segment data; TSO/GSO
+ * will include a different seg_header in each generated packet.
+ */
+
+struct seg_header {
+ /**
+ * @offset: Offset within message of the first byte of data in
+ * this segment.
+ */
+ __be32 offset;
+} __packed;
+
+struct data_header {
+ struct common_header common;
+
+ /** @message_length: Total #bytes in the message. */
+ __be32 message_length;
+
+ /**
+ * @incoming: The receiver can expect the sender to send all of the
+ * bytes in the message up to at least this offset (exclusive),
+ * even without additional grants. This includes unscheduled
+ * bytes, granted bytes, plus any additional bytes the sender
+ * transmits unilaterally (e.g., to round up to a full GSO batch).
+ */
+ __be32 incoming;
+
+ /** @ack: If the @client_id field of this is nonzero, provides info
+ * about an RPC that the recipient can now safely free. Note: in
+ * TSO packets this will get duplicated in each of the segments;
+ * in order to avoid repeated attempts to ack the same RPC,
+ * homa_gro_receive will clear this field in all segments but the
+ * first.
+ */
+ struct homa_ack ack;
+
+ __be16 dummy;
+
+ /**
+ * @retransmit: 1 means this packet was sent in response to a RESEND
+ * (it has already been sent previously).
+ */
+ __u8 retransmit;
+
+ __u8 pad;
+
+ /** @seg: First of possibly many segments. */
+ struct seg_header seg;
+} __packed;
+_Static_assert(sizeof(struct data_header) <= HOMA_MAX_HEADER,
+ "data_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER");
+_Static_assert(sizeof(struct data_header) >= HOMA_MIN_PKT_LENGTH,
+ "data_header too small: Homa doesn't currently have codeto pad data packets");
+_Static_assert(((sizeof(struct data_header) - sizeof(struct seg_header)) & 0x3) == 0,
+ " data_header length not a multiple of 4 bytes (required for TCP/TSO compatibility");
+
+/**
+ * homa_data_len() - Returns the total number of bytes in a DATA packet
+ * after the data_header. Note: if the packet is a GSO packet, the result
+ * may include metadata as well as packet data.
+ */
+static inline int homa_data_len(struct sk_buff *skb)
+{
+ return skb->len - skb_transport_offset(skb) - sizeof(struct data_header);
+}
+
+/**
+ * struct resend_header - Wire format for RESEND packets.
+ *
+ * A RESEND is sent by the receiver when it believes that message data may
+ * have been lost in transmission (or if it is concerned that the sender may
+ * have crashed). The receiver should resend the specified portion of the
+ * message, even if it already sent it previously.
+ */
+struct resend_header {
+ /** @common: Fields common to all packet types. */
+ struct common_header common;
+
+ /**
+ * @offset: Offset within the message of the first byte of data that
+ * should be retransmitted.
+ */
+ __be32 offset;
+
+ /**
+ * @length: Number of bytes of data to retransmit; this could specify
+ * a range longer than the total message size. Zero is a special case
+ * used by servers; in this case, there is no need to actually resend
+ * anything; the purpose of this packet is to trigger an UNKNOWN
+ * response if the client no longer cares about this RPC.
+ */
+ __be32 length;
+} __packed;
+_Static_assert(sizeof(struct resend_header) <= HOMA_MAX_HEADER,
+ "resend_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER");
+
+/**
+ * struct unknown_header - Wire format for UNKNOWN packets.
+ *
+ * An UNKNOWN packet is sent by either server or client when it receives a
+ * packet for an RPC that is unknown to it. When a client receives an
+ * UNKNOWN packet it will typically restart the RPC from the beginning;
+ * when a server receives an UNKNOWN packet it will typically discard its
+ * state for the RPC.
+ */
+struct unknown_header {
+ /** @common: Fields common to all packet types. */
+ struct common_header common;
+} __packed;
+_Static_assert(sizeof(struct unknown_header) <= HOMA_MAX_HEADER,
+ "unknown_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER");
+
+/**
+ * struct busy_header - Wire format for BUSY packets.
+ *
+ * These packets tell the recipient that the sender is still alive (even if
+ * it isn't sending data expected by the recipient).
+ */
+struct busy_header {
+ /** @common: Fields common to all packet types. */
+ struct common_header common;
+} __packed;
+_Static_assert(sizeof(struct busy_header) <= HOMA_MAX_HEADER,
+ "busy_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER");
+
+/**
+ * struct need_ack_header - Wire format for NEED_ACK packets.
+ *
+ * These packets ask the recipient (a client) to return an ACK message if
+ * the packet's RPC is no longer active.
+ */
+struct need_ack_header {
+ /** @common: Fields common to all packet types. */
+ struct common_header common;
+} __packed;
+_Static_assert(sizeof(struct need_ack_header) <= HOMA_MAX_HEADER,
+ "need_ack_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER");
+
+/**
+ * struct ack_header - Wire format for ACK packets.
+ *
+ * These packets are sent from a client to a server to indicate that
+ * a set of RPCs is no longer active on the client, so the server can
+ * free any state it may have for them.
+ */
+struct ack_header {
+ /** @common: Fields common to all packet types. */
+ struct common_header common;
+
+ /** @num_acks: number of (leading) elements in @acks that are valid. */
+ __be16 num_acks;
+
+#define HOMA_MAX_ACKS_PER_PKT 5
+ struct homa_ack acks[HOMA_MAX_ACKS_PER_PKT];
+} __packed;
+_Static_assert(sizeof(struct ack_header) <= HOMA_MAX_HEADER,
+ "ack_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER");
+
+/**
+ * homa_local_id(): given an RPC identifier from an input packet (which
+ * is network-encoded), return the decoded id we should use for that
+ * RPC on this machine.
+ * @sender_id: RPC id from an incoming packet, such as h->common.sender_id
+ */
+static inline __u64 homa_local_id(__be64 sender_id)
+{
+ /* If the client bit was set on the sender side, it needs to be
+ * removed here, and conversely.
+ */
+ return be64_to_cpu(sender_id) ^ 1;
+}
+
+#endif /* _HOMA_WIRE_H */
--
2.34.1
^ permalink raw reply related
* [PATCH net-next v2 01/12] net: homa: define user-visible API for Homa
From: John Ousterhout @ 2024-11-11 23:39 UTC (permalink / raw)
To: netdev, linux-api; +Cc: John Ousterhout
In-Reply-To: <20241111234006.5942-1-ouster@cs.stanford.edu>
Note: for man pages, see the Homa Wiki at:
https://homa-transport.atlassian.net/wiki/spaces/HOMA/overview
Signed-off-by: John Ousterhout <ouster@cs.stanford.edu>
---
include/uapi/linux/homa.h | 165 ++++++++++++++++++++++++++++++++++++++
1 file changed, 165 insertions(+)
create mode 100644 include/uapi/linux/homa.h
diff --git a/include/uapi/linux/homa.h b/include/uapi/linux/homa.h
new file mode 100644
index 000000000000..d06cfc49c101
--- /dev/null
+++ b/include/uapi/linux/homa.h
@@ -0,0 +1,165 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+
+/* This file defines the kernel call interface for the Homa
+ * transport protocol.
+ */
+
+#ifndef _UAPI_LINUX_HOMA_H
+#define _UAPI_LINUX_HOMA_H
+
+#include <linux/types.h>
+#ifndef __KERNEL__
+#include <netinet/in.h>
+#include <sys/socket.h>
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* IANA-assigned Internet Protocol number for Homa. */
+#define IPPROTO_HOMA 146
+
+/**
+ * define HOMA_MAX_MESSAGE_LENGTH - Maximum bytes of payload in a Homa
+ * request or response message.
+ */
+#define HOMA_MAX_MESSAGE_LENGTH 1000000
+
+/**
+ * define HOMA_BPAGE_SIZE - Number of bytes in pages used for receive
+ * buffers. Must be power of two.
+ */
+#define HOMA_BPAGE_SHIFT 16
+#define HOMA_BPAGE_SIZE (1 << HOMA_BPAGE_SHIFT)
+
+/**
+ * define HOMA_MAX_BPAGES: The largest number of bpages that will be required
+ * to store an incoming message.
+ */
+#define HOMA_MAX_BPAGES ((HOMA_MAX_MESSAGE_LENGTH + HOMA_BPAGE_SIZE - 1) \
+ >> HOMA_BPAGE_SHIFT)
+
+/**
+ * define HOMA_MIN_DEFAULT_PORT - The 16-bit port space is divided into
+ * two nonoverlapping regions. Ports 1-32767 are reserved exclusively
+ * for well-defined server ports. The remaining ports are used for client
+ * ports; these are allocated automatically by Homa. Port 0 is reserved.
+ */
+#define HOMA_MIN_DEFAULT_PORT 0x8000
+
+/**
+ * struct homa_sendmsg_args - Provides information needed by Homa's
+ * sendmsg; passed to sendmsg using the msg_control field.
+ */
+struct homa_sendmsg_args {
+ /**
+ * @id: (in/out) An initial value of 0 means a new request is
+ * being sent; nonzero means the message is a reply to the given
+ * id. If the message is a request, then the value is modified to
+ * hold the id of the new RPC.
+ */
+ uint64_t id;
+
+ /**
+ * @completion_cookie: (in) Used only for request messages; will be
+ * returned by recvmsg when the RPC completes. Typically used to
+ * locate app-specific info about the RPC.
+ */
+ uint64_t completion_cookie;
+};
+
+#if !defined(__cplusplus)
+_Static_assert(sizeof(struct homa_sendmsg_args) >= 16,
+ "homa_sendmsg_args shrunk");
+_Static_assert(sizeof(struct homa_sendmsg_args) <= 16,
+ "homa_sendmsg_args grew");
+#endif
+
+/**
+ * struct homa_recvmsg_args - Provides information needed by Homa's
+ * recvmsg; passed to recvmsg using the msg_control field.
+ */
+struct homa_recvmsg_args {
+ /**
+ * @id: (in/out) Initially specifies the id of the desired RPC, or 0
+ * if any RPC is OK; returns the actual id received.
+ */
+ uint64_t id;
+
+ /**
+ * @completion_cookie: (out) If the incoming message is a response,
+ * this will return the completion cookie specified when the
+ * request was sent. For requests this will always be zero.
+ */
+ uint64_t completion_cookie;
+
+ /**
+ * @flags: (in) OR-ed combination of bits that control the operation.
+ * See below for values.
+ */
+ uint32_t flags;
+
+ /**
+ * @num_bpages: (in/out) Number of valid entries in @bpage_offsets.
+ * Passes in bpages from previous messages that can now be
+ * recycled; returns bpages from the new message.
+ */
+ uint32_t num_bpages;
+
+ /**
+ * @bpage_offsets: (in/out) Each entry is an offset into the buffer
+ * region for the socket pool. When returned from recvmsg, the
+ * offsets indicate where fragments of the new message are stored. All
+ * entries but the last refer to full buffer pages (HOMA_BPAGE_SIZE bytes)
+ * and are bpage-aligned. The last entry may refer to a bpage fragment and
+ * is not necessarily aligned. The application now owns these bpages and
+ * must eventually return them to Homa, using bpage_offsets in a future
+ * recvmsg invocation.
+ */
+ uint32_t bpage_offsets[HOMA_MAX_BPAGES];
+};
+
+#if !defined(__cplusplus)
+_Static_assert(sizeof(struct homa_recvmsg_args) >= 88,
+ "homa_recvmsg_args shrunk");
+_Static_assert(sizeof(struct homa_recvmsg_args) <= 88,
+ "homa_recvmsg_args grew");
+#endif
+
+/* Flag bits for homa_recvmsg_args.flags (see man page for documentation):
+ */
+#define HOMA_RECVMSG_REQUEST 0x01
+#define HOMA_RECVMSG_RESPONSE 0x02
+#define HOMA_RECVMSG_NONBLOCKING 0x04
+#define HOMA_RECVMSG_VALID_FLAGS 0x07
+
+/** define SO_HOMA_SET_BUF: setsockopt option for specifying buffer region. */
+#define SO_HOMA_SET_BUF 10
+
+/** struct homa_set_buf - setsockopt argument for SO_HOMA_SET_BUF. */
+struct homa_set_buf_args {
+ /** @start: First byte of buffer region. */
+ void *start;
+
+ /** @length: Total number of bytes available at @start. */
+ size_t length;
+};
+
+/**
+ * Meanings of the bits in Homa's flag word, which can be set using
+ * "sysctl /net/homa/flags".
+ */
+
+/**
+ * Disable the output throttling mechanism: always send all packets
+ * immediately.
+ */
+#define HOMA_FLAG_DONT_THROTTLE 2
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _UAPI_LINUX_HOMA_H */
--
2.34.1
^ permalink raw reply related
* [PATCH net-next v2 04/12] net: homa: create homa_pool.h and homa_pool.c
From: John Ousterhout @ 2024-11-11 23:39 UTC (permalink / raw)
To: netdev, linux-api; +Cc: John Ousterhout
In-Reply-To: <20241111234006.5942-1-ouster@cs.stanford.edu>
These files implement Homa's mechanism for managing application-level
buffer space for incoming messages This mechanism is needed to allow
Homa to copy data out to user space in parallel with receiving packets;
it was discussed in a talk at NetDev 0x17.
Signed-off-by: John Ousterhout <ouster@cs.stanford.edu>
---
net/homa/homa_pool.c | 420 +++++++++++++++++++++++++++++++++++++++++++
net/homa/homa_pool.h | 152 ++++++++++++++++
2 files changed, 572 insertions(+)
create mode 100644 net/homa/homa_pool.c
create mode 100644 net/homa/homa_pool.h
diff --git a/net/homa/homa_pool.c b/net/homa/homa_pool.c
new file mode 100644
index 000000000000..4d166eb4ac19
--- /dev/null
+++ b/net/homa/homa_pool.c
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: BSD-2-Clause
+
+#include "homa_impl.h"
+#include "homa_pool.h"
+
+/* This file contains functions that manage user-space buffer pools. */
+
+/* Pools must always have at least this many bpages (no particular
+ * reasoning behind this value).
+ */
+#define MIN_POOL_SIZE 2
+
+/* Used when determining how many bpages to consider for allocation. */
+#define MIN_EXTRA 4
+
+/**
+ * set_bpages_needed() - Set the bpages_needed field of @pool based
+ * on the length of the first RPC that's waiting for buffer space.
+ * The caller must own the lock for @pool->hsk.
+ * @pool: Pool to update.
+ */
+static void set_bpages_needed(struct homa_pool *pool)
+{
+ struct homa_rpc *rpc = list_first_entry(&pool->hsk->waiting_for_bufs,
+ struct homa_rpc, buf_links);
+ pool->bpages_needed = (rpc->msgin.length + HOMA_BPAGE_SIZE - 1)
+ >> HOMA_BPAGE_SHIFT;
+}
+
+/**
+ * homa_pool_init() - Initialize a homa_pool; any previous contents of the
+ * objects are overwritten.
+ * @hsk: Socket containing the pool to initialize.
+ * @region: First byte of the memory region for the pool, allocated
+ * by the application; must be page-aligned.
+ * @region_size: Total number of bytes available at @buf_region.
+ * Return: Either zero (for success) or a negative errno for failure.
+ */
+int homa_pool_init(struct homa_sock *hsk, void __user *region, __u64 region_size)
+{
+ struct homa_pool *pool = hsk->buffer_pool;
+ int i, result;
+
+ if (((uintptr_t)region) & ~PAGE_MASK)
+ return -EINVAL;
+ pool->hsk = hsk;
+ pool->region = (char __user *)region;
+ pool->num_bpages = region_size >> HOMA_BPAGE_SHIFT;
+ pool->descriptors = NULL;
+ pool->cores = NULL;
+ if (pool->num_bpages < MIN_POOL_SIZE) {
+ result = -EINVAL;
+ goto error;
+ }
+ pool->descriptors = kmalloc_array(pool->num_bpages,
+ sizeof(struct homa_bpage), GFP_ATOMIC);
+ if (!pool->descriptors) {
+ result = -ENOMEM;
+ goto error;
+ }
+ for (i = 0; i < pool->num_bpages; i++) {
+ struct homa_bpage *bp = &pool->descriptors[i];
+
+ spin_lock_init(&bp->lock);
+ atomic_set(&bp->refs, 0);
+ bp->owner = -1;
+ bp->expiration = 0;
+ }
+ atomic_set(&pool->free_bpages, pool->num_bpages);
+ pool->bpages_needed = INT_MAX;
+
+ /* Allocate and initialize core-specific data. */
+ pool->cores = kmalloc_array(nr_cpu_ids, sizeof(struct homa_pool_core),
+ GFP_ATOMIC);
+ if (!pool->cores) {
+ result = -ENOMEM;
+ goto error;
+ }
+ pool->num_cores = nr_cpu_ids;
+ for (i = 0; i < pool->num_cores; i++) {
+ pool->cores[i].page_hint = 0;
+ pool->cores[i].allocated = 0;
+ pool->cores[i].next_candidate = 0;
+ }
+ pool->check_waiting_invoked = 0;
+
+ return 0;
+
+error:
+ kfree(pool->descriptors);
+ kfree(pool->cores);
+ pool->region = NULL;
+ return result;
+}
+
+/**
+ * homa_pool_destroy() - Destructor for homa_pool. After this method
+ * returns, the object should not be used unless it has been reinitialized.
+ * @pool: Pool to destroy.
+ */
+void homa_pool_destroy(struct homa_pool *pool)
+{
+ if (!pool->region)
+ return;
+ kfree(pool->descriptors);
+ kfree(pool->cores);
+ pool->region = NULL;
+}
+
+/**
+ * homa_pool_get_pages() - Allocate one or more full pages from the pool.
+ * @pool: Pool from which to allocate pages
+ * @num_pages: Number of pages needed
+ * @pages: The indices of the allocated pages are stored here; caller
+ * must ensure this array is big enough. Reference counts have
+ * been set to 1 on all of these pages (or 2 if set_owner
+ * was specified).
+ * @set_owner: If nonzero, the current core is marked as owner of all
+ * of the allocated pages (and the expiration time is also
+ * set). Otherwise the pages are left unowned.
+ * Return: 0 for success, -1 if there wasn't enough free space in the pool.
+ */
+int homa_pool_get_pages(struct homa_pool *pool, int num_pages, __u32 *pages,
+ int set_owner)
+{
+ int core_num = raw_smp_processor_id();
+ struct homa_pool_core *core;
+ __u64 now = sched_clock();
+ int alloced = 0;
+ int limit = 0;
+
+ core = &pool->cores[core_num];
+ if (atomic_sub_return(num_pages, &pool->free_bpages) < 0) {
+ atomic_add(num_pages, &pool->free_bpages);
+ return -1;
+ }
+
+ /* Once we get to this point we know we will be able to find
+ * enough free pages; now we just have to find them.
+ */
+ while (alloced != num_pages) {
+ struct homa_bpage *bpage;
+ int cur, ref_count;
+
+ /* If we don't need to use all of the bpages in the pool,
+ * then try to use only the ones with low indexes. This
+ * will reduce the cache footprint for the pool by reusing
+ * a few bpages over and over. Specifically this code will
+ * not consider any candidate page whose index is >= limit.
+ * Limit is chosen to make sure there are a reasonable
+ * number of free pages in the range, so we won't have to
+ * check a huge number of pages.
+ */
+ if (limit == 0) {
+ int extra;
+
+ limit = pool->num_bpages
+ - atomic_read(&pool->free_bpages);
+ extra = limit >> 2;
+ limit += (extra < MIN_EXTRA) ? MIN_EXTRA : extra;
+ if (limit > pool->num_bpages)
+ limit = pool->num_bpages;
+ }
+
+ cur = core->next_candidate;
+ core->next_candidate++;
+ if (cur >= limit) {
+ core->next_candidate = 0;
+
+ /* Must recompute the limit for each new loop through
+ * the bpage array: we may need to consider a larger
+ * range of pages because of concurrent allocations.
+ */
+ limit = 0;
+ continue;
+ }
+ bpage = &pool->descriptors[cur];
+
+ /* Figure out whether this candidate is free (or can be
+ * stolen). Do a quick check without locking the page, and
+ * if the page looks promising, then lock it and check again
+ * (must check again in case someone else snuck in and
+ * grabbed the page).
+ */
+ ref_count = atomic_read(&bpage->refs);
+ if (ref_count >= 2 || (ref_count == 1 && (bpage->owner < 0 ||
+ bpage->expiration > now)))
+ continue;
+ if (!spin_trylock_bh(&bpage->lock))
+ continue;
+ ref_count = atomic_read(&bpage->refs);
+ if (ref_count >= 2 || (ref_count == 1 && (bpage->owner < 0 ||
+ bpage->expiration > now))) {
+ spin_unlock_bh(&bpage->lock);
+ continue;
+ }
+ if (bpage->owner >= 0)
+ atomic_inc(&pool->free_bpages);
+ if (set_owner) {
+ atomic_set(&bpage->refs, 2);
+ bpage->owner = core_num;
+ bpage->expiration = now +
+ 1000 * pool->hsk->homa->bpage_lease_usecs;
+ } else {
+ atomic_set(&bpage->refs, 1);
+ bpage->owner = -1;
+ }
+ spin_unlock_bh(&bpage->lock);
+ pages[alloced] = cur;
+ alloced++;
+ }
+ return 0;
+}
+
+/**
+ * homa_pool_allocate() - Allocate buffer space for an RPC.
+ * @rpc: RPC that needs space allocated for its incoming message (space must
+ * not already have been allocated). The fields @msgin->num_buffers
+ * and @msgin->buffers are filled in. Must be locked by caller.
+ * Return: The return value is normally 0, which means either buffer space
+ * was allocated or the @rpc was queued on @hsk->waiting. If a fatal error
+ * occurred, such as no buffer pool present, then a negative errno is
+ * returned.
+ */
+int homa_pool_allocate(struct homa_rpc *rpc)
+{
+ struct homa_pool *pool = rpc->hsk->buffer_pool;
+ int full_pages, partial, i, core_id;
+ __u32 pages[HOMA_MAX_BPAGES];
+ struct homa_pool_core *core;
+ struct homa_bpage *bpage;
+ struct homa_rpc *other;
+
+ if (!pool->region)
+ return -ENOMEM;
+
+ /* First allocate any full bpages that are needed. */
+ full_pages = rpc->msgin.length >> HOMA_BPAGE_SHIFT;
+ if (unlikely(full_pages)) {
+ if (homa_pool_get_pages(pool, full_pages, pages, 0) != 0)
+ goto out_of_space;
+ for (i = 0; i < full_pages; i++)
+ rpc->msgin.bpage_offsets[i] = pages[i] << HOMA_BPAGE_SHIFT;
+ }
+ rpc->msgin.num_bpages = full_pages;
+
+ /* The last chunk may be less than a full bpage; for this we use
+ * the bpage that we own (and reuse it for multiple messages).
+ */
+ partial = rpc->msgin.length & (HOMA_BPAGE_SIZE - 1);
+ if (unlikely(partial == 0))
+ goto success;
+ core_id = raw_smp_processor_id();
+ core = &pool->cores[core_id];
+ bpage = &pool->descriptors[core->page_hint];
+ if (!spin_trylock_bh(&bpage->lock))
+ spin_lock_bh(&bpage->lock);
+ if (bpage->owner != core_id) {
+ spin_unlock_bh(&bpage->lock);
+ goto new_page;
+ }
+ if ((core->allocated + partial) > HOMA_BPAGE_SIZE) {
+ if (atomic_read(&bpage->refs) == 1) {
+ /* Bpage is totally free, so we can reuse it. */
+ core->allocated = 0;
+ } else {
+ bpage->owner = -1;
+
+ /* We know the reference count can't reach zero here
+ * because of check above, so we won't have to decrement
+ * pool->free_bpages.
+ */
+ atomic_dec_return(&bpage->refs);
+ spin_unlock_bh(&bpage->lock);
+ goto new_page;
+ }
+ }
+ bpage->expiration = sched_clock() +
+ 1000 * pool->hsk->homa->bpage_lease_usecs;
+ atomic_inc(&bpage->refs);
+ spin_unlock_bh(&bpage->lock);
+ goto allocate_partial;
+
+ /* Can't use the current page; get another one. */
+new_page:
+ if (homa_pool_get_pages(pool, 1, pages, 1) != 0) {
+ homa_pool_release_buffers(pool, rpc->msgin.num_bpages,
+ rpc->msgin.bpage_offsets);
+ rpc->msgin.num_bpages = 0;
+ goto out_of_space;
+ }
+ core->page_hint = pages[0];
+ core->allocated = 0;
+
+allocate_partial:
+ rpc->msgin.bpage_offsets[rpc->msgin.num_bpages] = core->allocated
+ + (core->page_hint << HOMA_BPAGE_SHIFT);
+ rpc->msgin.num_bpages++;
+ core->allocated += partial;
+
+success:
+ return 0;
+
+ /* We get here if there wasn't enough buffer space for this
+ * message; add the RPC to hsk->waiting_for_bufs.
+ */
+out_of_space:
+ homa_sock_lock(pool->hsk, "homa_pool_allocate");
+ list_for_each_entry(other, &pool->hsk->waiting_for_bufs, buf_links) {
+ if (other->msgin.length > rpc->msgin.length) {
+ list_add_tail(&rpc->buf_links, &other->buf_links);
+ goto queued;
+ }
+ }
+ list_add_tail_rcu(&rpc->buf_links, &pool->hsk->waiting_for_bufs);
+
+queued:
+ set_bpages_needed(pool);
+ homa_sock_unlock(pool->hsk);
+ return 0;
+}
+
+/**
+ * homa_pool_get_buffer() - Given an RPC, figure out where to store incoming
+ * message data.
+ * @rpc: RPC for which incoming message data is being processed; its
+ * msgin must be properly initialized and buffer space must have
+ * been allocated for the message.
+ * @offset: Offset within @rpc's incoming message.
+ * @available: Will be filled in with the number of bytes of space available
+ * at the returned address.
+ * Return: The application's virtual address for buffer space corresponding
+ * to @offset in the incoming message for @rpc.
+ */
+void __user *homa_pool_get_buffer(struct homa_rpc *rpc, int offset,
+ int *available)
+{
+ int bpage_index, bpage_offset;
+
+ bpage_index = offset >> HOMA_BPAGE_SHIFT;
+ BUG_ON(bpage_index >= rpc->msgin.num_bpages);
+ bpage_offset = offset & (HOMA_BPAGE_SIZE - 1);
+ *available = (bpage_index < (rpc->msgin.num_bpages - 1))
+ ? HOMA_BPAGE_SIZE - bpage_offset
+ : rpc->msgin.length - offset;
+ return rpc->hsk->buffer_pool->region + rpc->msgin.bpage_offsets[bpage_index]
+ + bpage_offset;
+}
+
+/**
+ * homa_pool_release_buffers() - Release buffer space so that it can be
+ * reused.
+ * @pool: Pool that the buffer space belongs to. Doesn't need to
+ * be locked.
+ * @num_buffers: How many buffers to release.
+ * @buffers: Points to @num_buffers values, each of which is an offset
+ * from the start of the pool to the buffer to be released.
+ */
+void homa_pool_release_buffers(struct homa_pool *pool, int num_buffers,
+ __u32 *buffers)
+{
+ int i;
+
+ if (!pool->region)
+ return;
+ for (i = 0; i < num_buffers; i++) {
+ __u32 bpage_index = buffers[i] >> HOMA_BPAGE_SHIFT;
+ struct homa_bpage *bpage = &pool->descriptors[bpage_index];
+
+ if (bpage_index < pool->num_bpages)
+ if (atomic_dec_return(&bpage->refs) == 0)
+ atomic_inc(&pool->free_bpages);
+ }
+ }
+
+/**
+ * homa_pool_check_waiting() - Checks to see if there are enough free
+ * bpages to wake up any RPCs that were blocked. Whenever
+ * homa_pool_release_buffers is invoked, this function must be invoked later,
+ * at a point when the caller holds no locks (homa_pool_release_buffers may
+ * be invoked with locks held, so it can't safely invoke this function).
+ * This is regrettably tricky, but I can't think of a better solution.
+ * @pool: Information about the buffer pool.
+ */
+void homa_pool_check_waiting(struct homa_pool *pool)
+{
+ while (atomic_read(&pool->free_bpages) >= pool->bpages_needed) {
+ struct homa_rpc *rpc;
+
+ homa_sock_lock(pool->hsk, "buffer pool");
+ if (list_empty(&pool->hsk->waiting_for_bufs)) {
+ pool->bpages_needed = INT_MAX;
+ homa_sock_unlock(pool->hsk);
+ break;
+ }
+ rpc = list_first_entry(&pool->hsk->waiting_for_bufs,
+ struct homa_rpc, buf_links);
+ if (!homa_bucket_try_lock(rpc->bucket, rpc->id,
+ "homa_pool_check_waiting")) {
+ /* Can't just spin on the RPC lock because we're
+ * holding the socket lock (see sync.txt). Instead,
+ * release the socket lock and try the entire
+ * operation again.
+ */
+ homa_sock_unlock(pool->hsk);
+ continue;
+ }
+ list_del_init(&rpc->buf_links);
+ if (list_empty(&pool->hsk->waiting_for_bufs))
+ pool->bpages_needed = INT_MAX;
+ else
+ set_bpages_needed(pool);
+ homa_sock_unlock(pool->hsk);
+ homa_pool_allocate(rpc);
+ if (rpc->msgin.num_bpages > 0)
+ /* Allocation succeeded; "wake up" the RPC. */
+ rpc->msgin.resend_all = 1;
+ homa_rpc_unlock(rpc);
+ }
+}
diff --git a/net/homa/homa_pool.h b/net/homa/homa_pool.h
new file mode 100644
index 000000000000..609d99b59707
--- /dev/null
+++ b/net/homa/homa_pool.h
@@ -0,0 +1,152 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+
+/* This file contains definitions used to manage user-space buffer pools.
+ */
+
+#ifndef _HOMA_POOL_H
+#define _HOMA_POOL_H
+
+#include "homa_rpc.h"
+
+/**
+ * struct homa_bpage - Contains information about a single page in
+ * a buffer pool.
+ */
+struct homa_bpage {
+ union {
+ /**
+ * @cache_line: Ensures that each homa_bpage object
+ * is exactly one cache line long.
+ */
+ char cache_line[L1_CACHE_BYTES];
+ struct {
+ /** @lock: to synchronize shared access. */
+ spinlock_t lock;
+
+ /**
+ * @refs: Counts number of distinct uses of this
+ * bpage (1 tick for each message that is using
+ * this page, plus an additional tick if the @owner
+ * field is set).
+ */
+ atomic_t refs;
+
+ /**
+ * @owner: kernel core that currently owns this page
+ * (< 0 if none).
+ */
+ int owner;
+
+ /**
+ * @expiration: time (in sched_clock() units) after
+ * which it's OK to steal this page from its current
+ * owner (if @refs is 1).
+ */
+ __u64 expiration;
+ };
+ };
+};
+
+/**
+ * struct homa_pool_core - Holds core-specific data for a homa_pool (a bpage
+ * out of which that core is allocating small chunks).
+ */
+struct homa_pool_core {
+ union {
+ /**
+ * @cache_line: Ensures that each object is exactly one
+ * cache line long.
+ */
+ char cache_line[L1_CACHE_BYTES];
+ struct {
+ /**
+ * @page_hint: Index of bpage in pool->descriptors,
+ * which may be owned by this core. If so, we'll use it
+ * for allocating partial pages.
+ */
+ int page_hint;
+
+ /**
+ * @allocated: if the page given by @page_hint is
+ * owned by this core, this variable gives the number of
+ * (initial) bytes that have already been allocated
+ * from the page.
+ */
+ int allocated;
+
+ /**
+ * @next_candidate: when searching for free bpages,
+ * check this index next.
+ */
+ int next_candidate;
+ };
+ };
+};
+
+/**
+ * struct homa_pool - Describes a pool of buffer space for incoming
+ * messages for a particular socket; managed by homa_pool.c. The pool is
+ * divided up into "bpages", which are a multiple of the hardware page size.
+ * A bpage may be owned by a particular core so that it can more efficiently
+ * allocate space for small messages.
+ */
+struct homa_pool {
+ /**
+ * @hsk: the socket that this pool belongs to.
+ */
+ struct homa_sock *hsk;
+
+ /**
+ * @region: beginning of the pool's region (in the app's virtual
+ * memory). Divided into bpages. 0 means the pool hasn't yet been
+ * initialized.
+ */
+ char __user *region;
+
+ /** @num_bpages: total number of bpages in the pool. */
+ int num_bpages;
+
+ /** @descriptors: kmalloced area containing one entry for each bpage. */
+ struct homa_bpage *descriptors;
+
+ /**
+ * @free_bpages: the number of pages still available for allocation
+ * by homa_pool_get pages. This equals the number of pages with zero
+ * reference counts, minus the number of pages that have been claimed
+ * by homa_get_pool_pages but not yet allocated.
+ */
+ atomic_t free_bpages;
+
+ /**
+ * The number of free bpages required to satisfy the needs of the
+ * first RPC on @hsk->waiting_for_bufs, or INT_MAX if that queue
+ * is empty.
+ */
+ int bpages_needed;
+
+ /** @cores: core-specific info; dynamically allocated. */
+ struct homa_pool_core *cores;
+
+ /** @num_cores: number of elements in @cores. */
+ int num_cores;
+
+ /**
+ * @check_waiting_invoked: incremented during unit tests when
+ * homa_pool_check_waiting is invoked.
+ */
+ int check_waiting_invoked;
+};
+
+int homa_pool_allocate(struct homa_rpc *rpc);
+void homa_pool_check_waiting(struct homa_pool *pool);
+void homa_pool_destroy(struct homa_pool *pool);
+void __user *homa_pool_get_buffer(struct homa_rpc *rpc, int offset,
+ int *available);
+int homa_pool_get_pages(struct homa_pool *pool, int num_pages,
+ __u32 *pages, int leave_locked);
+int homa_pool_init(struct homa_sock *hsk, void __user *buf_region,
+ __u64 region_size);
+void homa_pool_release_buffers(struct homa_pool *pool,
+ int num_buffers, __u32 *buffers);
+
+#endif /* _HOMA_POOL_H */
--
2.34.1
^ permalink raw reply related
* [PATCH net-next v2 00/12] Begin upstreaming Homa transport protocol
From: John Ousterhout @ 2024-11-11 23:39 UTC (permalink / raw)
To: netdev, linux-api; +Cc: John Ousterhout
This patch series begins the process of upstreaming the Homa transport
protocol. Homa is an alternative to TCP for use in datacenter
environments. It provides 10-100x reductions in tail latency for short
messages relative to TCP. Its benefits are greatest for mixed workloads
containing both short and long messages running under high network loads.
Homa is not API-compatible with TCP: it is connectionless and message-
oriented (but still reliable and flow-controlled). Homa's new API not
only contributes to its performance gains, but it also eliminates the
massive amount of connection state required by TCP for highly connected
datacenter workloads.
For more details on Homa, please consult the Homa Wiki:
https://homa-transport.atlassian.net/wiki/spaces/HOMA/overview
The Wiki has pointers to two papers on Homa (one of which describes
this implementation) as well as man pages describing the application
API and other information.
There is also a GitHub repo for Homa:
https://github.com/PlatformLab/HomaModule
The GitHub repo contains a superset of this patch set, including:
* Additional source code that will eventually be upstreamed
* Extensive unit tests (which will also be upstreamed eventually)
* Application-level library functions (which need to go in glibc?)
* Man pages (which need to be upstreamed as well)
* Benchmarking and instrumentation code
For this patch series, Homa has been stripped down to the bare minimum
functionality capable of actually executing remote procedure calls. (about
7500 lines of source code, compared to 15000 in the complete Homa). The
remaining code will be upstreamed in smaller batches once this patch
series has been accepted. Note: the code in this patch series is
functional but its performance is not very interesting (about the same
as TCP).
The patch series is arranged to introduce the major functional components
of Homa. Until the last patch has been applied, the code is inert (it
will not be compiled).
Note: this implementation of Homa supports both IPv4 and IPv6.
John Ousterhout (12):
net: homa: define user-visible API for Homa
net: homa: define Homa packet formats
net: homa: create shared Homa header files
net: homa: create homa_pool.h and homa_pool.c
net: homa: create homa_rpc.h and homa_rpc.c
net: homa: create homa_peer.h and homa_peer.c
net: homa: create homa_sock.h and homa_sock.c
net: homa: create homa_incoming.c
net: homa: create homa_outgoing.c
net: homa: create homa_timer.c
net: homa: create homa_plumbing.c homa_utils.c
net: homa: create Makefile and Kconfig
MAINTAINERS | 7 +
include/uapi/linux/homa.h | 165 ++++++
net/Kconfig | 1 +
net/Makefile | 1 +
net/homa/Kconfig | 19 +
net/homa/Makefile | 14 +
net/homa/homa_impl.h | 767 ++++++++++++++++++++++++++
net/homa/homa_incoming.c | 1076 +++++++++++++++++++++++++++++++++++++
net/homa/homa_outgoing.c | 854 +++++++++++++++++++++++++++++
net/homa/homa_peer.c | 319 +++++++++++
net/homa/homa_peer.h | 234 ++++++++
net/homa/homa_plumbing.c | 965 +++++++++++++++++++++++++++++++++
net/homa/homa_pool.c | 420 +++++++++++++++
net/homa/homa_pool.h | 152 ++++++
net/homa/homa_rpc.c | 488 +++++++++++++++++
net/homa/homa_rpc.h | 446 +++++++++++++++
net/homa/homa_sock.c | 380 +++++++++++++
net/homa/homa_sock.h | 426 +++++++++++++++
net/homa/homa_stub.h | 80 +++
net/homa/homa_timer.c | 156 ++++++
net/homa/homa_utils.c | 150 ++++++
net/homa/homa_wire.h | 378 +++++++++++++
22 files changed, 7498 insertions(+)
create mode 100644 include/uapi/linux/homa.h
create mode 100644 net/homa/Kconfig
create mode 100644 net/homa/Makefile
create mode 100644 net/homa/homa_impl.h
create mode 100644 net/homa/homa_incoming.c
create mode 100644 net/homa/homa_outgoing.c
create mode 100644 net/homa/homa_peer.c
create mode 100644 net/homa/homa_peer.h
create mode 100644 net/homa/homa_plumbing.c
create mode 100644 net/homa/homa_pool.c
create mode 100644 net/homa/homa_pool.h
create mode 100644 net/homa/homa_rpc.c
create mode 100644 net/homa/homa_rpc.h
create mode 100644 net/homa/homa_sock.c
create mode 100644 net/homa/homa_sock.h
create mode 100644 net/homa/homa_stub.h
create mode 100644 net/homa/homa_timer.c
create mode 100644 net/homa/homa_utils.c
create mode 100644 net/homa/homa_wire.h
--
2.34.1
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: David Hildenbrand @ 2024-11-11 22:14 UTC (permalink / raw)
To: Vlastimil Babka, Paolo Bonzini, Matthew Wilcox, Shivank Garg
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
kees, bharata, nikunj, michael.day, Neeraj.Upadhyay, linux-coco
In-Reply-To: <ff3174f8-c8b5-4fae-a9d9-87546d37c162@suse.cz>
On 11.11.24 12:02, Vlastimil Babka wrote:
> On 11/8/24 18:31, Paolo Bonzini wrote:
>> On 11/7/24 16:10, Matthew Wilcox wrote:
>>> On Thu, Nov 07, 2024 at 02:24:20PM +0530, Shivank Garg wrote:
>>>> The folio allocation path from guest_memfd typically looks like this...
>>>>
>>>> kvm_gmem_get_folio
>>>> filemap_grab_folio
>>>> __filemap_get_folio
>>>> filemap_alloc_folio
>>>> __folio_alloc_node_noprof
>>>> -> goes to the buddy allocator
>>>>
>>>> Hence, I am trying to have a version of filemap_alloc_folio() that takes an mpol.
>>>
>>> It only takes that path if cpuset_do_page_mem_spread() is true. Is the
>>> real problem that you're trying to solve that cpusets are being used
>>> incorrectly?
>>
>> If it's false it's not very different, it goes to alloc_pages_noprof().
>> Then it respects the process's policy, but the policy is not
>> customizable without mucking with state that is global to the process.
>>
>> Taking a step back: the problem is that a VM can be configured to have
>> multiple guest-side NUMA nodes, each of which will pick memory from the
>> right NUMA node in the host. Without a per-file operation it's not
>> possible to do this on guest_memfd. The discussion was whether to use
>> ioctl() or a new system call. The discussion ended with the idea of
>> posting a *proposal* asking for *comments* as to whether the system call
>> would be useful in general beyond KVM.
>>
>> Commenting on the system call itself I am not sure I like the
>> file_operations entry, though I understand that it's the simplest way to
>> implement this in an RFC series. It's a bit surprising that fbind() is
>> a total no-op for everything except KVM's guest_memfd.
>>
>> Maybe whatever you pass to fbind() could be stored in the struct file *,
>> and used as the default when creating VMAs; as if every mmap() was
>> followed by an mbind(), except that it also does the right thing with
>> MAP_POPULATE for example. Or maybe that's a horrible idea?
>
> mbind() manpage has this:
>
> The specified policy will be ignored for any MAP_SHARED
> mappings in the specified memory range. Rather the pages will be allocated
> according to the memory policy of the thread that caused the page to be
> allocated. Again, this may not be the thread that called mbind().
I recall discussing that a couple of times in the context of QEMU. I
have some faint recollection that the manpage is a bit imprecise:
IIRC, hugetlb also ends up using the VMA policy for MAP_SHARED mappings
during faults (huge_node()->get_vma_policy()) -- but in contrast to
shmem, it doesn't end up becoming the "shared" policy for the file, used
when accessed through other VMAs.
>
> So that seems like we're not very keen on having one user of a file set a
> policy that would affect other users of the file?
For VMs in QEMU we really want to configure the policy once in the main
process and have all other processes (e.g., vhost-user) not worry about
that when they mmap() guest memory.
With shmem this works by "shared policy" design (below). For hugetlb, we
rely on the fact that mbind()+MADV_POPULATE_WRITE allows us to
preallocate NUMA-aware. So with hugetlb we really preallocate all guest
RAM to guarantee the NUMA placement.
It would not be the worst idea to have a clean interface to configure
file-range policies instead of having this weird shmem mbind() behavior
and the hugetlb hack.
Having that said, other filesystem are rarely used for backing VMs, at
least in combination with NUMA. So nobody really cared that much for now.
Maybe fbind() would primarily only be useful for in-memory filesystems
(shmem/hugetlb/guest_memfd).
>
> Now the next paragraph of the manpage says that shmem is different, and
> guest_memfd is more like shmem than a regular file.
>
> My conclusion from that is that fbind() might be too broad and we don't want
> this for actual filesystem-backed files? And if it's limited to guest_memfd,
> it shouldn't be an fbind()?
I was just once again diving into how mbind() on shmem is handled. And
in fact, mbind() will call vma->set_policy() to update the per
file-range policy. I wonder why we didn't do the same for hugetlb ...
but of course, hugetlb must be special in any possible way.
Not saying it's the best idea, but as we are talking about mmap support
of guest_memfd (only allowing to fault in shared/faultable pages), one
*could* look into implementing mbind()+vma->set_policy() for guest_memfd
similar to how shmem handles it.
It would require a (temporary) dummy VMA in the worst case (all private
memory).
It sounds a bit weird, though, to require a VMA to configure this,
though. But at least it's similar to what shmem does ...
--
Cheers,
David / dhildenb
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: Vlastimil Babka @ 2024-11-11 11:02 UTC (permalink / raw)
To: Paolo Bonzini, Matthew Wilcox, Shivank Garg
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
kees, bharata, nikunj, michael.day, Neeraj.Upadhyay, linux-coco
In-Reply-To: <10ffac79-0dba-4c30-991e-f3ca2b5ff639@redhat.com>
On 11/8/24 18:31, Paolo Bonzini wrote:
> On 11/7/24 16:10, Matthew Wilcox wrote:
>> On Thu, Nov 07, 2024 at 02:24:20PM +0530, Shivank Garg wrote:
>>> The folio allocation path from guest_memfd typically looks like this...
>>>
>>> kvm_gmem_get_folio
>>> filemap_grab_folio
>>> __filemap_get_folio
>>> filemap_alloc_folio
>>> __folio_alloc_node_noprof
>>> -> goes to the buddy allocator
>>>
>>> Hence, I am trying to have a version of filemap_alloc_folio() that takes an mpol.
>>
>> It only takes that path if cpuset_do_page_mem_spread() is true. Is the
>> real problem that you're trying to solve that cpusets are being used
>> incorrectly?
>
> If it's false it's not very different, it goes to alloc_pages_noprof().
> Then it respects the process's policy, but the policy is not
> customizable without mucking with state that is global to the process.
>
> Taking a step back: the problem is that a VM can be configured to have
> multiple guest-side NUMA nodes, each of which will pick memory from the
> right NUMA node in the host. Without a per-file operation it's not
> possible to do this on guest_memfd. The discussion was whether to use
> ioctl() or a new system call. The discussion ended with the idea of
> posting a *proposal* asking for *comments* as to whether the system call
> would be useful in general beyond KVM.
>
> Commenting on the system call itself I am not sure I like the
> file_operations entry, though I understand that it's the simplest way to
> implement this in an RFC series. It's a bit surprising that fbind() is
> a total no-op for everything except KVM's guest_memfd.
>
> Maybe whatever you pass to fbind() could be stored in the struct file *,
> and used as the default when creating VMAs; as if every mmap() was
> followed by an mbind(), except that it also does the right thing with
> MAP_POPULATE for example. Or maybe that's a horrible idea?
mbind() manpage has this:
The specified policy will be ignored for any MAP_SHARED
mappings in the specified memory range. Rather the pages will be allocated
according to the memory policy of the thread that caused the page to be
allocated. Again, this may not be the thread that called mbind().
So that seems like we're not very keen on having one user of a file set a
policy that would affect other users of the file?
Now the next paragraph of the manpage says that shmem is different, and
guest_memfd is more like shmem than a regular file.
My conclusion from that is that fbind() might be too broad and we don't want
this for actual filesystem-backed files? And if it's limited to guest_memfd,
it shouldn't be an fbind()?
> Adding linux-api to get input; original thread is at
> https://lore.kernel.org/kvm/20241105164549.154700-1-shivankg@amd.com/.
>
> Paolo
>
>> Backing up, it seems like you want to make a change to the page cache,
>> you've had a long discussion with people who aren't the page cache
>> maintainer, and you all understand the pros and cons of everything,
>> and here you are dumping a solution on me without talking to me, even
>> though I was at Plumbers, you didn't find me to tell me I needed to go
>> to your talk.
>>
>> So you haven't explained a damned thing to me, and I'm annoyed at you.
>> Do better. Starting with your cover letter.
>>
>
>
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: Paolo Bonzini @ 2024-11-08 17:31 UTC (permalink / raw)
To: Matthew Wilcox, Shivank Garg
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
kees, bharata, nikunj, michael.day, Neeraj.Upadhyay, linux-coco,
Linux API
In-Reply-To: <ZyzYUOX_r3uWin5f@casper.infradead.org>
On 11/7/24 16:10, Matthew Wilcox wrote:
> On Thu, Nov 07, 2024 at 02:24:20PM +0530, Shivank Garg wrote:
>> The folio allocation path from guest_memfd typically looks like this...
>>
>> kvm_gmem_get_folio
>> filemap_grab_folio
>> __filemap_get_folio
>> filemap_alloc_folio
>> __folio_alloc_node_noprof
>> -> goes to the buddy allocator
>>
>> Hence, I am trying to have a version of filemap_alloc_folio() that takes an mpol.
>
> It only takes that path if cpuset_do_page_mem_spread() is true. Is the
> real problem that you're trying to solve that cpusets are being used
> incorrectly?
If it's false it's not very different, it goes to alloc_pages_noprof().
Then it respects the process's policy, but the policy is not
customizable without mucking with state that is global to the process.
Taking a step back: the problem is that a VM can be configured to have
multiple guest-side NUMA nodes, each of which will pick memory from the
right NUMA node in the host. Without a per-file operation it's not
possible to do this on guest_memfd. The discussion was whether to use
ioctl() or a new system call. The discussion ended with the idea of
posting a *proposal* asking for *comments* as to whether the system call
would be useful in general beyond KVM.
Commenting on the system call itself I am not sure I like the
file_operations entry, though I understand that it's the simplest way to
implement this in an RFC series. It's a bit surprising that fbind() is
a total no-op for everything except KVM's guest_memfd.
Maybe whatever you pass to fbind() could be stored in the struct file *,
and used as the default when creating VMAs; as if every mmap() was
followed by an mbind(), except that it also does the right thing with
MAP_POPULATE for example. Or maybe that's a horrible idea?
Adding linux-api to get input; original thread is at
https://lore.kernel.org/kvm/20241105164549.154700-1-shivankg@amd.com/.
Paolo
> Backing up, it seems like you want to make a change to the page cache,
> you've had a long discussion with people who aren't the page cache
> maintainer, and you all understand the pros and cons of everything,
> and here you are dumping a solution on me without talking to me, even
> though I was at Plumbers, you didn't find me to tell me I needed to go
> to your talk.
>
> So you haven't explained a damned thing to me, and I'm annoyed at you.
> Do better. Starting with your cover letter.
>
^ permalink raw reply
* Re: [PATCH v6 2/5] pidfd: add PIDFD_SELF_* sentinels to refer to own thread/process
From: Lorenzo Stoakes @ 2024-11-08 14:28 UTC (permalink / raw)
To: Christian Brauner
Cc: Oleg Nesterov, Christian Brauner, Shuah Khan, Liam R . Howlett,
Suren Baghdasaryan, Vlastimil Babka, pedro.falcato,
linux-kselftest, linux-mm, linux-fsdevel, linux-api, linux-kernel,
Oliver Sang, John Hubbard
In-Reply-To: <b8f4664c-b8f0-46ca-b9a3-8d73e398b5ca@lucifer.local>
On Wed, Oct 30, 2024 at 04:37:37PM +0000, Lorenzo Stoakes wrote:
> On Mon, Oct 28, 2024 at 04:06:07PM +0000, Lorenzo Stoakes wrote:
> > I guess I'll try to adapt that and respin a v7 when I get a chance.
>
> Hm looking at this draft patch, it seems like a total rework of pidfd's
> across the board right (now all pidfd's will need to be converted to
> pid_fd)? Correct me if I'm wrong.
>
> If only for the signal case, it seems like overkill to define a whole
> pid_fd and to use this CLASS() wrapper just for this one instance.
>
> If the intent is to convert _all_ pidfd's to use this type, it feels really
> out of scope for this series and I think we'd probably instead want to go
> off and do that as a separate series and put this on hold until that is
> done.
>
> If instead you mean that we ought to do something like this just for the
> signal case, it feels like it'd be quite a bit of extra abstraction just
> used in this one case but nowhere else, I think if you did an abstraction
> like this it would _have_ to be across the board right?
>
> I agree that the issue is with this one signal case that pins only the fd
> (rather than this pid) where this 'pinning' doesn't _necessary_ mess around
> with reference counts.
>
> So we definitely must address this, but the issue you had with the first
> approach was that I think (correct me if I'm wrong) I was passing a pointer
> to a struct fd which is not permitted right?
>
> Could we pass the struct fd by value to avoid this? I think we'd have to
> unfortunately special-case this and probably duplicate some code which is a
> pity as I liked the idea of abstracting everything to one place, but we can
> obviously do that.
>
> So I guess to TL;DR it, the options are:
>
> 1. Implement pid_fd everywhere, in which case I will leave off on
> this series and I guess, if I have time I could look at trying to
> implement that or perhaps you'd prefer to?
>
> 2. We are good for the sake of this series to special-case a pidfd_to_pid()
> implementation (used only by the pidfd_send_signal() syscall)
>
> 3. Something else, or I am misunderstanding your point :)
>
> Let me know how you want me to proceed on this as we're at v6 already and I
> want to be _really_ sure I'm doing what you want here.
>
> Thanks!
Hi Christian,
Just a gentle nudge on this - as I need some guidance in order to know how
to move the series forwards.
Obviously no rush if your workload is high at the moment as this is pretty
low priority, but just in case you missed it :)
Thanks, Lorenzo
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: Shivank Garg @ 2024-11-08 9:21 UTC (permalink / raw)
To: Matthew Wilcox
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
pbonzini, kees, bharata, nikunj, michael.day, Neeraj.Upadhyay,
linux-coco
In-Reply-To: <ZyzYUOX_r3uWin5f@casper.infradead.org>
On 11/7/2024 8:40 PM, Matthew Wilcox wrote:
> On Thu, Nov 07, 2024 at 02:24:20PM +0530, Shivank Garg wrote:
>> The folio allocation path from guest_memfd typically looks like this...
>>
>> kvm_gmem_get_folio
>> filemap_grab_folio
>> __filemap_get_folio
>> filemap_alloc_folio
>> __folio_alloc_node_noprof
>> -> goes to the buddy allocator
>>
>> Hence, I am trying to have a version of filemap_alloc_folio() that takes an mpol.
>
> It only takes that path if cpuset_do_page_mem_spread() is true. Is the
> real problem that you're trying to solve that cpusets are being used
> incorrectly?
>
> Backing up, it seems like you want to make a change to the page cache,
> you've had a long discussion with people who aren't the page cache
> maintainer, and you all understand the pros and cons of everything,
> and here you are dumping a solution on me without talking to me, even
> though I was at Plumbers, you didn't find me to tell me I needed to go
> to your talk.
>
> So you haven't explained a damned thing to me, and I'm annoyed at you.
> Do better. Starting with your cover letter.
Hi Matthew,
I apologize for any misunderstanding and not providing adequate context.
To clarify:
- You may recall this work from its earlier iteration as an
IOCTL-based approach, where you provided valuable review comments [1].
- I was not physically present at LPC. The discussion happened through
the mailing list [2] and lobby discussion with my colleagues who visited
Vienna.
- Based on feedback, particularly regarding the suggestion to consider
fbind() as a more generic solution, we shifted to the current approach.
I posted this as *RFC* specifically to gather feedback on the feasibility of
this approach and to ensure I'm heading in the right direction.
Would you be willing to help me understand:
1. What additional information would be helpful to you and other reviewers?
2. How cpusets can be used correctly to fix this? (your point on
cpuset_do_page_mem_spread() is interesting and I'll investigate it more
thoroughly to understand).
I'll work on improving the cover letter to better explain the problem space
and proposed solution.
Thank you for the valuable feedback.
[1] https://lore.kernel.org/linux-mm/ZuimLtrpv1dXczf5@casper.infradead.org
[2] https://lore.kernel.org/linux-mm/ZvEga7srKhympQBt@intel.com
Best regards,
Shivank
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Matt Muggeridge @ 2024-11-08 2:20 UTC (permalink / raw)
To: idosch
Cc: Matt.Muggeridge, davem, dsahern, edumazet, horms, kuba, linux-api,
linux-kernel, netdev, pabeni, stable
In-Reply-To: <ZyyN2bSgrpbhbkpp@shredder>
> > You probably already know how to reproduce it, but in case it helps, I still
> > have the packet captures and can share them with you. Let me know if you'd
> > like me to share them (and how to share them).
>
> It would be best if you could provide a reproducer using iproute2:
> Configure a dummy device using ip-link, install the multipath route
> using ip-route, configure the neighbour table using ip-neigh and then
> perform route queries using "ip route get ..." showing the problem. We
> can then use it as the basis for a new test case in
> tools/testing/selftests/net/fib_tests.sh
I'll try to do that next week.
> BTW, do you have CONFIG_IPV6_ROUTER_PREF=y in your config?
Yes.
$ gunzip -c /proc/config.gz | grep ROUTER_PREF
CONFIG_IPV6_ROUTER_PREF=y
> >
> > As such, it still seems appropriate (to me) that this be implemented in the
> > legacy API as well as ensuring it works with the NH API.
>
> As I understand it you currently get different results because the
> kernel installs two default routes whereas user space can only create
> one default multipath route.
Yes, that's the end result of an underlying problem.
Perhaps more to the point, the fact that a coalesced, INCOMPLETE, multipath
route is selected when a REACHABLE alternative exists, is what prevents us
from using coalesced multipath routes. This seems like a bug, since it violates
RFC4861 6.3.6, bullet 1.
Imagine adding a 2nd router to an IPv6 network for added resiliency, but when
one becomes unreachable, some network flows keep choosing the unreachable
router. This is what is happening with ECMP routes. It doesn't happen with
multiple default routes.
I'll just reiterate earlier comments, this doesn't happen all of the time.
It seems I have a 50/50 chance of the INCOMPLETE route being selected.
> Before adding a new uAPI I want to
> understand the source of the difference and see if we can improve / fix
> the current multipath code so that the two behave the same. If we can
> get them to behave the same then I don't think user space will care
> about two default routes versus one default multipath route.
Exactly, I totally support that approach.
Regards,
Matt.
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: Matthew Wilcox @ 2024-11-07 15:10 UTC (permalink / raw)
To: Shivank Garg
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
pbonzini, kees, bharata, nikunj, michael.day, Neeraj.Upadhyay,
linux-coco
In-Reply-To: <6004eaa4-934c-48f4-b502-cf7e436462fc@amd.com>
On Thu, Nov 07, 2024 at 02:24:20PM +0530, Shivank Garg wrote:
> The folio allocation path from guest_memfd typically looks like this...
>
> kvm_gmem_get_folio
> filemap_grab_folio
> __filemap_get_folio
> filemap_alloc_folio
> __folio_alloc_node_noprof
> -> goes to the buddy allocator
>
> Hence, I am trying to have a version of filemap_alloc_folio() that takes an mpol.
It only takes that path if cpuset_do_page_mem_spread() is true. Is the
real problem that you're trying to solve that cpusets are being used
incorrectly?
Backing up, it seems like you want to make a change to the page cache,
you've had a long discussion with people who aren't the page cache
maintainer, and you all understand the pros and cons of everything,
and here you are dumping a solution on me without talking to me, even
though I was at Plumbers, you didn't find me to tell me I needed to go
to your talk.
So you haven't explained a damned thing to me, and I'm annoyed at you.
Do better. Starting with your cover letter.
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Ido Schimmel @ 2024-11-07 9:52 UTC (permalink / raw)
To: Matt Muggeridge
Cc: davem, dsahern, edumazet, horms, kuba, linux-api, linux-kernel,
netdev, pabeni, stable
In-Reply-To: <20241107035303.24057-1-Matt.Muggeridge@hpe.com>
On Wed, Nov 06, 2024 at 10:53:03PM -0500, Matt Muggeridge wrote:
> Hi Ido,
>
> > >>> Is the problem that fib6_table_lookup() chooses a reachable
> > >>> nexthop and then fib6_select_path() overrides it with an unreachable
> > >>> one?
> >
> > >> I'm afraid I don't know.
> > >>
> > > We need to understand the current behavior before adding a new interface
> > > that we will never be able to remove. It is possible we can improve /
> > > fix the current code. I won't have time to look into it myself until
> > > next week.
>
> I am grateful that you want to look into it. Thank you! And I look forward to
> learning what you discover.
>
> You probably already know how to reproduce it, but in case it helps, I still
> have the packet captures and can share them with you. Let me know if you'd
> like me to share them (and how to share them).
It would be best if you could provide a reproducer using iproute2:
Configure a dummy device using ip-link, install the multipath route
using ip-route, configure the neighbour table using ip-neigh and then
perform route queries using "ip route get ..." showing the problem. We
can then use it as the basis for a new test case in
tools/testing/selftests/net/fib_tests.sh
BTW, do you have CONFIG_IPV6_ROUTER_PREF=y in your config?
>
> > >
> > > The objective is to allow IPv6 Netlink clients to be able to create default
> > > routes from RAs in the same way the kernel creates default routes from RAs.
> > > Essentially, I'm trying to have Netlink and Kernel behaviors match.
> >
> > I understand, but it's essentially an extension for the legacy IPv6
> > multipath API which we are trying to move away from towards the nexthop
> > API (see more below).
>
> Very interesting, I wasn't aware of this movement.
>
> While this change is an extension of the legacy IPv6 multipath API, won't it
> still need to support Netlink clients that have been designed around it? I
> imagine that transitioning Netlink clients to the NH API will take many years?
FRR already supports it and I saw that there is some support for nexthop
objects in systemd:
https://github.com/systemd/systemd/pull/13735
>
> As such, it still seems appropriate (to me) that this be implemented in the
> legacy API as well as ensuring it works with the NH API.
As I understand it you currently get different results because the
kernel installs two default routes whereas user space can only create
one default multipath route. Before adding a new uAPI I want to
understand the source of the difference and see if we can improve / fix
the current multipath code so that the two behave the same. If we can
get them to behave the same then I don't think user space will care
about two default routes versus one default multipath route.
>
> Another consideration...
>
> Will the kernel RA processing go through the same nh pathway? The reason I
> ask is because I faced several challenges with IPv6 Logo certification due to
> Netlink clients being unable to achieve the same as the kernel's behavior.
If you are asking if the kernel can install RA routes using nexthop
objects, then the answer is no. Only user space can create nexthop
objects and I don't think we want to allow the kernel to do that.
>
> As long as the kernel is creating RA routes in a way that meets RFC4861, then
> I'd hope that Netlink clients would be able to leverage that for 'free'.
>
> > >
> > > My analysis led me to the need for Netlink clients to set the kernel's
> > > fib6_config flags RTF_RA_ROUTER, where:
> > >
> > > #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
> > >
> > >>> + if (rtm->rtm_flags & RTM_F_RA_ROUTER)
> > >>> + cfg->fc_flags |= RTF_RA_ROUTER;
> > >>> +
> > >>
> > >> It is possible there are user space programs out there that set this bit
> > >> (knowingly or not) when sending requests to the kernel and this change
> > >> will result in a behavior change for them. So, if we were to continue in
> > >> this path, this would need to be converted to a new netlink attribute to
> > >> avoid such potential problems.
> > >>
> > >
> > > Is this a mandated approach to implementing unspecified bits in a flag?
> > >
> > > I'm a little surprised by this consideration. If we account for poorly
> > > written buggy user-programs, doesn't this open any API to an explosion
> > > of new attributes or other odd extensions? I'd imagine the same argument
> > > would be applicable to ioctl flags, socket flags, and so on. Why would we
> > > treat implementing unspecified Netlink bits differently to implementing
> > > unspecified ioctl bits, etc.
> > >
> > > Naturally, if this is the mandated approach, then I'll reimplement it with
> > > a new Netlink attribute. I'm just trying to understand what is the
> > > Linux-lore, here?
> >
> > Using this bit could have been valid if previously the kernel rejected
> > requests with this bit set, but as evident by your patch the kernel does
> > not do it. It is therefore possible that there are user space programs
> > out there that are working perfectly fine right now and they will break
> > / misbehave after this change.
> >
>
> Understood and I agree.
>
> > >
> > >> BTW, you can avoid the coalescing problem by using the nexthop API (man
> > >> ip-nexthop).
> > >
> > > I'm not sure how that would help in this case. We need the nexthop to be
> > > determined according to its REACHABILITY and other considerations described
> > > in RFC4861.
> >
> > Using your example:
> >
> > # ip nexthop add id 1 via fe80::200:10ff:fe10:1060 dev enp0s9
> > # ip -6 route add default nhid 1 expires 600 proto ra
> > # ip nexthop add id 2 via fe80::200:10ff:fe10:1061 dev enp0s9
> > # ip -6 route append default nhid 2 expires 600 proto ra
> > # ip -6 route
> > fe80::/64 dev enp0s9 proto kernel metric 256 pref medium
> > default nhid 1 via fe80::200:10ff:fe10:1060 dev enp0s9 proto ra metric 1024 expires 563sec pref medium
> > default nhid 2 via fe80::200:10ff:fe10:1061 dev enp0s9 proto ra metric 1024 expires 594sec pref medium
>
> Thanks! That looks like it should work. I'll raise this with the the developers
> of systemd-networkd.
>
> Just to confirm; are these two nhid routes equivalent to having two separate
> default routes that are created when the kernel processes IPv6 RAs?
>
> Specifically, if one of these nhid routes becomes UNREACHABLE, will that be
> taken into consideration during the routing decision? (I'm guessing so?)
I didn't test it, but I don't see a reason for these two routes to
behave differently than two default routes installed with legacy
nexthops.
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: Shivank Garg @ 2024-11-07 8:54 UTC (permalink / raw)
To: Matthew Wilcox
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
pbonzini, kees, bharata, nikunj, michael.day, Neeraj.Upadhyay,
linux-coco
In-Reply-To: <ZypqJ0e-J3C_K8LA@casper.infradead.org>
Hi Matthew,
On 11/6/2024 12:25 AM, Matthew Wilcox wrote:
> On Tue, Nov 05, 2024 at 04:45:45PM +0000, Shivank Garg wrote:
>> This patch series introduces fbind() syscall to support NUMA memory
>> policies for KVM guest_memfd, allowing VMMs to configure memory placement
>> for guest memory. This addresses the current limitation where guest_memfd
>> allocations ignore NUMA policies, potentially impacting performance of
>> memory-locality-sensitive workloads.
>
> Why does guest_memfd ignore numa policies? The pagecache doesn't,
> eg in vma_alloc_folio_noprof().
guest_memfd doesn't have VMAs and hence can't store policy information in
VMA and use vma_alloc_folio_noprof() that fetches mpol from VMA.
The folio allocation path from guest_memfd typically looks like this...
kvm_gmem_get_folio
filemap_grab_folio
__filemap_get_folio
filemap_alloc_folio
__folio_alloc_node_noprof
-> goes to the buddy allocator
Hence, I am trying to have a version of filemap_alloc_folio() that takes an mpol.
Thanks,
Shivank
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Matt Muggeridge @ 2024-11-07 3:53 UTC (permalink / raw)
To: idosch
Cc: Matt.Muggeridge, davem, dsahern, edumazet, horms, kuba, linux-api,
linux-kernel, netdev, pabeni, stable
In-Reply-To: <ZytjEINNRmtpadr_@shredder>
Hi Ido,
> >>> Is the problem that fib6_table_lookup() chooses a reachable
> >>> nexthop and then fib6_select_path() overrides it with an unreachable
> >>> one?
>
> >> I'm afraid I don't know.
> >>
> > We need to understand the current behavior before adding a new interface
> > that we will never be able to remove. It is possible we can improve /
> > fix the current code. I won't have time to look into it myself until
> > next week.
I am grateful that you want to look into it. Thank you! And I look forward to
learning what you discover.
You probably already know how to reproduce it, but in case it helps, I still
have the packet captures and can share them with you. Let me know if you'd
like me to share them (and how to share them).
> >
> > The objective is to allow IPv6 Netlink clients to be able to create default
> > routes from RAs in the same way the kernel creates default routes from RAs.
> > Essentially, I'm trying to have Netlink and Kernel behaviors match.
>
> I understand, but it's essentially an extension for the legacy IPv6
> multipath API which we are trying to move away from towards the nexthop
> API (see more below).
Very interesting, I wasn't aware of this movement.
While this change is an extension of the legacy IPv6 multipath API, won't it
still need to support Netlink clients that have been designed around it? I
imagine that transitioning Netlink clients to the NH API will take many years?
As such, it still seems appropriate (to me) that this be implemented in the
legacy API as well as ensuring it works with the NH API.
Another consideration...
Will the kernel RA processing go through the same nh pathway? The reason I
ask is because I faced several challenges with IPv6 Logo certification due to
Netlink clients being unable to achieve the same as the kernel's behavior.
As long as the kernel is creating RA routes in a way that meets RFC4861, then
I'd hope that Netlink clients would be able to leverage that for 'free'.
> >
> > My analysis led me to the need for Netlink clients to set the kernel's
> > fib6_config flags RTF_RA_ROUTER, where:
> >
> > #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
> >
> >>> + if (rtm->rtm_flags & RTM_F_RA_ROUTER)
> >>> + cfg->fc_flags |= RTF_RA_ROUTER;
> >>> +
> >>
> >> It is possible there are user space programs out there that set this bit
> >> (knowingly or not) when sending requests to the kernel and this change
> >> will result in a behavior change for them. So, if we were to continue in
> >> this path, this would need to be converted to a new netlink attribute to
> >> avoid such potential problems.
> >>
> >
> > Is this a mandated approach to implementing unspecified bits in a flag?
> >
> > I'm a little surprised by this consideration. If we account for poorly
> > written buggy user-programs, doesn't this open any API to an explosion
> > of new attributes or other odd extensions? I'd imagine the same argument
> > would be applicable to ioctl flags, socket flags, and so on. Why would we
> > treat implementing unspecified Netlink bits differently to implementing
> > unspecified ioctl bits, etc.
> >
> > Naturally, if this is the mandated approach, then I'll reimplement it with
> > a new Netlink attribute. I'm just trying to understand what is the
> > Linux-lore, here?
>
> Using this bit could have been valid if previously the kernel rejected
> requests with this bit set, but as evident by your patch the kernel does
> not do it. It is therefore possible that there are user space programs
> out there that are working perfectly fine right now and they will break
> / misbehave after this change.
>
Understood and I agree.
> >
> >> BTW, you can avoid the coalescing problem by using the nexthop API (man
> >> ip-nexthop).
> >
> > I'm not sure how that would help in this case. We need the nexthop to be
> > determined according to its REACHABILITY and other considerations described
> > in RFC4861.
>
> Using your example:
>
> # ip nexthop add id 1 via fe80::200:10ff:fe10:1060 dev enp0s9
> # ip -6 route add default nhid 1 expires 600 proto ra
> # ip nexthop add id 2 via fe80::200:10ff:fe10:1061 dev enp0s9
> # ip -6 route append default nhid 2 expires 600 proto ra
> # ip -6 route
> fe80::/64 dev enp0s9 proto kernel metric 256 pref medium
> default nhid 1 via fe80::200:10ff:fe10:1060 dev enp0s9 proto ra metric 1024 expires 563sec pref medium
> default nhid 2 via fe80::200:10ff:fe10:1061 dev enp0s9 proto ra metric 1024 expires 594sec pref medium
Thanks! That looks like it should work. I'll raise this with the the developers
of systemd-networkd.
Just to confirm; are these two nhid routes equivalent to having two separate
default routes that are created when the kernel processes IPv6 RAs?
Specifically, if one of these nhid routes becomes UNREACHABLE, will that be
taken into consideration during the routing decision? (I'm guessing so?)
Thank you for your interest in this.
Regards,
Matt.
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: David Ahern @ 2024-11-06 18:59 UTC (permalink / raw)
To: Matt Muggeridge, idosch
Cc: davem, edumazet, horms, kuba, linux-api, linux-kernel, netdev,
pabeni, stable
In-Reply-To: <20241106025056.11241-1-Matt.Muggeridge@hpe.com>
On 11/5/24 7:50 PM, Matt Muggeridge wrote:
> I'm a little surprised by this consideration. If we account for poorly
> written buggy user-programs, doesn't this open any API to an explosion
> of new attributes or other odd extensions? I'd imagine the same argument
yes, it does and there are many examples. UAPIs are hard to get right
yet persistent forever, so I do agree with Ido's push back on making
sure it is needed.
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Ido Schimmel @ 2024-11-06 12:37 UTC (permalink / raw)
To: Matt Muggeridge
Cc: davem, dsahern, edumazet, horms, kuba, linux-api, linux-kernel,
netdev, pabeni, stable
In-Reply-To: <20241106025056.11241-1-Matt.Muggeridge@hpe.com>
On Tue, Nov 05, 2024 at 09:50:56PM -0500, Matt Muggeridge wrote:
> Thank you for your review and feedback, Ido.
>
> >> Without this flag, when there are mutliple default routers, the kernel
> >> coalesces multiple default routes into an ECMP route. The ECMP route
> >> ignores per-route REACHABILITY information. If one of the default
> >> routers is unresponsive, with a Neighbor Cache entry of INCOMPLETE, then
> >> it can still be selected as the nexthop for outgoing packets. This
> >> results in an inability to communicate with remote hosts, even though
> >> one of the default routers remains REACHABLE. This violates RFC4861
> >> section 6.3.6, bullet 1.
> >
> >Do you have forwarding disabled (it causes RT6_LOOKUP_F_REACHABLE to be
> >set)?
>
> Yes, forwarding is disabled on our embedded system. Though, this needs to
> work on systems regardless of the state of forwarding.
>
> > Is the problem that fib6_table_lookup() chooses a reachable
> >nexthop and then fib6_select_path() overrides it with an unreachable
> >one?
>
> I'm afraid I don't know.
We need to understand the current behavior before adding a new interface
that we will never be able to remove. It is possible we can improve /
fix the current code. I won't have time to look into it myself until
next week.
>
> The objective is to allow IPv6 Netlink clients to be able to create default
> routes from RAs in the same way the kernel creates default routes from RAs.
> Essentially, I'm trying to have Netlink and Kernel behaviors match.
I understand, but it's essentially an extension for the legacy IPv6
multipath API which we are trying to move away from towards the nexthop
API (see more below).
>
> My analysis led me to the need for Netlink clients to set the kernel's
> fib6_config flags RTF_RA_ROUTER, where:
>
> #define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
>
> >> + if (rtm->rtm_flags & RTM_F_RA_ROUTER)
> >> + cfg->fc_flags |= RTF_RA_ROUTER;
> >> +
> >
> > It is possible there are user space programs out there that set this bit
> > (knowingly or not) when sending requests to the kernel and this change
> > will result in a behavior change for them. So, if we were to continue in
> > this path, this would need to be converted to a new netlink attribute to
> > avoid such potential problems.
> >
>
> Is this a mandated approach to implementing unspecified bits in a flag?
>
> I'm a little surprised by this consideration. If we account for poorly
> written buggy user-programs, doesn't this open any API to an explosion
> of new attributes or other odd extensions? I'd imagine the same argument
> would be applicable to ioctl flags, socket flags, and so on. Why would we
> treat implementing unspecified Netlink bits differently to implementing
> unspecified ioctl bits, etc.
>
> Naturally, if this is the mandated approach, then I'll reimplement it with
> a new Netlink attribute. I'm just trying to understand what is the
> Linux-lore, here?
Using this bit could have been valid if previously the kernel rejected
requests with this bit set, but as evident by your patch the kernel does
not do it. It is therefore possible that there are user space programs
out there that are working perfectly fine right now and they will break
/ misbehave after this change.
>
> > BTW, you can avoid the coalescing problem by using the nexthop API (man
> > ip-nexthop).
>
> I'm not sure how that would help in this case. We need the nexthop to be
> determined according to its REACHABILITY and other considerations described
> in RFC4861.
Using your example:
# ip nexthop add id 1 via fe80::200:10ff:fe10:1060 dev enp0s9
# ip -6 route add default nhid 1 expires 600 proto ra
# ip nexthop add id 2 via fe80::200:10ff:fe10:1061 dev enp0s9
# ip -6 route append default nhid 2 expires 600 proto ra
# ip -6 route
fe80::/64 dev enp0s9 proto kernel metric 256 pref medium
default nhid 1 via fe80::200:10ff:fe10:1060 dev enp0s9 proto ra metric 1024 expires 563sec pref medium
default nhid 2 via fe80::200:10ff:fe10:1061 dev enp0s9 proto ra metric 1024 expires 594sec pref medium
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Matt Muggeridge @ 2024-11-06 2:50 UTC (permalink / raw)
To: idosch
Cc: Matt.Muggeridge, davem, dsahern, edumazet, horms, kuba, linux-api,
linux-kernel, netdev, pabeni, stable
In-Reply-To: <Zypgu5l7F1FpIpqo@shredder>
Thank you for your review and feedback, Ido.
>> Without this flag, when there are mutliple default routers, the kernel
>> coalesces multiple default routes into an ECMP route. The ECMP route
>> ignores per-route REACHABILITY information. If one of the default
>> routers is unresponsive, with a Neighbor Cache entry of INCOMPLETE, then
>> it can still be selected as the nexthop for outgoing packets. This
>> results in an inability to communicate with remote hosts, even though
>> one of the default routers remains REACHABLE. This violates RFC4861
>> section 6.3.6, bullet 1.
>
>Do you have forwarding disabled (it causes RT6_LOOKUP_F_REACHABLE to be
>set)?
Yes, forwarding is disabled on our embedded system. Though, this needs to
work on systems regardless of the state of forwarding.
> Is the problem that fib6_table_lookup() chooses a reachable
>nexthop and then fib6_select_path() overrides it with an unreachable
>one?
I'm afraid I don't know.
The objective is to allow IPv6 Netlink clients to be able to create default
routes from RAs in the same way the kernel creates default routes from RAs.
Essentially, I'm trying to have Netlink and Kernel behaviors match.
My analysis led me to the need for Netlink clients to set the kernel's
fib6_config flags RTF_RA_ROUTER, where:
#define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
>> + if (rtm->rtm_flags & RTM_F_RA_ROUTER)
>> + cfg->fc_flags |= RTF_RA_ROUTER;
>> +
>
> It is possible there are user space programs out there that set this bit
> (knowingly or not) when sending requests to the kernel and this change
> will result in a behavior change for them. So, if we were to continue in
> this path, this would need to be converted to a new netlink attribute to
> avoid such potential problems.
>
Is this a mandated approach to implementing unspecified bits in a flag?
I'm a little surprised by this consideration. If we account for poorly
written buggy user-programs, doesn't this open any API to an explosion
of new attributes or other odd extensions? I'd imagine the same argument
would be applicable to ioctl flags, socket flags, and so on. Why would we
treat implementing unspecified Netlink bits differently to implementing
unspecified ioctl bits, etc.
Naturally, if this is the mandated approach, then I'll reimplement it with
a new Netlink attribute. I'm just trying to understand what is the
Linux-lore, here?
> BTW, you can avoid the coalescing problem by using the nexthop API (man
> ip-nexthop).
I'm not sure how that would help in this case. We need the nexthop to be
determined according to its REACHABILITY and other considerations described
in RFC4861.
Kind regards,
Matt.
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Matt Muggeridge @ 2024-11-06 1:44 UTC (permalink / raw)
To: nicolas.dichtel
Cc: Matt.Muggeridge, davem, dsahern, edumazet, horms, kuba, linux-api,
linux-kernel, netdev, pabeni, stable
In-Reply-To: <0a8d6565-fdc0-452f-b132-5d237a1b7dec@6wind.com>
> Please, don't mix whitespace changes with the changes related to the new flag.
Thanks, Nicolas. I will revert the changes that tidied up trailing whitespace in the next version.
^ permalink raw reply
* Re: [RFC PATCH 0/4] Add fbind() and NUMA mempolicy support for KVM guest_memfd
From: Matthew Wilcox @ 2024-11-05 18:55 UTC (permalink / raw)
To: Shivank Garg
Cc: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm, chao.gao, pgonda,
thomas.lendacky, seanjc, luto, tglx, mingo, bp, dave.hansen, arnd,
pbonzini, kees, bharata, nikunj, michael.day, Neeraj.Upadhyay
In-Reply-To: <20241105164549.154700-1-shivankg@amd.com>
On Tue, Nov 05, 2024 at 04:45:45PM +0000, Shivank Garg wrote:
> This patch series introduces fbind() syscall to support NUMA memory
> policies for KVM guest_memfd, allowing VMMs to configure memory placement
> for guest memory. This addresses the current limitation where guest_memfd
> allocations ignore NUMA policies, potentially impacting performance of
> memory-locality-sensitive workloads.
Why does guest_memfd ignore numa policies? The pagecache doesn't,
eg in vma_alloc_folio_noprof().
^ permalink raw reply
* Re: [PATCH net 1/1] net/ipv6: Netlink flag for new IPv6 Default Routes
From: Ido Schimmel @ 2024-11-05 18:15 UTC (permalink / raw)
To: Matt Muggeridge
Cc: David Ahern, David S . Miller, linux-api, stable, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, Simon Horman, netdev, linux-kernel
In-Reply-To: <20241105031841.10730-2-Matt.Muggeridge@hpe.com>
On Mon, Nov 04, 2024 at 10:18:39PM -0500, Matt Muggeridge wrote:
> Add a Netlink rtm_flag, RTM_F_RA_ROUTER for the RTM_NEWROUTE message.
> This allows an IPv6 Netlink client to indicate the default route came
> from an RA. This results in the kernel creating individual default
> routes, rather than coalescing multiple default routes into a single
> ECMP route.
>
> Details:
>
> For IPv6, a Netlink client is unable to create default routes in the
> same manner as the kernel. This leads to failures when there are
> multiple default routers, as they were being coalesced into a single
> ECMP route. When one of the ECMP default routers becomes UNREACHABLE, it
> was still being selected as the nexthop.
>
> Meanwhile, when the kernel processes RAs from multiple default routers,
> it sets the fib6_flags: RTF_ADDRCONF | RTF_DEFAULT. The RTF_ADDRCONF
> flag is checked by rt6_qualify_for_ecmp(), which returns false when
> ADDRCONF is set. As such, the kernel creates separate default routes.
>
> E.g. compare the routing tables when RAs are processed by the kernel
> versus a Netlink client (systemd-networkd, in my case).
>
> 1) RA Processed by kernel (accept_ra = 2)
> $ ip -6 route
> 2001:2:0:1000::/64 dev enp0s9 proto kernel metric 256 expires ...
> fe80::/64 dev enp0s9 proto kernel metric 256 pref medium
> default via fe80::200:10ff:fe10:1060 dev enp0s9 proto ra ...
> default via fe80::200:10ff:fe10:1061 dev enp0s9 proto ra ...
>
> 2) RA Processed by Netlink client (accept_ra = 0)
> $ ip -6 route
> 2001:2:0:1000::/64 dev enp0s9 proto ra metric 1024 expires ...
> fe80::/64 dev enp0s3 proto kernel metric 256 pref medium
> fe80::/64 dev enp0s9 proto kernel metric 256 pref medium
> default proto ra metric 1024 expires 595sec pref medium
> nexthop via fe80::200:10ff:fe10:1060 dev enp0s9 weight 1
> nexthop via fe80::200:10ff:fe10:1061 dev enp0s9 weight 1
>
> IPv6 Netlink clients need a mechanism to identify a route as coming from
> an RA. i.e. a Netlink client needs a method to set the kernel flags:
>
> RTF_ADDRCONF | RTF_DEFAULT
>
> This is needed when there are multiple default routers that each send
> an RA. Setting the RTF_ADDRCONF flag ensures their fib entries do not
> qualify for ECMP routes, see rt6_qualify_for_ecmp().
>
> To achieve this, introduce a user-level flag RTM_F_RA_ROUTER that a
> Netlink client can pass to the kernel.
>
> A Netlink user-level network manager, such as systemd-networkd, may set
> the RTM_F_RA_ROUTER flag in the Netlink RTM_NEWROUTE rtmsg. When set,
> the kernel sets RTF_RA_ROUTER in the fib6_config fc_flags. This causes a
> default route to be created in the same way as if the kernel processed
> the RA, via rt6add_dflt_router().
>
> This is needed by user-level network managers, like systemd-networkd,
> that prefer to do the RA processing themselves. ie. they disable the
> kernel's RA processing by setting net.ipv6.conf.<intf>.accept_ra=0.
>
> Without this flag, when there are mutliple default routers, the kernel
> coalesces multiple default routes into an ECMP route. The ECMP route
> ignores per-route REACHABILITY information. If one of the default
> routers is unresponsive, with a Neighbor Cache entry of INCOMPLETE, then
> it can still be selected as the nexthop for outgoing packets. This
> results in an inability to communicate with remote hosts, even though
> one of the default routers remains REACHABLE. This violates RFC4861
> section 6.3.6, bullet 1.
Do you have forwarding disabled (it causes RT6_LOOKUP_F_REACHABLE to be
set)? Is the problem that fib6_table_lookup() chooses a reachable
nexthop and then fib6_select_path() overrides it with an unreachable
one?
>
> Extract from RFC4861 6.3.6 bullet 1:
> 1) Routers that are reachable or probably reachable (i.e., in any
> state other than INCOMPLETE) SHOULD be preferred over routers
> whose reachability is unknown or suspect (i.e., in the
> INCOMPLETE state, or for which no Neighbor Cache entry exists).
>
> This fixes the IPv6 Logo conformance test v6LC_2_2_11, and others that
> test with multiple default routers. Also see systemd issue #33470:
> https://github.com/systemd/systemd/issues/33470.
>
> Signed-off-by: Matt Muggeridge <Matt.Muggeridge@hpe.com>
> Cc: David Ahern <dsahern@kernel.org>
> Cc: David S. Miller <davem@davemloft.net>
> Cc: linux-api@vger.kernel.org
> Cc: stable@vger.kernel.org
> ---
> include/uapi/linux/rtnetlink.h | 9 +++++----
> net/ipv6/route.c | 3 +++
> 2 files changed, 8 insertions(+), 4 deletions(-)
>
> diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h
> index 3b687d20c9ed..9f0259f6e4ed 100644
> --- a/include/uapi/linux/rtnetlink.h
> +++ b/include/uapi/linux/rtnetlink.h
> @@ -202,7 +202,7 @@ enum {
> #define RTM_NR_FAMILIES (RTM_NR_MSGTYPES >> 2)
> #define RTM_FAM(cmd) (((cmd) - RTM_BASE) >> 2)
>
> -/*
> +/*
> Generic structure for encapsulation of optional route information.
> It is reminiscent of sockaddr, but with sa_family replaced
> with attribute type.
> @@ -242,7 +242,7 @@ struct rtmsg {
>
> unsigned char rtm_table; /* Routing table id */
> unsigned char rtm_protocol; /* Routing protocol; see below */
> - unsigned char rtm_scope; /* See below */
> + unsigned char rtm_scope; /* See below */
> unsigned char rtm_type; /* See below */
>
> unsigned rtm_flags;
> @@ -336,6 +336,7 @@ enum rt_scope_t {
> #define RTM_F_FIB_MATCH 0x2000 /* return full fib lookup match */
> #define RTM_F_OFFLOAD 0x4000 /* route is offloaded */
> #define RTM_F_TRAP 0x8000 /* route is trapping packets */
> +#define RTM_F_RA_ROUTER 0x10000 /* route is a default route from RA */
> #define RTM_F_OFFLOAD_FAILED 0x20000000 /* route offload failed, this value
> * is chosen to avoid conflicts with
> * other flags defined in
> @@ -568,7 +569,7 @@ struct ifinfomsg {
> };
>
> /********************************************************************
> - * prefix information
> + * prefix information
> ****/
>
> struct prefixmsg {
> @@ -582,7 +583,7 @@ struct prefixmsg {
> unsigned char prefix_pad3;
> };
>
> -enum
> +enum
> {
> PREFIX_UNSPEC,
> PREFIX_ADDRESS,
> diff --git a/net/ipv6/route.c b/net/ipv6/route.c
> index b4251915585f..5b0c16422720 100644
> --- a/net/ipv6/route.c
> +++ b/net/ipv6/route.c
> @@ -5055,6 +5055,9 @@ static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
> if (rtm->rtm_flags & RTM_F_CLONED)
> cfg->fc_flags |= RTF_CACHE;
>
> + if (rtm->rtm_flags & RTM_F_RA_ROUTER)
> + cfg->fc_flags |= RTF_RA_ROUTER;
> +
It is possible there are user space programs out there that set this bit
(knowingly or not) when sending requests to the kernel and this change
will result in a behavior change for them. So, if we were to continue in
this path, this would need to be converted to a new netlink attribute to
avoid such potential problems.
BTW, you can avoid the coalescing problem by using the nexthop API (man
ip-nexthop).
> cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);
>
> if (tb[RTA_NH_ID]) {
> --
> 2.35.3
>
>
^ permalink raw reply
* [RFC PATCH 4/4] KVM: guest_memfd: Enforce NUMA mempolicy if available
From: Shivank Garg @ 2024-11-05 16:55 UTC (permalink / raw)
To: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm
Cc: chao.gao, pgonda, thomas.lendacky, seanjc, luto, tglx, mingo, bp,
dave.hansen, willy, arnd, pbonzini, kees, shivankg, bharata,
nikunj, michael.day, Neeraj.Upadhyay
In-Reply-To: <20241105165515.154941-1-shivankg@amd.com>
Enforce memory policy on guest-memfd to provide proper NUMA support.
Previously, guest-memfd allocations were following local NUMA node id
in absence of process mempolicy, resulting in random memory allocation.
Moreover, it cannot use mbind() since memory isn't mapped to userspace.
To support NUMA policies, call fbind() syscall from VMM to store
mempolicy as f_policy in struct kvm_gmem of guest_memfd. The f_policy
is retrieved to pass in filemap_grab_folio_mpol() to ensure that
allocations follow the specified memory policy.
Signed-off-by: Shivank Garg <shivankg@amd.com>
---
mm/mempolicy.c | 2 ++
virt/kvm/guest_memfd.c | 49 ++++++++++++++++++++++++++++++++++++++----
2 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 3a697080ecad..af2e1ef4dae7 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -347,6 +347,7 @@ void __mpol_put(struct mempolicy *pol)
return;
kmem_cache_free(policy_cache, pol);
}
+EXPORT_SYMBOL(__mpol_put);
static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
{
@@ -2599,6 +2600,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old)
atomic_set(&new->refcnt, 1);
return new;
}
+EXPORT_SYMBOL(__mpol_dup);
/* Slow path of a mempolicy comparison */
bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index 2c6fcf7c3ec9..0237bda4382c 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -4,6 +4,7 @@
#include <linux/kvm_host.h>
#include <linux/pagemap.h>
#include <linux/anon_inodes.h>
+#include <linux/mempolicy.h>
#include "kvm_mm.h"
@@ -11,6 +12,7 @@ struct kvm_gmem {
struct kvm *kvm;
struct xarray bindings;
struct list_head entry;
+ struct mempolicy *f_policy;
};
/**
@@ -87,7 +89,8 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
}
static struct folio *kvm_gmem_get_huge_folio(struct inode *inode, pgoff_t index,
- unsigned int order)
+ unsigned int order,
+ struct mempolicy *policy)
{
pgoff_t npages = 1UL << order;
pgoff_t huge_index = round_down(index, npages);
@@ -104,7 +107,7 @@ static struct folio *kvm_gmem_get_huge_folio(struct inode *inode, pgoff_t index,
(loff_t)(huge_index + npages - 1) << PAGE_SHIFT))
return NULL;
- folio = filemap_alloc_folio(gfp, order);
+ folio = filemap_alloc_folio_mpol(gfp, order, policy);
if (!folio)
return NULL;
@@ -129,12 +132,26 @@ static struct folio *__kvm_gmem_get_folio(struct file *file, pgoff_t index,
bool allow_huge)
{
struct folio *folio = NULL;
+ struct kvm_gmem *gmem = file->private_data;
+ struct mempolicy *policy = NULL;
+
+ /*
+ * RCU lock is required to prevent any race condition with set_policy().
+ */
+ if (IS_ENABLED(CONFIG_NUMA)) {
+ rcu_read_lock();
+ policy = READ_ONCE(gmem->f_policy);
+ mpol_get(policy);
+ rcu_read_unlock();
+ }
if (gmem_2m_enabled && allow_huge)
- folio = kvm_gmem_get_huge_folio(file_inode(file), index, PMD_ORDER);
+ folio = kvm_gmem_get_huge_folio(file_inode(file), index, PMD_ORDER, policy);
if (!folio)
- folio = filemap_grab_folio(file_inode(file)->i_mapping, index);
+ folio = filemap_grab_folio_mpol(file_inode(file)->i_mapping, index, policy);
+
+ mpol_put(policy);
pr_debug("%s: allocate folio with PFN %lx order %d\n",
__func__, folio_pfn(folio), folio_order(folio));
@@ -338,6 +355,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
mutex_unlock(&kvm->slots_lock);
xa_destroy(&gmem->bindings);
+ mpol_put(gmem->f_policy);
kfree(gmem);
kvm_put_kvm(kvm);
@@ -356,10 +374,32 @@ static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot)
return get_file_active(&slot->gmem.file);
}
+#ifdef CONFIG_NUMA
+static int kvm_gmem_set_policy(struct file *file, struct mempolicy *mpol)
+{
+ struct mempolicy *old, *new;
+ struct kvm_gmem *gmem = file->private_data;
+
+ new = mpol_dup(mpol);
+ if (IS_ERR(new))
+ return PTR_ERR(new);
+
+ old = gmem->f_policy;
+ WRITE_ONCE(gmem->f_policy, new);
+ synchronize_rcu();
+ mpol_put(old);
+
+ return 0;
+}
+#endif
+
static struct file_operations kvm_gmem_fops = {
.open = generic_file_open,
.release = kvm_gmem_release,
.fallocate = kvm_gmem_fallocate,
+#ifdef CONFIG_NUMA
+ .set_policy = kvm_gmem_set_policy,
+#endif
};
void kvm_gmem_init(struct module *module)
@@ -489,6 +529,7 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
kvm_get_kvm(kvm);
gmem->kvm = kvm;
+ gmem->f_policy = NULL;
xa_init(&gmem->bindings);
list_add(&gmem->entry, &inode->i_mapping->i_private_list);
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 3/4] KVM: guest_memfd: Pass file pointer instead of inode in guest_memfd APIs
From: Shivank Garg @ 2024-11-05 16:55 UTC (permalink / raw)
To: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm
Cc: chao.gao, pgonda, thomas.lendacky, seanjc, luto, tglx, mingo, bp,
dave.hansen, willy, arnd, pbonzini, kees, shivankg, bharata,
nikunj, michael.day, Neeraj.Upadhyay
In-Reply-To: <20241105165515.154941-1-shivankg@amd.com>
Change the KVM guest_memfd APIs to pass file pointers instead of
inodes in the folio allocation functions. This is preparatory patch
for adding NUMA support to guest memory allocations.
The functional behavior remains unchanged.
Signed-off-by: Shivank Garg <shivankg@amd.com>
---
virt/kvm/guest_memfd.c | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index e930014b4bdc..2c6fcf7c3ec9 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -91,7 +91,7 @@ static struct folio *kvm_gmem_get_huge_folio(struct inode *inode, pgoff_t index,
{
pgoff_t npages = 1UL << order;
pgoff_t huge_index = round_down(index, npages);
- struct address_space *mapping = inode->i_mapping;
+ struct address_space *mapping = inode->i_mapping;
gfp_t gfp = mapping_gfp_mask(mapping) | __GFP_NOWARN;
loff_t size = i_size_read(inode);
struct folio *folio;
@@ -125,16 +125,16 @@ static struct folio *kvm_gmem_get_huge_folio(struct inode *inode, pgoff_t index,
* Ignore accessed, referenced, and dirty flags. The memory is
* unevictable and there is no storage to write back to.
*/
-static struct folio *__kvm_gmem_get_folio(struct inode *inode, pgoff_t index,
+static struct folio *__kvm_gmem_get_folio(struct file *file, pgoff_t index,
bool allow_huge)
{
struct folio *folio = NULL;
if (gmem_2m_enabled && allow_huge)
- folio = kvm_gmem_get_huge_folio(inode, index, PMD_ORDER);
+ folio = kvm_gmem_get_huge_folio(file_inode(file), index, PMD_ORDER);
if (!folio)
- folio = filemap_grab_folio(inode->i_mapping, index);
+ folio = filemap_grab_folio(file_inode(file)->i_mapping, index);
pr_debug("%s: allocate folio with PFN %lx order %d\n",
__func__, folio_pfn(folio), folio_order(folio));
@@ -150,9 +150,9 @@ static struct folio *__kvm_gmem_get_folio(struct inode *inode, pgoff_t index,
* Ignore accessed, referenced, and dirty flags. The memory is
* unevictable and there is no storage to write back to.
*/
-static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
+static struct folio *kvm_gmem_get_folio(struct file *file, pgoff_t index)
{
- return __kvm_gmem_get_folio(inode, index, true);
+ return __kvm_gmem_get_folio(file, index, true);
}
static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,
@@ -228,8 +228,9 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
return 0;
}
-static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
+static long kvm_gmem_allocate(struct file *file, loff_t offset, loff_t len)
{
+ struct inode *inode = file_inode(file);
struct address_space *mapping = inode->i_mapping;
pgoff_t start, index, end;
int r;
@@ -252,7 +253,7 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
break;
}
- folio = kvm_gmem_get_folio(inode, index);
+ folio = kvm_gmem_get_folio(file, index);
if (IS_ERR(folio)) {
r = PTR_ERR(folio);
break;
@@ -292,7 +293,7 @@ static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
if (mode & FALLOC_FL_PUNCH_HOLE)
ret = kvm_gmem_punch_hole(file_inode(file), offset, len);
else
- ret = kvm_gmem_allocate(file_inode(file), offset, len);
+ ret = kvm_gmem_allocate(file, offset, len);
if (!ret)
file_modified(file);
@@ -626,7 +627,7 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,
return ERR_PTR(-EIO);
}
- folio = __kvm_gmem_get_folio(file_inode(file), index, allow_huge);
+ folio = __kvm_gmem_get_folio(file, index, allow_huge);
if (IS_ERR(folio))
return folio;
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 2/4] Introduce fbind syscall
From: Shivank Garg @ 2024-11-05 16:55 UTC (permalink / raw)
To: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm
Cc: chao.gao, pgonda, thomas.lendacky, seanjc, luto, tglx, mingo, bp,
dave.hansen, willy, arnd, pbonzini, kees, shivankg, bharata,
nikunj, michael.day, Neeraj.Upadhyay, Shivansh Dhiman
In-Reply-To: <20241105164549.154700-1-shivankg@amd.com>
The new fbind syscall sets the NUMA memory policy for file-backed memory
and has following signature:
long fbind(unsigned int fd, unsigned long mode,
const unsigned long nodemask[(.maxnode + ULONG_WIDTH - 1)
/ ULONG_WIDTH],
unsigned long maxnode, unsigned int flags);
fbind behaves similar to mbind except that it takes file descriptor as
input instead of address ranges.
TODO:
1. Support fbind syscall on all architectures.
2. Expand commit msg and add documentation.
3. clean-up the code.
[Shivansh: add create_mpol_from_args()]
Signed-off-by: Shivansh Dhiman <shivansh.dhiman@amd.com>
Signed-off-by: Shivank Garg <shivankg@amd.com>
---
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
include/linux/fs.h | 3 ++
include/linux/mempolicy.h | 3 ++
include/linux/syscalls.h | 3 ++
include/uapi/asm-generic/unistd.h | 5 ++-
kernel/sys_ni.c | 1 +
mm/Makefile | 2 +-
mm/fbind.c | 49 +++++++++++++++++++++++
mm/mempolicy.c | 55 ++++++++++++++++++++++++++
10 files changed, 121 insertions(+), 2 deletions(-)
create mode 100644 mm/fbind.c
diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl
index 534c74b14fab..0660ce6d08d8 100644
--- a/arch/x86/entry/syscalls/syscall_32.tbl
+++ b/arch/x86/entry/syscalls/syscall_32.tbl
@@ -468,3 +468,4 @@
460 i386 lsm_set_self_attr sys_lsm_set_self_attr
461 i386 lsm_list_modules sys_lsm_list_modules
462 i386 mseal sys_mseal
+463 i386 fbind sys_fbind
diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl
index 7093ee21c0d1..9794347cc2e6 100644
--- a/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/arch/x86/entry/syscalls/syscall_64.tbl
@@ -386,6 +386,7 @@
460 common lsm_set_self_attr sys_lsm_set_self_attr
461 common lsm_list_modules sys_lsm_list_modules
462 common mseal sys_mseal
+463 common fbind sys_fbind
#
# Due to a historical design error, certain syscalls are numbered differently
diff --git a/include/linux/fs.h b/include/linux/fs.h
index fd34b5755c0b..42042b62bdcd 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2058,6 +2058,9 @@ struct file_operations {
struct file *file_out, loff_t pos_out,
loff_t len, unsigned int remap_flags);
int (*fadvise)(struct file *, loff_t, loff_t, int);
+#ifdef CONFIG_NUMA
+ int (*set_policy)(struct file *, struct mempolicy *);
+#endif
int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
unsigned int poll_flags);
diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h
index 1add16f21612..b9023f6246a7 100644
--- a/include/linux/mempolicy.h
+++ b/include/linux/mempolicy.h
@@ -299,4 +299,7 @@ static inline bool mpol_is_preferred_many(struct mempolicy *pol)
}
#endif /* CONFIG_NUMA */
+struct mempolicy *create_mpol_from_args(unsigned char mode,
+ const unsigned long __user *nmask,
+ unsigned short maxnode);
#endif
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index 4bcf6754738d..2dc686921b9f 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -502,6 +502,9 @@ asmlinkage long sys_readlinkat(int dfd, const char __user *path, char __user *bu
asmlinkage long sys_newfstatat(int dfd, const char __user *filename,
struct stat __user *statbuf, int flag);
asmlinkage long sys_newfstat(unsigned int fd, struct stat __user *statbuf);
+asmlinkage long sys_fbind(unsigned int fd, unsigned long mode,
+ const unsigned long __user *nmask,
+ unsigned long maxnode, unsigned int flags);
#if defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_COMPAT_STAT64)
asmlinkage long sys_fstat64(unsigned long fd, struct stat64 __user *statbuf);
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h
index 5bf6148cac2b..550730f36dae 100644
--- a/include/uapi/asm-generic/unistd.h
+++ b/include/uapi/asm-generic/unistd.h
@@ -841,8 +841,11 @@ __SYSCALL(__NR_lsm_list_modules, sys_lsm_list_modules)
#define __NR_mseal 462
__SYSCALL(__NR_mseal, sys_mseal)
+#define __NR_fbind 463
+__SYSCALL(__NR_fbind, sys_fbind)
+
#undef __NR_syscalls
-#define __NR_syscalls 463
+#define __NR_syscalls 464
/*
* 32 bit systems traditionally used different
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index c00a86931f8c..f57350e581f6 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -195,6 +195,7 @@ COND_SYSCALL(move_pages);
COND_SYSCALL(set_mempolicy_home_node);
COND_SYSCALL(cachestat);
COND_SYSCALL(mseal);
+COND_SYSCALL(fbind);
COND_SYSCALL(perf_event_open);
COND_SYSCALL(accept4);
diff --git a/mm/Makefile b/mm/Makefile
index d2915f8c9dc0..ba339ddc0be2 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -79,7 +79,7 @@ obj-$(CONFIG_ZSWAP) += zswap.o
obj-$(CONFIG_HAS_DMA) += dmapool.o
obj-$(CONFIG_HUGETLBFS) += hugetlb.o
obj-$(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) += hugetlb_vmemmap.o
-obj-$(CONFIG_NUMA) += mempolicy.o
+obj-$(CONFIG_NUMA) += mempolicy.o fbind.o
obj-$(CONFIG_SPARSEMEM) += sparse.o
obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o
obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
diff --git a/mm/fbind.c b/mm/fbind.c
new file mode 100644
index 000000000000..85ec7d13345c
--- /dev/null
+++ b/mm/fbind.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Implement fbind() syscall.
+ *
+ * Copyright (c) 2024 AMD
+ *
+ * Author: Shivank Garg <shivankg@amd.com>
+ */
+
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/mempolicy.h>
+#include <linux/syscalls.h>
+
+static long do_fbind(unsigned int fd, unsigned long mode,
+ const unsigned long __user *nmask,
+ unsigned long maxnode, unsigned int flags)
+{
+ struct mempolicy *mpol;
+ struct fd f;
+ int ret;
+
+ f = fdget(fd);
+ if (!f.file)
+ return -EBADF;
+
+ mpol = create_mpol_from_args(mode, nmask, maxnode);
+ if (IS_ERR_OR_NULL(mpol)) {
+ ret = PTR_ERR(mpol);
+ goto out_putf;
+ }
+
+ if (f.file->f_op->set_policy)
+ ret = f.file->f_op->set_policy(f.file, mpol);
+ else
+ ret = -EOPNOTSUPP;
+
+ mpol_put(mpol);
+out_putf:
+ fdput(f);
+ return ret;
+}
+
+SYSCALL_DEFINE5(fbind, unsigned int, fd, unsigned long, mode,
+ const unsigned long __user *, nmask,
+ unsigned long, maxnode, unsigned int, flags)
+{
+ return do_fbind(fd, mode, nmask, maxnode, flags);
+}
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index b858e22b259d..3a697080ecad 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -3557,3 +3557,58 @@ static int __init mempolicy_sysfs_init(void)
late_initcall(mempolicy_sysfs_init);
#endif /* CONFIG_SYSFS */
+
+/**
+ * create_mpol_from_args - create a mempolicy structure from args
+ * @mode: NUMA memory policy mode
+ * @nmask: bitmask of NUMA nodes
+ * @maxnode: number of bits in the nodes bitmask
+ *
+ * Create a mempolicy from given nodemask and memory policy such as
+ * default, preferred, interleave or bind.
+ *
+ * Return: error encoded in a pointer or memory policy on success.
+ */
+struct mempolicy *create_mpol_from_args(unsigned char mode,
+ const unsigned long __user *nmask,
+ unsigned short maxnode)
+{
+ struct mm_struct *mm = current->mm;
+ unsigned short mode_flags;
+ struct mempolicy *mpol;
+ nodemask_t nodes;
+ int lmode = mode;
+ int err = -ENOMEM;
+
+ err = sanitize_mpol_flags(&lmode, &mode_flags);
+ if (err)
+ return ERR_PTR(err);
+
+ err = get_nodes(&nodes, nmask, maxnode);
+ if (err)
+ return ERR_PTR(err);
+
+ mpol = mpol_new(mode, mode_flags, &nodes);
+ if (IS_ERR_OR_NULL(mpol))
+ return mpol;
+
+ NODEMASK_SCRATCH(scratch);
+ if (!scratch) {
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ mmap_write_lock(mm);
+ err = mpol_set_nodemask(mpol, &nodes, scratch);
+ mmap_write_unlock(mm);
+ NODEMASK_SCRATCH_FREE(scratch);
+
+ if (err)
+ goto err_out;
+
+ return mpol;
+
+err_out:
+ mpol_put(mpol);
+ return ERR_PTR(err);
+}
--
2.34.1
^ permalink raw reply related
* [RFC PATCH 1/4] mm: Add mempolicy support to the filemap layer
From: Shivank Garg @ 2024-11-05 16:45 UTC (permalink / raw)
To: x86, viro, brauner, jack, akpm, linux-kernel, linux-fsdevel,
linux-mm, linux-api, linux-arch, kvm
Cc: chao.gao, pgonda, thomas.lendacky, seanjc, luto, tglx, mingo, bp,
dave.hansen, willy, arnd, pbonzini, kees, shivankg, bharata,
nikunj, michael.day, Neeraj.Upadhyay, Shivansh Dhiman
In-Reply-To: <20241105164549.154700-1-shivankg@amd.com>
From: Shivansh Dhiman <shivansh.dhiman@amd.com>
Introduce mempolicy support to the filemap. Add filemap_grab_folio_mpol,
filemap_alloc_folio_mpol_noprof() and __filemap_get_folio_mpol() APIs that
take mempolicy struct as an argument.
The API is required by VMs using KVM guest-memfd memory backends for NUMA
mempolicy aware allocations.
Signed-off-by: Shivansh Dhiman <shivansh.dhiman@amd.com>
Signed-off-by: Shivank Garg <shivankg@amd.com>
---
include/linux/pagemap.h | 40 ++++++++++++++++++++++++++++++++++++++++
mm/filemap.c | 30 +++++++++++++++++++++++++-----
2 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index d9c7edb6422b..b05b696f310b 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -564,15 +564,25 @@ static inline void *detach_page_private(struct page *page)
#ifdef CONFIG_NUMA
struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order);
+struct folio *filemap_alloc_folio_mpol_noprof(gfp_t gfp, unsigned int order,
+ struct mempolicy *mpol);
#else
static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)
{
return folio_alloc_noprof(gfp, order);
}
+static inline struct folio *filemap_alloc_folio_mpol_noprof(gfp_t gfp,
+ unsigned int order,
+ struct mempolicy *mpol)
+{
+ return filemap_alloc_folio_noprof(gfp, order);
+}
#endif
#define filemap_alloc_folio(...) \
alloc_hooks(filemap_alloc_folio_noprof(__VA_ARGS__))
+#define filemap_alloc_folio_mpol(...) \
+ alloc_hooks(filemap_alloc_folio_mpol_noprof(__VA_ARGS__))
static inline struct page *__page_cache_alloc(gfp_t gfp)
{
@@ -652,6 +662,8 @@ static inline fgf_t fgf_set_order(size_t size)
void *filemap_get_entry(struct address_space *mapping, pgoff_t index);
struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
fgf_t fgp_flags, gfp_t gfp);
+struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
+ pgoff_t index, fgf_t fgp_flags, gfp_t gfp, struct mempolicy *mpol);
struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
fgf_t fgp_flags, gfp_t gfp);
@@ -710,6 +722,34 @@ static inline struct folio *filemap_grab_folio(struct address_space *mapping,
mapping_gfp_mask(mapping));
}
+/**
+ * filemap_grab_folio_mpol - grab a folio from the page cache
+ * @mapping: The address space to search
+ * @index: The page index
+ * @mpol: The mempolicy to apply
+ *
+ * Same as filemap_grab_folio(), except that it allocates the folio using
+ * given memory policy.
+ *
+ * Return: A found or created folio. ERR_PTR(-ENOMEM) if no folio is found
+ * and failed to create a folio.
+ */
+#ifdef CONFIG_NUMA
+static inline struct folio *filemap_grab_folio_mpol(struct address_space *mapping,
+ pgoff_t index, struct mempolicy *mpol)
+{
+ return __filemap_get_folio_mpol(mapping, index,
+ FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
+ mapping_gfp_mask(mapping), mpol);
+}
+#else
+static inline struct folio *filemap_grab_folio_mpol(struct address_space *mapping,
+ pgoff_t index, struct mempolicy *mpol)
+{
+ return filemap_grab_folio(mapping, index);
+}
+#endif /* CONFIG_NUMA */
+
/**
* find_get_page - find and get a page reference
* @mapping: the address_space to search
diff --git a/mm/filemap.c b/mm/filemap.c
index d62150418b91..a870a05296c8 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -990,8 +990,13 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio,
EXPORT_SYMBOL_GPL(filemap_add_folio);
#ifdef CONFIG_NUMA
-struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)
+struct folio *filemap_alloc_folio_mpol_noprof(gfp_t gfp, unsigned int order,
+ struct mempolicy *mpol)
{
+ if (mpol)
+ return folio_alloc_mpol_noprof(gfp, order, mpol,
+ NO_INTERLEAVE_INDEX, numa_node_id());
+
int n;
struct folio *folio;
@@ -1007,6 +1012,12 @@ struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)
}
return folio_alloc_noprof(gfp, order);
}
+EXPORT_SYMBOL(filemap_alloc_folio_mpol_noprof);
+
+struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)
+{
+ return filemap_alloc_folio_mpol_noprof(gfp, order, NULL);
+}
EXPORT_SYMBOL(filemap_alloc_folio_noprof);
#endif
@@ -1861,11 +1872,12 @@ void *filemap_get_entry(struct address_space *mapping, pgoff_t index)
}
/**
- * __filemap_get_folio - Find and get a reference to a folio.
+ * __filemap_get_folio_mpol - Find and get a reference to a folio.
* @mapping: The address_space to search.
* @index: The page index.
* @fgp_flags: %FGP flags modify how the folio is returned.
* @gfp: Memory allocation flags to use if %FGP_CREAT is specified.
+ * @mpol: The mempolicy to apply.
*
* Looks up the page cache entry at @mapping & @index.
*
@@ -1876,8 +1888,8 @@ void *filemap_get_entry(struct address_space *mapping, pgoff_t index)
*
* Return: The found folio or an ERR_PTR() otherwise.
*/
-struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
- fgf_t fgp_flags, gfp_t gfp)
+struct folio *__filemap_get_folio_mpol(struct address_space *mapping, pgoff_t index,
+ fgf_t fgp_flags, gfp_t gfp, struct mempolicy *mpol)
{
struct folio *folio;
@@ -1947,7 +1959,7 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
err = -ENOMEM;
if (order > 0)
alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
- folio = filemap_alloc_folio(alloc_gfp, order);
+ folio = filemap_alloc_folio_mpol(alloc_gfp, order, mpol);
if (!folio)
continue;
@@ -1978,6 +1990,14 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
return ERR_PTR(-ENOENT);
return folio;
}
+EXPORT_SYMBOL(__filemap_get_folio_mpol);
+
+struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
+ fgf_t fgp_flags, gfp_t gfp)
+{
+ return __filemap_get_folio_mpol(mapping, index,
+ fgp_flags, gfp, NULL);
+}
EXPORT_SYMBOL(__filemap_get_folio);
static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
--
2.34.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox