* [PATCH net-next 3/5] netvsc: optimize receive completions
From: Stephen Hemminger @ 2017-07-25 20:04 UTC (permalink / raw)
To: kys, haiyangz, sthemmin; +Cc: devel, netdev
In-Reply-To: <20170725200422.13795-1-sthemmin@microsoft.com>
Optimize how receive completion ring are managed.
* Allocate only as many slots as needed for all buffers from host
* Allocate before setting up sub channel for better error detection
* Don't need to keep copy of initial receive section message
* Only needt keep the transaction id, status doesn't matter
* Precompute the watermark for when receive flushing is needed
* Replace division with conditional test
* Replace atomic per-device variable with per-channel check.
* Handle corner case where receive completion send
fails if ring buffer to host is full.
Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
drivers/net/hyperv/hyperv_net.h | 19 +--
drivers/net/hyperv/netvsc.c | 270 +++++++++++++++-----------------------
drivers/net/hyperv/rndis_filter.c | 20 +--
3 files changed, 125 insertions(+), 184 deletions(-)
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index fb62ea632914..b0259b12c5ee 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -186,6 +186,7 @@ struct net_device_context;
struct netvsc_device *netvsc_device_add(struct hv_device *device,
const struct netvsc_device_info *info);
+int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx);
void netvsc_device_remove(struct hv_device *device);
int netvsc_send(struct net_device_context *ndc,
struct hv_netvsc_packet *packet,
@@ -652,18 +653,10 @@ struct multi_send_data {
u32 count; /* counter of batched packets */
};
-struct recv_comp_data {
- u64 tid; /* transaction id */
- u32 status;
-};
-
-/* Netvsc Receive Slots Max */
-#define NETVSC_RECVSLOT_MAX (NETVSC_RECEIVE_BUFFER_SIZE / ETH_DATA_LEN + 1)
-
struct multi_recv_comp {
- void *buf; /* queued receive completions */
- u32 first; /* first data entry */
- u32 next; /* next entry for writing */
+ u64 *slots; /* pending completion transactions */
+ u32 first; /* first data entry */
+ u32 next; /* next entry for writing */
};
struct netvsc_stats {
@@ -750,7 +743,7 @@ struct netvsc_device {
u32 recv_buf_size;
u32 recv_buf_gpadl_handle;
u32 recv_section_cnt;
- struct nvsp_1_receive_buffer_section *recv_section;
+ u32 recv_completion_cnt;
/* Send buffer allocated by us */
void *send_buf;
@@ -778,8 +771,6 @@ struct netvsc_device {
u32 max_pkt; /* max number of pkt in one send, e.g. 8 */
u32 pkt_align; /* alignment bytes, e.g. 8 */
- atomic_t num_outstanding_recvs;
-
atomic_t open_cnt;
struct netvsc_channel chan_table[VRSS_CHANNEL_MAX];
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 94c00acac58a..03c1fec762c7 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -72,9 +72,6 @@ static struct netvsc_device *alloc_net_device(void)
if (!net_device)
return NULL;
- net_device->chan_table[0].mrc.buf
- = vzalloc(NETVSC_RECVSLOT_MAX * sizeof(struct recv_comp_data));
-
init_waitqueue_head(&net_device->wait_drain);
net_device->destroy = false;
atomic_set(&net_device->open_cnt, 0);
@@ -92,7 +89,7 @@ static void free_netvsc_device(struct rcu_head *head)
int i;
for (i = 0; i < VRSS_CHANNEL_MAX; i++)
- vfree(nvdev->chan_table[i].mrc.buf);
+ vfree(nvdev->chan_table[i].mrc.slots);
kfree(nvdev);
}
@@ -171,12 +168,6 @@ static void netvsc_destroy_buf(struct hv_device *device)
net_device->recv_buf = NULL;
}
- if (net_device->recv_section) {
- net_device->recv_section_cnt = 0;
- kfree(net_device->recv_section);
- net_device->recv_section = NULL;
- }
-
/* Deal with the send buffer we may have setup.
* If we got a send section size, it means we received a
* NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent
@@ -239,11 +230,26 @@ static void netvsc_destroy_buf(struct hv_device *device)
kfree(net_device->send_section_map);
}
+int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)
+{
+ struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
+ int node = cpu_to_node(nvchan->channel->target_cpu);
+ size_t size;
+
+ size = net_device->recv_completion_cnt * sizeof(u64);
+ nvchan->mrc.slots = vzalloc_node(size, node);
+ if (!nvchan->mrc.slots)
+ nvchan->mrc.slots = vzalloc(size);
+
+ return nvchan->mrc.slots ? 0 : -ENOMEM;
+}
+
static int netvsc_init_buf(struct hv_device *device,
struct netvsc_device *net_device)
{
int ret = 0;
struct nvsp_message *init_packet;
+ struct nvsp_1_message_send_receive_buffer_complete *resp;
struct net_device *ndev;
size_t map_words;
int node;
@@ -300,43 +306,41 @@ static int netvsc_init_buf(struct hv_device *device,
wait_for_completion(&net_device->channel_init_wait);
/* Check the response */
- if (init_packet->msg.v1_msg.
- send_recv_buf_complete.status != NVSP_STAT_SUCCESS) {
- netdev_err(ndev, "Unable to complete receive buffer "
- "initialization with NetVsp - status %d\n",
- init_packet->msg.v1_msg.
- send_recv_buf_complete.status);
+ resp = &init_packet->msg.v1_msg.send_recv_buf_complete;
+ if (resp->status != NVSP_STAT_SUCCESS) {
+ netdev_err(ndev,
+ "Unable to complete receive buffer initialization with NetVsp - status %d\n",
+ resp->status);
ret = -EINVAL;
goto cleanup;
}
/* Parse the response */
+ netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n",
+ resp->num_sections, resp->sections[0].sub_alloc_size,
+ resp->sections[0].num_sub_allocs);
- net_device->recv_section_cnt = init_packet->msg.
- v1_msg.send_recv_buf_complete.num_sections;
-
- net_device->recv_section = kmemdup(
- init_packet->msg.v1_msg.send_recv_buf_complete.sections,
- net_device->recv_section_cnt *
- sizeof(struct nvsp_1_receive_buffer_section),
- GFP_KERNEL);
- if (net_device->recv_section == NULL) {
- ret = -EINVAL;
- goto cleanup;
- }
+ net_device->recv_section_cnt = resp->num_sections;
/*
* For 1st release, there should only be 1 section that represents the
* entire receive buffer
*/
if (net_device->recv_section_cnt != 1 ||
- net_device->recv_section->offset != 0) {
+ resp->sections[0].offset != 0) {
ret = -EINVAL;
goto cleanup;
}
- /* Now setup the send buffer.
- */
+ /* Setup receive completion ring */
+ net_device->recv_completion_cnt
+ = round_up(resp->sections[0].num_sub_allocs + 1,
+ PAGE_SIZE / sizeof(u64));
+ ret = netvsc_alloc_recv_comp_ring(net_device, 0);
+ if (ret)
+ goto cleanup;
+
+ /* Now setup the send buffer. */
net_device->send_buf = vzalloc_node(net_device->send_buf_size, node);
if (!net_device->send_buf)
net_device->send_buf = vzalloc(net_device->send_buf_size);
@@ -950,139 +954,96 @@ int netvsc_send(struct net_device_context *ndev_ctx,
return ret;
}
-static int netvsc_send_recv_completion(struct vmbus_channel *channel,
- u64 transaction_id, u32 status)
-{
- struct nvsp_message recvcompMessage;
- int ret;
-
- recvcompMessage.hdr.msg_type =
- NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE;
-
- recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status;
-
- /* Send the completion */
- ret = vmbus_sendpacket(channel, &recvcompMessage,
- sizeof(struct nvsp_message_header) + sizeof(u32),
- transaction_id, VM_PKT_COMP, 0);
-
- return ret;
-}
-
-static inline void count_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx,
- u32 *filled, u32 *avail)
+/* Send pending recv completions */
+static int send_recv_completions(struct netvsc_device *nvdev,
+ struct vmbus_channel *channel, u16 q_idx)
{
struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
- u32 first = mrc->first;
- u32 next = mrc->next;
+ struct recv_comp_msg {
+ struct nvsp_message_header hdr;
+ u32 status;
+ } __packed;
+ struct recv_comp_msg msg = {
+ .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE,
+ .status = NVSP_STAT_SUCCESS,
+ };
+ int ret;
- *filled = (first > next) ? NETVSC_RECVSLOT_MAX - first + next :
- next - first;
+ while (mrc->first != mrc->next) {
+ u64 tid = mrc->slots[mrc->first];
- *avail = NETVSC_RECVSLOT_MAX - *filled - 1;
-}
+ ret = vmbus_sendpacket(channel, &msg, sizeof(msg),
+ tid, VM_PKT_COMP, 0);
+ if (unlikely(ret))
+ return ret;
-/* Read the first filled slot, no change to index */
-static inline struct recv_comp_data *read_recv_comp_slot(struct netvsc_device
- *nvdev, u16 q_idx)
-{
- struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
- u32 filled, avail;
-
- if (unlikely(!mrc->buf))
- return NULL;
+ if (++mrc->first == nvdev->recv_completion_cnt)
+ mrc->first = 0;
+ }
- count_recv_comp_slot(nvdev, q_idx, &filled, &avail);
- if (!filled)
- return NULL;
+ /* receive completion ring has been emptied */
+ if (unlikely(nvdev->destroy))
+ wake_up(&nvdev->wait_drain);
- return mrc->buf + mrc->first * sizeof(struct recv_comp_data);
+ return 0;
}
-/* Put the first filled slot back to available pool */
-static inline void put_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx)
+/* Count how many receive completions are outstanding */
+static void recv_comp_slot_avail(const struct netvsc_device *nvdev,
+ const struct multi_recv_comp *mrc,
+ u32 *filled, u32 *avail)
{
- struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
- int num_recv;
+ u32 count = nvdev->recv_completion_cnt;
- mrc->first = (mrc->first + 1) % NETVSC_RECVSLOT_MAX;
-
- num_recv = atomic_dec_return(&nvdev->num_outstanding_recvs);
+ if (mrc->next >= mrc->first)
+ *filled = mrc->next - mrc->first;
+ else
+ *filled = (count - mrc->first) + mrc->next;
- if (nvdev->destroy && num_recv == 0)
- wake_up(&nvdev->wait_drain);
+ *avail = count - *filled - 1;
}
-/* Check and send pending recv completions */
-static void netvsc_chk_recv_comp(struct netvsc_device *nvdev,
- struct vmbus_channel *channel, u16 q_idx)
+/* Add receive complete to ring to send to host. */
+static void enq_receive_complete(struct net_device *ndev,
+ struct netvsc_device *nvdev, u16 q_idx,
+ u64 tid)
{
- struct recv_comp_data *rcd;
- int ret;
-
- while (true) {
- rcd = read_recv_comp_slot(nvdev, q_idx);
- if (!rcd)
- break;
+ struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx];
+ struct multi_recv_comp *mrc = &nvchan->mrc;
+ u32 filled, avail;
- ret = netvsc_send_recv_completion(channel, rcd->tid,
- rcd->status);
- if (ret)
- break;
+ recv_comp_slot_avail(nvdev, mrc, &filled, &avail);
- put_recv_comp_slot(nvdev, q_idx);
+ if (unlikely(filled > NAPI_POLL_WEIGHT)) {
+ send_recv_completions(nvdev, nvchan->channel, q_idx);
+ recv_comp_slot_avail(nvdev, mrc, &filled, &avail);
}
-}
-
-#define NETVSC_RCD_WATERMARK 80
-
-/* Get next available slot */
-static inline struct recv_comp_data *get_recv_comp_slot(
- struct netvsc_device *nvdev, struct vmbus_channel *channel, u16 q_idx)
-{
- struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
- u32 filled, avail, next;
- struct recv_comp_data *rcd;
-
- if (unlikely(!nvdev->recv_section))
- return NULL;
-
- if (unlikely(!mrc->buf))
- return NULL;
-
- if (atomic_read(&nvdev->num_outstanding_recvs) >
- nvdev->recv_section->num_sub_allocs * NETVSC_RCD_WATERMARK / 100)
- netvsc_chk_recv_comp(nvdev, channel, q_idx);
-
- count_recv_comp_slot(nvdev, q_idx, &filled, &avail);
- if (!avail)
- return NULL;
-
- next = mrc->next;
- rcd = mrc->buf + next * sizeof(struct recv_comp_data);
- mrc->next = (next + 1) % NETVSC_RECVSLOT_MAX;
- atomic_inc(&nvdev->num_outstanding_recvs);
+ if (unlikely(!avail)) {
+ netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",
+ q_idx, tid);
+ return;
+ }
- return rcd;
+ mrc->slots[mrc->next] = tid;
+ if (++mrc->next == nvdev->recv_completion_cnt)
+ mrc->next = 0;
}
static int netvsc_receive(struct net_device *ndev,
- struct netvsc_device *net_device,
- struct net_device_context *net_device_ctx,
- struct hv_device *device,
- struct vmbus_channel *channel,
- const struct vmpacket_descriptor *desc,
- struct nvsp_message *nvsp)
+ struct netvsc_device *net_device,
+ struct net_device_context *net_device_ctx,
+ struct hv_device *device,
+ struct vmbus_channel *channel,
+ const struct vmpacket_descriptor *desc,
+ struct nvsp_message *nvsp)
{
const struct vmtransfer_page_packet_header *vmxferpage_packet
= container_of(desc, const struct vmtransfer_page_packet_header, d);
u16 q_idx = channel->offermsg.offer.sub_channel_index;
char *recv_buf = net_device->recv_buf;
- u32 status = NVSP_STAT_SUCCESS;
int i;
int count = 0;
- int ret;
/* Make sure this is a valid nvsp packet */
if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) {
@@ -1109,29 +1070,13 @@ static int netvsc_receive(struct net_device *ndev,
u32 buflen = vmxferpage_packet->ranges[i].byte_count;
/* Pass it to the upper layer */
- status = rndis_filter_receive(ndev, net_device, device,
- channel, data, buflen);
+ rndis_filter_receive(ndev, net_device, device,
+ channel, data, buflen);
}
- if (net_device->chan_table[q_idx].mrc.buf) {
- struct recv_comp_data *rcd;
+ enq_receive_complete(ndev, net_device, q_idx,
+ vmxferpage_packet->d.trans_id);
- rcd = get_recv_comp_slot(net_device, channel, q_idx);
- if (rcd) {
- rcd->tid = vmxferpage_packet->d.trans_id;
- rcd->status = status;
- } else {
- netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",
- q_idx, vmxferpage_packet->d.trans_id);
- }
- } else {
- ret = netvsc_send_recv_completion(channel,
- vmxferpage_packet->d.trans_id,
- status);
- if (ret)
- netdev_err(ndev, "Recv_comp q:%hd, tid:%llx, err:%d\n",
- q_idx, vmxferpage_packet->d.trans_id, ret);
- }
return count;
}
@@ -1244,17 +1189,18 @@ int netvsc_poll(struct napi_struct *napi, int budget)
nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc);
}
- /* If receive ring was exhausted
- * and not doing busy poll
- * then re-enable host interrupts
- * and reschedule if ring is not empty.
+ /* If send of pending receive completions suceeded
+ * and did not exhaust NAPI budget
+ * and not doing busy poll
+ * then reschedule if more data has arrived from host
*/
- if (work_done < budget &&
+ if (send_recv_completions(net_device, channel, q_idx) == 0 &&
+ work_done < budget &&
napi_complete_done(napi, work_done) &&
- hv_end_read(&channel->inbound) != 0)
+ hv_end_read(&channel->inbound)) {
+ hv_begin_read(&channel->inbound);
napi_reschedule(napi);
-
- netvsc_chk_recv_comp(net_device, channel, q_idx);
+ }
/* Driver may overshoot since multiple packets per descriptor */
return min(work_done, budget);
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index bf21ea92c743..db9bac31dd09 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -928,12 +928,12 @@ static bool netvsc_device_idle(const struct netvsc_device *nvdev)
{
int i;
- if (atomic_read(&nvdev->num_outstanding_recvs) > 0)
- return false;
-
for (i = 0; i < nvdev->num_chn; i++) {
const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
+ if (nvchan->mrc.first != nvchan->mrc.next)
+ return false;
+
if (atomic_read(&nvchan->queue_sends) > 0)
return false;
}
@@ -1031,11 +1031,6 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
return;
nvchan = nvscdev->chan_table + chn_index;
- nvchan->mrc.buf
- = vzalloc(NETVSC_RECVSLOT_MAX * sizeof(struct recv_comp_data));
-
- if (!nvchan->mrc.buf)
- return;
/* Because the device uses NAPI, all the interrupt batching and
* control is done via Net softirq, not the channel handling
@@ -1225,6 +1220,15 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
if (num_rss_qs == 0)
return net_device;
+ for (i = 1; i < net_device->num_chn; i++) {
+ ret = netvsc_alloc_recv_comp_ring(net_device, i);
+ if (ret) {
+ while (--i != 0)
+ vfree(net_device->chan_table[i].mrc.slots);
+ goto out;
+ }
+ }
+
refcount_set(&net_device->sc_offered, num_rss_qs);
vmbus_set_sc_create_callback(dev->channel, netvsc_sc_open);
--
2.11.0
^ permalink raw reply related
* [PATCH net-next 0/5] netvsc: fixes and performance related changes
From: Stephen Hemminger @ 2017-07-25 20:04 UTC (permalink / raw)
To: kys, haiyangz, sthemmin; +Cc: devel, netdev
The first two are more fixes to problems introduced in
latest net-next changes. The rest are changes to improve performance
and reduce memory footprint. The driver used to allocate large
amounts of memory (33M) per interface, this reduces that down to
about 7M.
Stephen Hemminger (5):
netvsc: fix return value for set_channels
netvsc: fix warnings reported by lockdep
netvsc: optimize receive completions
netvsc: signal host if receive ring is emptied
netvsc: allow smaller send/recv buffer size
drivers/net/hyperv/hyperv_net.h | 24 ++--
drivers/net/hyperv/netvsc.c | 293 ++++++++++++++++----------------------
drivers/net/hyperv/netvsc_drv.c | 48 ++++++-
drivers/net/hyperv/rndis_filter.c | 104 +++++++-------
4 files changed, 228 insertions(+), 241 deletions(-)
--
2.11.0
^ permalink raw reply
* [PATCH net-next 2/5] netvsc: fix warnings reported by lockdep
From: Stephen Hemminger @ 2017-07-25 20:04 UTC (permalink / raw)
To: kys, haiyangz, sthemmin; +Cc: devel, netdev
In-Reply-To: <20170725200422.13795-1-sthemmin@microsoft.com>
This includes a bunch of fixups for issues reported by sparse and
lockdep.
* ethtool routines can assume RTNL
* send is done with RCU lock (and BH disable)
Most of the changes involve passing internal device struct (netvsc)
as parameter, so that code called in both initialization and changes
to MTU and channels doesn't have to worry about whether the value
is protected by RTNL, RCU, or the fact it isn't registered yet.
One special case is the callback that happens when a new VMBUS
channel is added. The setup code is waiting for completion with
RTNL held, so it is already safe.
Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
drivers/net/hyperv/hyperv_net.h | 3 +-
drivers/net/hyperv/netvsc.c | 2 +-
drivers/net/hyperv/netvsc_drv.c | 15 ++++---
drivers/net/hyperv/rndis_filter.c | 84 +++++++++++++++++++--------------------
4 files changed, 54 insertions(+), 50 deletions(-)
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 4e7ff348327e..fb62ea632914 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -217,7 +217,8 @@ int rndis_filter_receive(struct net_device *ndev,
struct vmbus_channel *channel,
void *data, u32 buflen);
-int rndis_filter_set_device_mac(struct net_device *ndev, char *mac);
+int rndis_filter_set_device_mac(struct netvsc_device *ndev,
+ const char *mac);
void netvsc_switch_datapath(struct net_device *nv_dev, bool vf);
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 06f39a99da7c..94c00acac58a 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -832,7 +832,7 @@ int netvsc_send(struct net_device_context *ndev_ctx,
struct sk_buff *skb)
{
struct netvsc_device *net_device
- = rcu_dereference_rtnl(ndev_ctx->nvdev);
+ = rcu_dereference_bh(ndev_ctx->nvdev);
struct hv_device *device = ndev_ctx->device_ctx;
int ret = 0;
struct netvsc_channel *nvchan;
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index f1eaf675d2e9..a04f2efbbc25 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -923,6 +923,8 @@ static void netvsc_get_stats64(struct net_device *net,
static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
{
+ struct net_device_context *ndc = netdev_priv(ndev);
+ struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
struct sockaddr *addr = p;
char save_adr[ETH_ALEN];
unsigned char save_aatype;
@@ -935,7 +937,10 @@ static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
if (err != 0)
return err;
- err = rndis_filter_set_device_mac(ndev, addr->sa_data);
+ if (!nvdev)
+ return -ENODEV;
+
+ err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
if (err != 0) {
/* roll back to saved MAC */
memcpy(ndev->dev_addr, save_adr, ETH_ALEN);
@@ -981,7 +986,7 @@ static void netvsc_get_ethtool_stats(struct net_device *dev,
struct ethtool_stats *stats, u64 *data)
{
struct net_device_context *ndc = netdev_priv(dev);
- struct netvsc_device *nvdev = rcu_dereference(ndc->nvdev);
+ struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
const void *nds = &ndc->eth_stats;
const struct netvsc_stats *qstats;
unsigned int start;
@@ -1019,7 +1024,7 @@ static void netvsc_get_ethtool_stats(struct net_device *dev,
static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
{
struct net_device_context *ndc = netdev_priv(dev);
- struct netvsc_device *nvdev = rcu_dereference(ndc->nvdev);
+ struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
u8 *p = data;
int i;
@@ -1077,7 +1082,7 @@ netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
u32 *rules)
{
struct net_device_context *ndc = netdev_priv(dev);
- struct netvsc_device *nvdev = rcu_dereference(ndc->nvdev);
+ struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
if (!nvdev)
return -ENODEV;
@@ -1127,7 +1132,7 @@ static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
u8 *hfunc)
{
struct net_device_context *ndc = netdev_priv(dev);
- struct netvsc_device *ndev = rcu_dereference(ndc->nvdev);
+ struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
struct rndis_device *rndis_dev;
int i;
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index eaa3f0d5682a..bf21ea92c743 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -85,14 +85,6 @@ static struct rndis_device *get_rndis_device(void)
return device;
}
-static struct netvsc_device *
-net_device_to_netvsc_device(struct net_device *ndev)
-{
- struct net_device_context *net_device_ctx = netdev_priv(ndev);
-
- return rtnl_dereference(net_device_ctx->nvdev);
-}
-
static struct rndis_request *get_rndis_request(struct rndis_device *dev,
u32 msg_type,
u32 msg_len)
@@ -252,7 +244,10 @@ static int rndis_filter_send_request(struct rndis_device *dev,
pb[0].len;
}
+ rcu_read_lock_bh();
ret = netvsc_send(net_device_ctx, packet, NULL, &pb, NULL);
+ rcu_read_unlock_bh();
+
return ret;
}
@@ -452,8 +447,9 @@ int rndis_filter_receive(struct net_device *ndev,
return 0;
}
-static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
- void *result, u32 *result_size)
+static int rndis_filter_query_device(struct rndis_device *dev,
+ struct netvsc_device *nvdev,
+ u32 oid, void *result, u32 *result_size)
{
struct rndis_request *request;
u32 inresult_size = *result_size;
@@ -480,8 +476,6 @@ static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
query->dev_vc_handle = 0;
if (oid == OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES) {
- struct net_device_context *ndevctx = netdev_priv(dev->ndev);
- struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
struct ndis_offload *hwcaps;
u32 nvsp_version = nvdev->nvsp_version;
u8 ndis_rev;
@@ -550,14 +544,15 @@ static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
/* Get the hardware offload capabilities */
static int
-rndis_query_hwcaps(struct rndis_device *dev, struct ndis_offload *caps)
+rndis_query_hwcaps(struct rndis_device *dev, struct netvsc_device *net_device,
+ struct ndis_offload *caps)
{
u32 caps_len = sizeof(*caps);
int ret;
memset(caps, 0, sizeof(*caps));
- ret = rndis_filter_query_device(dev,
+ ret = rndis_filter_query_device(dev, net_device,
OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
caps, &caps_len);
if (ret)
@@ -586,11 +581,12 @@ rndis_query_hwcaps(struct rndis_device *dev, struct ndis_offload *caps)
return 0;
}
-static int rndis_filter_query_device_mac(struct rndis_device *dev)
+static int rndis_filter_query_device_mac(struct rndis_device *dev,
+ struct netvsc_device *net_device)
{
u32 size = ETH_ALEN;
- return rndis_filter_query_device(dev,
+ return rndis_filter_query_device(dev, net_device,
RNDIS_OID_802_3_PERMANENT_ADDRESS,
dev->hw_mac_adr, &size);
}
@@ -598,9 +594,9 @@ static int rndis_filter_query_device_mac(struct rndis_device *dev)
#define NWADR_STR "NetworkAddress"
#define NWADR_STRLEN 14
-int rndis_filter_set_device_mac(struct net_device *ndev, char *mac)
+int rndis_filter_set_device_mac(struct netvsc_device *nvdev,
+ const char *mac)
{
- struct netvsc_device *nvdev = net_device_to_netvsc_device(ndev);
struct rndis_device *rdev = nvdev->extension;
struct rndis_request *request;
struct rndis_set_request *set;
@@ -654,11 +650,8 @@ int rndis_filter_set_device_mac(struct net_device *ndev, char *mac)
wait_for_completion(&request->wait_event);
set_complete = &request->response_msg.msg.set_complete;
- if (set_complete->status != RNDIS_STATUS_SUCCESS) {
- netdev_err(ndev, "Fail to set MAC on host side:0x%x\n",
- set_complete->status);
- ret = -EINVAL;
- }
+ if (set_complete->status != RNDIS_STATUS_SUCCESS)
+ ret = -EIO;
cleanup:
put_rndis_request(rdev, request);
@@ -791,27 +784,27 @@ int rndis_filter_set_rss_param(struct rndis_device *rdev,
return ret;
}
-static int rndis_filter_query_device_link_status(struct rndis_device *dev)
+static int rndis_filter_query_device_link_status(struct rndis_device *dev,
+ struct netvsc_device *net_device)
{
u32 size = sizeof(u32);
u32 link_status;
- int ret;
- ret = rndis_filter_query_device(dev,
- RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
- &link_status, &size);
-
- return ret;
+ return rndis_filter_query_device(dev, net_device,
+ RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
+ &link_status, &size);
}
-static int rndis_filter_query_link_speed(struct rndis_device *dev)
+static int rndis_filter_query_link_speed(struct rndis_device *dev,
+ struct netvsc_device *net_device)
{
u32 size = sizeof(u32);
u32 link_speed;
struct net_device_context *ndc;
int ret;
- ret = rndis_filter_query_device(dev, RNDIS_OID_GEN_LINK_SPEED,
+ ret = rndis_filter_query_device(dev, net_device,
+ RNDIS_OID_GEN_LINK_SPEED,
&link_speed, &size);
if (!ret) {
@@ -880,14 +873,14 @@ void rndis_filter_update(struct netvsc_device *nvdev)
schedule_work(&rdev->mcast_work);
}
-static int rndis_filter_init_device(struct rndis_device *dev)
+static int rndis_filter_init_device(struct rndis_device *dev,
+ struct netvsc_device *nvdev)
{
struct rndis_request *request;
struct rndis_initialize_request *init;
struct rndis_initialize_complete *init_complete;
u32 status;
int ret;
- struct netvsc_device *nvdev = net_device_to_netvsc_device(dev->ndev);
request = get_rndis_request(dev, RNDIS_MSG_INIT,
RNDIS_MESSAGE_SIZE(struct rndis_initialize_request));
@@ -1024,12 +1017,17 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
{
struct net_device *ndev =
hv_get_drvdata(new_sc->primary_channel->device_obj);
- struct netvsc_device *nvscdev = net_device_to_netvsc_device(ndev);
+ struct net_device_context *ndev_ctx = netdev_priv(ndev);
+ struct netvsc_device *nvscdev;
u16 chn_index = new_sc->offermsg.offer.sub_channel_index;
struct netvsc_channel *nvchan;
int ret;
- if (chn_index >= nvscdev->num_chn)
+ /* This is safe because this callback only happens when
+ * new device is being setup and waiting on the channel_init_wait.
+ */
+ nvscdev = rcu_dereference_raw(ndev_ctx->nvdev);
+ if (!nvscdev || chn_index >= nvscdev->num_chn)
return;
nvchan = nvscdev->chan_table + chn_index;
@@ -1104,27 +1102,27 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
rndis_device->ndev = net;
/* Send the rndis initialization message */
- ret = rndis_filter_init_device(rndis_device);
+ ret = rndis_filter_init_device(rndis_device, net_device);
if (ret != 0)
goto err_dev_remv;
/* Get the MTU from the host */
size = sizeof(u32);
- ret = rndis_filter_query_device(rndis_device,
+ ret = rndis_filter_query_device(rndis_device, net_device,
RNDIS_OID_GEN_MAXIMUM_FRAME_SIZE,
&mtu, &size);
if (ret == 0 && size == sizeof(u32) && mtu < net->mtu)
net->mtu = mtu;
/* Get the mac address */
- ret = rndis_filter_query_device_mac(rndis_device);
+ ret = rndis_filter_query_device_mac(rndis_device, net_device);
if (ret != 0)
goto err_dev_remv;
memcpy(device_info->mac_adr, rndis_device->hw_mac_adr, ETH_ALEN);
/* Find HW offload capabilities */
- ret = rndis_query_hwcaps(rndis_device, &hwcaps);
+ ret = rndis_query_hwcaps(rndis_device, net_device, &hwcaps);
if (ret != 0)
goto err_dev_remv;
@@ -1185,7 +1183,7 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
if (ret)
goto err_dev_remv;
- rndis_filter_query_device_link_status(rndis_device);
+ rndis_filter_query_device_link_status(rndis_device, net_device);
netdev_dbg(net, "Device MAC %pM link state %s\n",
rndis_device->hw_mac_adr,
@@ -1194,11 +1192,11 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
if (net_device->nvsp_version < NVSP_PROTOCOL_VERSION_5)
return net_device;
- rndis_filter_query_link_speed(rndis_device);
+ rndis_filter_query_link_speed(rndis_device, net_device);
/* vRSS setup */
memset(&rsscap, 0, rsscap_size);
- ret = rndis_filter_query_device(rndis_device,
+ ret = rndis_filter_query_device(rndis_device, net_device,
OID_GEN_RECEIVE_SCALE_CAPABILITIES,
&rsscap, &rsscap_size);
if (ret || rsscap.num_recv_que < 2)
--
2.11.0
^ permalink raw reply related
* [PATCH net-next 1/5] netvsc: fix return value for set_channels
From: Stephen Hemminger @ 2017-07-25 20:04 UTC (permalink / raw)
To: kys, haiyangz, sthemmin; +Cc: devel, netdev
In-Reply-To: <20170725200422.13795-1-sthemmin@microsoft.com>
The error and normal case got swapped.
Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Signed-off-by: Stephen Hemminger <sthemmin@microsoft.com>
---
drivers/net/hyperv/netvsc_drv.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 262486ce8e2a..f1eaf675d2e9 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -758,8 +758,8 @@ static int netvsc_set_channels(struct net_device *net,
if (!IS_ERR(nvdev)) {
netif_set_real_num_tx_queues(net, nvdev->num_chn);
netif_set_real_num_rx_queues(net, nvdev->num_chn);
- ret = PTR_ERR(nvdev);
} else {
+ ret = PTR_ERR(nvdev);
device_info.num_chn = orig;
rndis_filter_device_add(dev, &device_info);
}
--
2.11.0
^ permalink raw reply related
* Re: [PATCH V2 net-next 01/21] net-next/hinic: Initialize hw interface
From: Francois Romieu @ 2017-07-25 20:02 UTC (permalink / raw)
To: Aviad Krawczyk
Cc: davem, linux-kernel, netdev, bc.y, victor.gissin, zhaochen6,
tony.qu
In-Reply-To: <01800b7a-c11d-980f-0e5e-d4684eaa0f2a@huawei.com>
Aviad Krawczyk <aviad.krawczyk@huawei.com> :
[...]
> module_pci_driver - is not used in other drivers in the same segments, it
> is necessary ?
/me checks... Ok, there seems to be some overenthusiastic copy'paste.
See drivers/net/ethernet/intel/ixgb/ixgb_main.c:
[...]
/**
* ixgb_init_module - Driver Registration Routine
*
* ixgb_init_module is the first routine called when the driver is
* loaded. All it does is register with the PCI subsystem.
**/
static int __init
ixgb_init_module(void)
{
pr_info("%s - version %s\n", ixgb_driver_string, ixgb_driver_version);
pr_info("%s\n", ixgb_copyright);
return pci_register_driver(&ixgb_driver);
}
module_init(ixgb_init_module);
/**
* ixgb_exit_module - Driver Exit Cleanup Routine
*
* ixgb_exit_module is called just before the driver is removed
* from memory.
**/
static void __exit
ixgb_exit_module(void)
{
pci_unregister_driver(&ixgb_driver);
}
module_exit(ixgb_exit_module);
Driver version ought to be fed through ethtool, if ever. Copyright message
mildly contributes to a better world. So the whole stuff above could be:
module_pci_driver(ixgb_driver);
--
Ueimor
^ permalink raw reply
* Re: [PATCH net-next 0/3] bnxt_en: Fix kbuild errors and rename phys_port_name.
From: David Miller @ 2017-07-25 19:48 UTC (permalink / raw)
To: michael.chan; +Cc: netdev
In-Reply-To: <1501003721-12407-1-git-send-email-michael.chan@broadcom.com>
From: Michael Chan <michael.chan@broadcom.com>
Date: Tue, 25 Jul 2017 13:28:38 -0400
> Fix 2 more kbuild errors (the first one already fixed by DaveM), and rename
> the physical port name.
Series applied, thanks for working to clear this all up.
^ permalink raw reply
* Re: [PATCH net-next v2] bpf: install libbpf headers on 'make install'
From: Daniel Borkmann @ 2017-07-25 19:45 UTC (permalink / raw)
To: Jakub Kicinski, netdev; +Cc: oss-drivers, alexei.starovoitov, acme
In-Reply-To: <20170725181711.21236-1-jakub.kicinski@netronome.com>
On 07/25/2017 08:17 PM, Jakub Kicinski wrote:
> Add a new target to install the bpf.h header to $(prefix)/include/bpf/
> directory. This is necessary to build standalone applications using
> libbpf, without the need to clone the kernel sources and point to them.
>
> Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Looks good to me, we might do the same for libbpf.h later on as
well, though the naming scheme is a confusing choice (bpf.h and
libbpf.h).
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
> ---
> v2:
> - make the header installation similar to tools/lib/traceevent (Daniel).
>
> The functional change is that the header will not be installed as part
> of make install, one has to make install_headers, which seems OK.
>
> Out of curiosity - why are only "force elfdep bpfdep" added to the PHONY
> target? Is there some Makefile magic that makes adding install* targets
> there unnecessary? Or does the PHONY target just not matter in practice
> so it's not updated?
Arnaldo might know better.
> tools/lib/bpf/Makefile | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
> index 1f5300e56b44..445289555487 100644
> --- a/tools/lib/bpf/Makefile
> +++ b/tools/lib/bpf/Makefile
> @@ -189,6 +189,10 @@ install_lib: all_cmd
> $(call QUIET_INSTALL, $(LIB_FILE)) \
> $(call do_install,$(LIB_FILE),$(libdir_SQ))
>
> +install_headers:
> + $(call QUIET_INSTALL, headers) \
> + $(call do_install,bpf.h,$(prefix)/include/bpf,644)
> +
> install: install_lib
>
> ### Cleaning rules
>
^ permalink raw reply
* [net:master 16/16] net//ipv4/udp.c:1789:47: error: 'struct sk_buff' has no member named 'sp'; did you mean 'sk'?
From: kbuild test robot @ 2017-07-25 19:38 UTC (permalink / raw)
To: Paolo Abeni; +Cc: kbuild-all, netdev
[-- Attachment #1: Type: text/plain, Size: 3090 bytes --]
tree: https://git.kernel.org/pub/scm/linux/kernel/git/davem/net.git master
head: dce4551cb2adb1ac9a30f8ab5299d614392b3cff
commit: dce4551cb2adb1ac9a30f8ab5299d614392b3cff [16/16] udp: preserve head state for IP_CMSG_PASSSEC
config: arm-at91_dt_defconfig (attached as .config)
compiler: arm-linux-gnueabi-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
git checkout dce4551cb2adb1ac9a30f8ab5299d614392b3cff
# save the attached .config to linux build tree
make.cross ARCH=arm
All errors (new ones prefixed by >>):
In file included from include/uapi/linux/stddef.h:1:0,
from include/linux/stddef.h:4,
from include/uapi/linux/posix_types.h:4,
from include/uapi/linux/types.h:13,
from include/linux/types.h:5,
from include/linux/thread_info.h:10,
from include/asm-generic/current.h:4,
from ./arch/arm/include/generated/asm/current.h:1,
from include/linux/sched.h:11,
from include/linux/uaccess.h:4,
from net//ipv4/udp.c:82:
net//ipv4/udp.c: In function '__udp_queue_rcv_skb':
>> net//ipv4/udp.c:1789:47: error: 'struct sk_buff' has no member named 'sp'; did you mean 'sk'?
if (likely(IPCB(skb)->opt.optlen == 0 && !skb->sp))
^
include/linux/compiler.h:174:40: note: in definition of macro 'likely'
# define likely(x) __builtin_expect(!!(x), 1)
^
vim +1789 net//ipv4/udp.c
1772
1773 static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1774 {
1775 int rc;
1776
1777 if (inet_sk(sk)->inet_daddr) {
1778 sock_rps_save_rxhash(sk, skb);
1779 sk_mark_napi_id(sk, skb);
1780 sk_incoming_cpu_update(sk);
1781 } else {
1782 sk_mark_napi_id_once(sk, skb);
1783 }
1784
1785 /* At recvmsg() time we may access skb->dst or skb->sp depending on
1786 * the IP options and the cmsg flags, elsewhere can we clear all
1787 * pending head states while they are hot in the cache
1788 */
> 1789 if (likely(IPCB(skb)->opt.optlen == 0 && !skb->sp))
1790 skb_release_head_state(skb);
1791
1792 rc = __udp_enqueue_schedule_skb(sk, skb);
1793 if (rc < 0) {
1794 int is_udplite = IS_UDPLITE(sk);
1795
1796 /* Note that an ENOMEM error is charged twice */
1797 if (rc == -ENOMEM)
1798 UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1799 is_udplite);
1800 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1801 kfree_skb(skb);
1802 trace_udp_fail_queue_rcv_skb(rc, sk);
1803 return -1;
1804 }
1805
1806 return 0;
1807 }
1808
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 23094 bytes --]
^ permalink raw reply
* Re: [PATCH v2 net-next 1/5] drop_monitor: import netnamespace framework
From: David Miller @ 2017-07-25 19:36 UTC (permalink / raw)
To: martinbj2008; +Cc: nhorman, xiyou.wangcong, netdev, zhangjunweimartin
In-Reply-To: <1500982739-15805-1-git-send-email-zhangjunweimartin@didichuxing.com>
Every patch series must start with an appropriate "[PATCH ... 0/N] ..." header
posting explaining what the patch series is doing, how it is doing it, and
why it is doing it that way.
Thank you.
^ permalink raw reply
* Re: [PATCH] lib: test_rhashtable: Fix KASAN warning
From: David Miller @ 2017-07-25 19:36 UTC (permalink / raw)
To: phil; +Cc: herbert, tgraf, netdev
In-Reply-To: <20170725113621.1425-1-phil@nwl.cc>
From: Phil Sutter <phil@nwl.cc>
Date: Tue, 25 Jul 2017 13:36:21 +0200
> I forgot one spot when introducing struct test_obj_val.
>
> Fixes: e859afe1ee0c5 ("lib: test_rhashtable: fix for large entry counts")
> Reported by: kernel test robot <fengguang.wu@intel.com>
>
> Signed-off-by: Phil Sutter <phil@nwl.cc>
Please don't put empty lines in the middle of a set of tags like that.
I fixed it up this time :)
Applied, thanks.
^ permalink raw reply
* Re: [PATCH] net: phy: Remove trailing semicolon in macro definition
From: David Miller @ 2017-07-25 19:33 UTC (permalink / raw)
To: marc_gonzalez; +Cc: andrew, f.fainelli, netdev, slash.tmp
In-Reply-To: <d1f4d4f1-e60b-6b4d-6f98-3be70da859b5@sigmadesigns.com>
From: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
Date: Tue, 25 Jul 2017 11:08:15 +0200
> Commit e5a03bfd873c2 ("phy: Add an mdio_device structure")
> introduced a spurious trailing semicolon. Remove it.
>
> Signed-off-by: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
Good catch, applied, thanks.
^ permalink raw reply
* Re: [PATCH] net: phy: Log only PHY state transitions
From: David Miller @ 2017-07-25 19:33 UTC (permalink / raw)
To: marc_gonzalez; +Cc: andrew, f.fainelli, netdev, slash.tmp
In-Reply-To: <8fc7eb8f-9d64-04c3-a1ca-0cc8eec1ee30@sigmadesigns.com>
From: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
Date: Tue, 25 Jul 2017 11:31:46 +0200
> In the current code, old and new PHY states are always logged.
> From now on, log only PHY state transitions.
>
> Signed-off-by: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
> ---
> drivers/net/phy/phy.c | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
> index d0626bf5c540..6bb764e716fc 100644
> --- a/drivers/net/phy/phy.c
> +++ b/drivers/net/phy/phy.c
> @@ -1226,7 +1226,8 @@ void phy_state_machine(struct work_struct *work)
> if (err < 0)
> phy_error(phydev);
>
> - phydev_dbg(phydev, "PHY state change %s -> %s\n",
> + if (old_state != phydev->state)
> + phydev_dbg(phydev, "PHY state change %s -> %s\n",
> phy_state_to_str(old_state),
> phy_state_to_str(phydev->state));
Something is not kosher with this indentation at all.
^ permalink raw reply
* Re: [PATCH net-next 0/6] network related warning fixes
From: David Miller @ 2017-07-25 19:31 UTC (permalink / raw)
To: stephen
Cc: marcel, gustavo, johan.hedberg, pablo, kadlec, fw, netdev,
linux-bluetooth, netfilter-devel, coreteam, sthemmin
In-Reply-To: <20170724172523.13135-1-sthemmin@microsoft.com>
From: Stephen Hemminger <stephen@networkplumber.org>
Date: Mon, 24 Jul 2017 10:25:17 -0700
> Various fixes for warnings in network code and drivers.
Series applied, thanks Stephen.
^ permalink raw reply
* Re: [PATCH net-next] nfp: set config bit (ifup/ifdown) on netdev open/close
From: David Miller @ 2017-07-25 19:29 UTC (permalink / raw)
To: jakub.kicinski; +Cc: netdev, oss-drivers, dirk.vandermerwe
In-Reply-To: <20170725075108.18537-1-jakub.kicinski@netronome.com>
From: Jakub Kicinski <jakub.kicinski@netronome.com>
Date: Tue, 25 Jul 2017 00:51:08 -0700
> From: Dirk van der Merwe <dirk.vandermerwe@netronome.com>
>
> When a netdev (PF netdev or representor) is opened or closed, set the
> physical port config bit appropriately - which powers UP/DOWN the PHY
> module for the physical interface.
>
> The PHY is powered first in the HW/FW configuration step when opening
> the netdev and again last in the HW/FW configuration step when closing
> the netdev.
>
> This is only applicable when there is a physical port associated with
> the netdev and if the NSP support this. Otherwise we silently ignore
> this step.
>
> The 'nfp_eth_set_configured' can actually return positive values -
> updated the function documentation appropriately.
>
> Signed-off-by: Dirk van der Merwe <dirk.vandermerwe@netronome.com>
> Reviewed-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Applied, thanks Jakub.
^ permalink raw reply
* Re: [PATCH] drivers/net: Fix ptr_ret.cocci warnings.
From: David Miller @ 2017-07-25 19:27 UTC (permalink / raw)
To: xiangxia.m.yue; +Cc: netdev
In-Reply-To: <1500966026-88504-1-git-send-email-xiangxia.m.yue@gmail.com>
aFrom: Tonghao Zhang <xiangxia.m.yue@gmail.com>
Date: Tue, 25 Jul 2017 00:00:26 -0700
> we can use PTR_ERR_OR_ZERO rather than if(IS_ERR(...)) + PTR_ERR
> 1. drivers/net/appletalk/ipddp.c
> 2. drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
>
> Generated by: scripts/coccinelle/api/ptr_ret.cocci
>
> Signed-off-by: Tonghao Zhang <xiangxia.m.yue@gmail.com>
Applied to net-next, thanks.
^ permalink raw reply
* Re: [PATCH net-next v2 01/10] net: dsa: lan9303: Fixed MDIO interface
From: Vivien Didelot @ 2017-07-25 19:15 UTC (permalink / raw)
To: Egil Hjelmeland, corbet, andrew, f.fainelli, davem, kernel,
linux-doc, linux-kernel, netdev
Cc: Egil Hjelmeland
In-Reply-To: <20170725161553.30147-2-privat@egil-hjelmeland.no>
Hi Egil,
Egil Hjelmeland <privat@egil-hjelmeland.no> writes:
> Fixes after testing on actual HW:
>
> - lan9303_mdio_write()/_read() must multiply register number
> by 4 to get offset
>
> - Indirect access (PMI) to phy register only work in I2C mode. In
> MDIO mode phy registers must be accessed directly. Introduced
> struct lan9303_phy_ops to handle the two modes. Renamed functions
> to clarify.
>
> - lan9303_detect_phy_setup() : Failed MDIO read return 0xffff.
> Handle that.
Small patch series when possible are better. Bullet points in commit
messages are likely to describe how a patch or series may be split up
;-)
This patch seems to be the unique patch of the series resolving what is
described in the cover letter as "Make the MDIO interface work".
I'd suggest you to split up this one commit in several *atomic* and easy
to review patches and send them separately as on thread named "net: dsa:
lan9303: fix MDIO interface" (also note that imperative is prefered for
subject lines, see: https://chris.beams.io/posts/git-commit/#imperative)
<...>
> -static int lan9303_port_phy_reg_wait_for_completion(struct lan9303 *chip)
> +static int lan9303_indirect_phy_wait_for_completion(struct lan9303 *chip)
For instance you can have a first commit only renaming the functions.
The reason for it is to separate the functional changes from cosmetic
changes, which makes it easier for review.
<...>
> - if (reg != 0)
> + if ((reg != 0) && (reg != 0xffff))
if (reg && reg != 0xffff) should be enough.
> chip->phy_addr_sel_strap = 1;
> else
> chip->phy_addr_sel_strap = 0;
<...>
> +struct lan9303;
> +
> +struct lan9303_phy_ops {
> + /* PHY 1 &2 access*/
The spacing is weird in the comment. "/* PHY 1 & 2 access */" maybe?
<...>
> +int lan9303_mdio_phy_write(struct lan9303 *chip, int phy, int regnum, u16 val)
> +{
> + struct lan9303_mdio *sw_dev = dev_get_drvdata(chip->dev);
> + struct mdio_device *mdio = sw_dev->device;
> +
> + mutex_lock(&mdio->bus->mdio_lock);
> + mdio->bus->write(mdio->bus, phy, regnum, val);
> + mutex_unlock(&mdio->bus->mdio_lock);
This is exactly what mdiobus_write(mdio->bus, phy, regnum, val) is
doing. There are very few valid reasons to go play in the mii_bus
structure, using generic APIs are strongly prefered. Plus you have
checks and traces for free!
> +
> + return 0;
> +}
> +
> +int lan9303_mdio_phy_read(struct lan9303 *chip, int phy, int reg)
> +{
> + struct lan9303_mdio *sw_dev = dev_get_drvdata(chip->dev);
> + struct mdio_device *mdio = sw_dev->device;
> + int val;
> +
> + mutex_lock(&mdio->bus->mdio_lock);
> + val = mdio->bus->read(mdio->bus, phy, reg);
> + mutex_unlock(&mdio->bus->mdio_lock);
Same here, mdiobus_read().
Thanks,
Vivien
^ permalink raw reply
* Re: [PATCH iproute2 master v2 0/2] Minor BPF updates
From: Stephen Hemminger @ 2017-07-25 18:48 UTC (permalink / raw)
To: Daniel Borkmann; +Cc: ast, netdev
In-Reply-To: <1500765739-10681-1-git-send-email-daniel@iogearbox.net>
On Sun, 23 Jul 2017 01:22:17 +0200
Daniel Borkmann <daniel@iogearbox.net> wrote:
> Two minor updates to the BPF code, first one makes use of the
> recently exposed owner_jited in fdinfo to report whether a
> load issue related to tail calls occured, and second one fixes
> up custom mount of bpf fs when passed via env.
>
> Thanks!
>
> v1 -> v2:
> - Moved bpf_derive_prog_from_fdinfo() under HAVE_ELF.
> - Rest as is.
>
> Daniel Borkmann (2):
> bpf: improve error reporting around tail calls
> bpf: fix mnt path when from env
>
> lib/bpf.c | 281 +++++++++++++++++++++++++++++++++++++++++++++++---------------
> 1 file changed, 214 insertions(+), 67 deletions(-)
>
Applied to master
^ permalink raw reply
* Re: [PATCH] ip route: replace exits with returns
From: Stephen Hemminger @ 2017-07-25 18:42 UTC (permalink / raw)
To: elie; +Cc: netdev
In-Reply-To: <20170722224202.5602-1-elie@bouttier.eu>
On Sun, 23 Jul 2017 00:42:02 +0200
elie@bouttier.eu wrote:
> From: Élie Bouttier <elie@bouttier.eu>
>
> This patch replaces exits with returns in ip route
> commands.
>
> Allows to continue when invoked with ip -batch.
>
> Signed-off-by: Élie Bouttier <elie@bouttier.eu>
Sure applied, but any attempt to continue after some of these kernel
communication errors is unlikely to work.
^ permalink raw reply
* Fw: [Bug 196469] New: UDPv4+UDPv6 receive silently fails after heavy UDP load
From: Stephen Hemminger @ 2017-07-25 18:40 UTC (permalink / raw)
To: netdev
Begin forwarded message:
Date: Mon, 24 Jul 2017 20:52:01 +0000
From: bugzilla-daemon@bugzilla.kernel.org
To: stephen@networkplumber.org
Subject: [Bug 196469] New: UDPv4+UDPv6 receive silently fails after heavy UDP load
https://bugzilla.kernel.org/show_bug.cgi?id=196469
Bug ID: 196469
Summary: UDPv4+UDPv6 receive silently fails after heavy UDP
load
Product: Networking
Version: 2.5
Kernel Version: 4.12.2
Hardware: All
OS: Linux
Tree: Mainline
Status: NEW
Severity: normal
Priority: P1
Component: IPV4
Assignee: stephen@networkplumber.org
Reporter: CFSworks@gmail.com
Regression: No
I have a system that's performing heavy SNMP monitoring on various devices over
IPv6. After upgrading it from 4.11.5 to 4.12.2 last week, I noticed this
weekend that all UDP packets received by the system were being silently
dropped.
The problem happened this weekend, after the system had been running for about
2 days with no issue, and without any human intervention at the time (i.e. it
broke on its own - no configuration changes were made). I verified UDP replies
were coming in via tcpdump, and checked netfilter and conntrack to make sure
nothing was blocking it (indeed netfilter/conntrack is able to see the traffic,
indicating the problem is deeper in the network stack). There is no relevant or
even unusual output whatsoever in dmesg.
This affects ALL received UDPv4/UDPv6 packets, even traffic on a loopback
interface: opening a pair of Python sessions and trying to send UDP traffic
back and forth on 127.0.0.1 results in the traffic getting dropped as well.
ICMP and TCP are unaffected.
My gut says it's a race condition somewhere in the UDP receive queue(s). What
puzzles me is why heavy UDPv6 load caused the kernel to shut out all UDPv4
traffic too.
--
You are receiving this mail because:
You are the assignee for the bug.
^ permalink raw reply
* [PATCH] netpoll: Fix device name check in netpoll_setup()
From: Matthias Kaehlcke @ 2017-07-25 18:36 UTC (permalink / raw)
To: David S . Miller; +Cc: netdev, linux-kernel, Doug Anderson, Matthias Kaehlcke
Apparently netpoll_setup() assumes that netpoll.dev_name is a pointer
when checking if the device name is set:
if (np->dev_name) {
...
However the field is a character array, therefore the condition always
yields true. Check instead whether the first byte of the array has a
non-zero value.
Signed-off-by: Matthias Kaehlcke <mka@chromium.org>
---
net/core/netpoll.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index 8357f164c660..912731bed7b7 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -666,7 +666,7 @@ int netpoll_setup(struct netpoll *np)
int err;
rtnl_lock();
- if (np->dev_name) {
+ if (np->dev_name[0]) {
struct net *net = current->nsproxy->net_ns;
ndev = __dev_get_by_name(net, np->dev_name);
}
--
2.14.0.rc0.284.gd933b75aa4-goog
^ permalink raw reply related
* Re: [PATCH net-next] bpf: add helper capable of reading out instructions
From: Daniel Borkmann @ 2017-07-25 18:25 UTC (permalink / raw)
To: Jakub Kicinski; +Cc: netdev, alexei.starovoitov, oss-drivers, kafai
In-Reply-To: <20170725112006.261bf04f@cakuba.netronome.com>
On 07/25/2017 08:20 PM, Jakub Kicinski wrote:
> On Tue, 25 Jul 2017 18:40:23 +0200, Daniel Borkmann wrote:
>> [ +Martin ]
>
> Sorry, I thought I CCed Martin.
>
>> On 07/24/2017 11:22 PM, Jakub Kicinski wrote:
>>> To read translated and jited instructions from the kernel,
>>> one has to set certain pointers of struct bpf_prog_info to
>>> pre-allocated user buffers. Unfortunately, the existing
>>> bpf_obj_get_info_by_fd() helper zeros struct bpf_prog_info
>>> before passing it to the kernel.
>>>
>>> Keeping the zeroing seems like a good idea in general, since
>>> kernel will check if the structure was zeroed. Add a new
>>> helper for those more advanced users who can be trusted to
>>> take care of zeroing themselves.
>>>
>>> Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
>>> ---
>>> I'm happy to change the name of the new function.
>>>
>>> tools/lib/bpf/bpf.c | 10 ++++++++--
>>> tools/lib/bpf/bpf.h | 2 ++
>>> 2 files changed, 10 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
>>> index 412a7c82995a..2703fa282b65 100644
>>> --- a/tools/lib/bpf/bpf.c
>>> +++ b/tools/lib/bpf/bpf.c
>>> @@ -308,13 +308,12 @@ int bpf_map_get_fd_by_id(__u32 id)
>>> return sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr));
>>> }
>>>
>>> -int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len)
>>> +int __bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len)
>>> {
>>> union bpf_attr attr;
>>> int err;
>>>
>>> bzero(&attr, sizeof(attr));
>>> - bzero(info, *info_len);
>>
>> Looks a bit unintentional to me, e.g. 95b9afd3987f ("bpf: Test for bpf
>> ID") did set up pointers in test_bpf_obj_id(), but later only checked
>> for the {jited,xlated}_prog_len.
>>
>> Clearing out the pointers looks not to useful. Lets just push the need
>> for bzero() to call-sites in general in this case.
>
> Should I target this at net then? To avoid backwards compatibility
> issues?
Yep, sounds reasonable. Thanks!
^ permalink raw reply
* Re: [PATCH net-next] bpf: add helper capable of reading out instructions
From: Jakub Kicinski @ 2017-07-25 18:20 UTC (permalink / raw)
To: Daniel Borkmann; +Cc: netdev, alexei.starovoitov, oss-drivers, kafai
In-Reply-To: <59777477.7050609@iogearbox.net>
On Tue, 25 Jul 2017 18:40:23 +0200, Daniel Borkmann wrote:
> [ +Martin ]
Sorry, I thought I CCed Martin.
> On 07/24/2017 11:22 PM, Jakub Kicinski wrote:
> > To read translated and jited instructions from the kernel,
> > one has to set certain pointers of struct bpf_prog_info to
> > pre-allocated user buffers. Unfortunately, the existing
> > bpf_obj_get_info_by_fd() helper zeros struct bpf_prog_info
> > before passing it to the kernel.
> >
> > Keeping the zeroing seems like a good idea in general, since
> > kernel will check if the structure was zeroed. Add a new
> > helper for those more advanced users who can be trusted to
> > take care of zeroing themselves.
> >
> > Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
> > ---
> > I'm happy to change the name of the new function.
> >
> > tools/lib/bpf/bpf.c | 10 ++++++++--
> > tools/lib/bpf/bpf.h | 2 ++
> > 2 files changed, 10 insertions(+), 2 deletions(-)
> >
> > diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
> > index 412a7c82995a..2703fa282b65 100644
> > --- a/tools/lib/bpf/bpf.c
> > +++ b/tools/lib/bpf/bpf.c
> > @@ -308,13 +308,12 @@ int bpf_map_get_fd_by_id(__u32 id)
> > return sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr));
> > }
> >
> > -int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len)
> > +int __bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len)
> > {
> > union bpf_attr attr;
> > int err;
> >
> > bzero(&attr, sizeof(attr));
> > - bzero(info, *info_len);
>
> Looks a bit unintentional to me, e.g. 95b9afd3987f ("bpf: Test for bpf
> ID") did set up pointers in test_bpf_obj_id(), but later only checked
> for the {jited,xlated}_prog_len.
>
> Clearing out the pointers looks not to useful. Lets just push the need
> for bzero() to call-sites in general in this case.
Should I target this at net then? To avoid backwards compatibility
issues?
^ permalink raw reply
* [PATCH net-next v2] bpf: install libbpf headers on 'make install'
From: Jakub Kicinski @ 2017-07-25 18:17 UTC (permalink / raw)
To: netdev, daniel; +Cc: oss-drivers, alexei.starovoitov, Jakub Kicinski
In-Reply-To: <20170724212236.21903-1-jakub.kicinski@netronome.com>
Add a new target to install the bpf.h header to $(prefix)/include/bpf/
directory. This is necessary to build standalone applications using
libbpf, without the need to clone the kernel sources and point to them.
Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
---
v2:
- make the header installation similar to tools/lib/traceevent (Daniel).
The functional change is that the header will not be installed as part
of make install, one has to make install_headers, which seems OK.
Out of curiosity - why are only "force elfdep bpfdep" added to the PHONY
target? Is there some Makefile magic that makes adding install* targets
there unnecessary? Or does the PHONY target just not matter in practice
so it's not updated?
tools/lib/bpf/Makefile | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
index 1f5300e56b44..445289555487 100644
--- a/tools/lib/bpf/Makefile
+++ b/tools/lib/bpf/Makefile
@@ -189,6 +189,10 @@ install_lib: all_cmd
$(call QUIET_INSTALL, $(LIB_FILE)) \
$(call do_install,$(LIB_FILE),$(libdir_SQ))
+install_headers:
+ $(call QUIET_INSTALL, headers) \
+ $(call do_install,bpf.h,$(prefix)/include/bpf,644)
+
install: install_lib
### Cleaning rules
--
2.11.0
^ permalink raw reply related
* Re: [PATCH v2 2/4] can: fixed-transceiver: Add documentation for CAN fixed transceiver bindings
From: Franklin S Cooper Jr @ 2017-07-25 18:14 UTC (permalink / raw)
To: Oliver Hartkopp, linux-kernel-u79uwXL29TY76Z2rM5mHXA,
devicetree-u79uwXL29TY76Z2rM5mHXA, netdev-u79uwXL29TY76Z2rM5mHXA,
linux-can-u79uwXL29TY76Z2rM5mHXA, wg-5Yr1BZd7O62+XT7JhA+gdA,
mkl-bIcnvbaLZ9MEGnE8C9+IrQ, robh+dt-DgEjT+Ai2ygdnm+yROfE0A,
quentin.schulz-wi1+55ScJUtKEb57/3fJTNBPR1lH4CV8,
dev.kurt-yI9piX4KPfawT/RRk36CISFp6vIno51x, andrew-g2DYL2Zd6BY,
sergei.shtylyov-M4DtvfQ/ZS1MRgGoP+s0PdBPR1lH4CV8
In-Reply-To: <29df7e04-01c6-a09b-491e-1354dab98cd0-fJ+pQTUTwRTk1uMJSBkQmQ@public.gmane.org>
On 07/25/2017 11:32 AM, Oliver Hartkopp wrote:
>
>> + max-data-speed: a positive non 0 value that determines the max
>> data rate
>> + that can be used in CAN-FD mode. A value of -1 implies
>> + CAN-FD is not supported by the transceiver.
>> +
>> +Examples:
>
> (..)
>
>> + fixed-transceiver {
>> + max-data-speed = <(-1)>;
>
> Looks ugly IMHO.
>
> Why didn't you stay on '0' for 'not supported'??
Unless a driver specifically calls of_can_transceiver_fixed
priv->max_trans_data_speed will be by default 0. Therefore, all drivers
that support CAN-FD will claim that the transceiver indicates that it
isn't supported. So one option was to update every single driver to set
this property by default which I started to do but it end up becoming a
massive patch and it was risky in case I missed a driver which would of
resulted in major regressions. Its also problematic for new drivers that
miss this property or the many out of tree CAN drivers. The other option
was to create another variable to track to see if
of_can_transceiver_fixed was called but I didn't think that was the
better solution. So using signed values in DT is a bit ugly due to
syntax but was valid and I made sure I documented it so its clear.
>
> Regards,
> Oliver
>
--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: Problem with PHY state machine when using interrupts
From: Florian Fainelli @ 2017-07-25 17:55 UTC (permalink / raw)
To: Mason, Andrew Lunn, Mans Rullgard; +Cc: netdev, Linux ARM
In-Reply-To: <e2f4ef4a-c667-3bdf-e4ea-2767dc6a4922@free.fr>
On July 25, 2017 4:41:32 AM PDT, Mason <slash.tmp@free.fr> wrote:
>On 25/07/2017 12:51, Mason wrote:
>
>> Moving the call to phy_stop() down after all the MAC tear down
>> avoids the hang.
>>
>> As far as I understand, when we are shutting everything down,
>> we don't need the phy_state_machine to run asynchronously.
>> We can run it synchronously one last time after the delayed
>> stuff has been disabled.
>
>Below is my current WIP diff. (It conflates the two issues
>I've been discussing. Splitting the diff required.)
>
>Tested in interrupt mode:
>
># ip link set eth0 up
>[ 11.107547] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change UP -> AN
>[ 14.530329] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full -
>flow control rx/tx
>[ 14.538136] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change AN -> RUNNING
># ip link set eth0 down
>[ 23.801018] nb8800 26000.ethernet eth0: Link is Down
># ip link set eth0 up
>[ 28.740870] Atheros 8035 ethernet UP26000.nb8800-mii:04: PHY state
>change UP -> AN
>[ 31.431528] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full -
>flow control rx/tx
>[ 31.439350] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change AN -> RUNNING
>
>Works as expected.
It does indeed, although this probably also contains your change that only logs the PHY state machine transitions, which makes me wonder why the UP -> HALTED state is not logged?
>
>Tested in polling mode:
>
># ip link set eth0 up
>[ 23.001199] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change UP -> AN
>[ 24.024315] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change AN -> NOLINK
>[ 27.064355] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full -
>flow control rx/tx
>[ 27.072156] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change NOLINK -> RUNNING
># ip link set eth0 down
>[ 42.134617] nb8800 26000.ethernet eth0: Link is Down
># ip link set eth0 up
>[ 48.381185] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change UP -> AN
>[ 49.410976] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change AN -> NOLINK
>[ 51.437686] nb8800 26000.ethernet eth0: Link is Up - 1Gbps/Full -
>flow control rx/tx
>[ 51.445486] Atheros 8035 ethernet 26000.nb8800-mii:04: PHY state
>change NOLINK -> RUNNING
>
>Works as expected.
>
>Also tested on my old board, no regression seen.
>
>Can you confirm that the changes to drivers/net/phy/phy.c
>look reasonable?
The flush is correct, but I am not sure about the explicit state change, in two ways:
- in polling mode, we should already be reaching that state with the flush call AFAICT
- in interrupt driven mode (phy_interrupt_is_valid or PHY_IGNORE_INTERRUPT) we do indeed need to make sure the HALTED state is reached
Not sure why I did not see the interrupt imbalance problem with phy_mac_interrupt...
I am out today but will follow up tomorrow. thanks!
--
Florian
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox