Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/988/?format=api
{ "id": 988, "url": "http://patchwork.ozlabs.org/api/patches/988/?format=api", "web_url": "http://patchwork.ozlabs.org/project/netdev/patch/20080922.151233.229805934.davem@davemloft.net/", "project": { "id": 7, "url": "http://patchwork.ozlabs.org/api/projects/7/?format=api", "name": "Linux network development", "link_name": "netdev", "list_id": "netdev.vger.kernel.org", "list_email": "netdev@vger.kernel.org", "web_url": null, "scm_url": null, "webscm_url": null, "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20080922.151233.229805934.davem@davemloft.net>", "list_archive_url": null, "date": "2008-09-22T22:12:33", "name": "[0/2] : Remote softirq invocation infrastructure.", "commit_ref": null, "pull_url": null, "state": "rfc", "archived": true, "hash": "c0baf8db40c16fb6c77151890320431409fd10eb", "submitter": { "id": 15, "url": "http://patchwork.ozlabs.org/api/people/15/?format=api", "name": "David Miller", "email": "davem@davemloft.net" }, "delegate": null, "mbox": "http://patchwork.ozlabs.org/project/netdev/patch/20080922.151233.229805934.davem@davemloft.net/mbox/", "series": [], "comments": "http://patchwork.ozlabs.org/api/patches/988/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/988/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<netdev-owner@vger.kernel.org>", "X-Original-To": "patchwork-incoming@ozlabs.org", "Delivered-To": "patchwork-incoming@ozlabs.org", "Received": [ "from vger.kernel.org (vger.kernel.org [209.132.176.167])\n\tby ozlabs.org (Postfix) with ESMTP id 24A82DDEE7\n\tfor <patchwork-incoming@ozlabs.org>;\n\tTue, 23 Sep 2008 08:12:58 +1000 (EST)", "(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1754464AbYIVWMs (ORCPT <rfc822;patchwork-incoming@ozlabs.org>);\n\tMon, 22 Sep 2008 18:12:48 -0400", "(majordomo@vger.kernel.org) by vger.kernel.org id S1754360AbYIVWMr\n\t(ORCPT <rfc822; netdev-outgoing>); Mon, 22 Sep 2008 18:12:47 -0400", "from 74-93-104-97-Washington.hfc.comcastbusiness.net\n\t([74.93.104.97]:54929\n\t\"EHLO sunset.davemloft.net\" rhost-flags-OK-FAIL-OK-OK)\n\tby vger.kernel.org with ESMTP id S1754190AbYIVWMp (ORCPT\n\t<rfc822;netdev@vger.kernel.org>); Mon, 22 Sep 2008 18:12:45 -0400", "from localhost (localhost [127.0.0.1])\n\tby sunset.davemloft.net (Postfix) with ESMTP id B369FC8C181;\n\tMon, 22 Sep 2008 15:12:33 -0700 (PDT)" ], "Date": "Mon, 22 Sep 2008 15:12:33 -0700 (PDT)", "Message-Id": "<20080922.151233.229805934.davem@davemloft.net>", "To": "cfriesen@nortel.com", "Cc": "linux-kernel@vger.kernel.org, netdev@vger.kernel.org,\n\tjens.axboe@oracle.com, steffen.klassert@secunet.com", "Subject": "Re: [PATCH 0/2]: Remote softirq invocation infrastructure.", "From": "David Miller <davem@davemloft.net>", "In-Reply-To": "<48D80C9C.2070108@nortel.com>", "References": "<20080919.234824.223177211.davem@davemloft.net>\n\t<48D80C9C.2070108@nortel.com>", "X-Mailer": "Mew version 6.1 on Emacs 22.1 / Mule 5.0 (SAKAKI)", "Mime-Version": "1.0", "Content-Type": "Text/Plain; charset=us-ascii", "Content-Transfer-Encoding": "7bit", "Sender": "netdev-owner@vger.kernel.org", "Precedence": "bulk", "List-ID": "<netdev.vger.kernel.org>", "X-Mailing-List": "netdev@vger.kernel.org" }, "content": "From: \"Chris Friesen\" <cfriesen@nortel.com>\nDate: Mon, 22 Sep 2008 15:22:36 -0600\n\n> I'm not sure this belongs in this particular thread but I was\n> interested in how you're planning on doing this?\n\nSomething like this patch which I posted last week on\nnetdev.\n\nnet: Do software flow seperation on receive.\n\nPush netif_receive_skb() work to remote cpus via flow\nhashing and remove softirqs.\n\nSigned-off-by: David S. Miller <davem@davemloft.net>\n---\n include/linux/interrupt.h | 1 +\n include/linux/netdevice.h | 2 -\n include/linux/skbuff.h | 3 +\n net/core/dev.c | 273 +++++++++++++++++++++++++--------------------\n 4 files changed, 157 insertions(+), 122 deletions(-)", "diff": "diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h\nindex 806b38f..223e68f 100644\n--- a/include/linux/interrupt.h\n+++ b/include/linux/interrupt.h\n@@ -247,6 +247,7 @@ enum\n \tTIMER_SOFTIRQ,\n \tNET_TX_SOFTIRQ,\n \tNET_RX_SOFTIRQ,\n+\tNET_RECEIVE_SOFTIRQ,\n \tBLOCK_SOFTIRQ,\n \tTASKLET_SOFTIRQ,\n \tSCHED_SOFTIRQ,\ndiff --git a/include/linux/netdevice.h b/include/linux/netdevice.h\nindex 488c56e..a044caa 100644\n--- a/include/linux/netdevice.h\n+++ b/include/linux/netdevice.h\n@@ -965,11 +965,9 @@ static inline int unregister_gifconf(unsigned int family)\n struct softnet_data\n {\n \tstruct Qdisc\t\t*output_queue;\n-\tstruct sk_buff_head\tinput_pkt_queue;\n \tstruct list_head\tpoll_list;\n \tstruct sk_buff\t\t*completion_queue;\n \n-\tstruct napi_struct\tbacklog;\n #ifdef CONFIG_NET_DMA\n \tstruct dma_chan\t\t*net_dma;\n #endif\ndiff --git a/include/linux/skbuff.h b/include/linux/skbuff.h\nindex 9099237..e36bc86 100644\n--- a/include/linux/skbuff.h\n+++ b/include/linux/skbuff.h\n@@ -18,6 +18,7 @@\n #include <linux/compiler.h>\n #include <linux/time.h>\n #include <linux/cache.h>\n+#include <linux/smp.h>\n \n #include <asm/atomic.h>\n #include <asm/types.h>\n@@ -255,6 +256,8 @@ struct sk_buff {\n \tstruct sk_buff\t\t*next;\n \tstruct sk_buff\t\t*prev;\n \n+\tstruct call_single_data\tcsd;\n+\n \tstruct sock\t\t*sk;\n \tktime_t\t\t\ttstamp;\n \tstruct net_device\t*dev;\ndiff --git a/net/core/dev.c b/net/core/dev.c\nindex e719ed2..09827c7 100644\n--- a/net/core/dev.c\n+++ b/net/core/dev.c\n@@ -1660,8 +1660,8 @@ out_kfree_skb:\n \treturn 0;\n }\n \n-static u32 simple_tx_hashrnd;\n-static int simple_tx_hashrnd_initialized = 0;\n+static u32 simple_hashrnd;\n+static int simple_hashrnd_initialized = 0;\n \n static u16 simple_tx_hash(struct net_device *dev, struct sk_buff *skb)\n {\n@@ -1669,9 +1669,9 @@ static u16 simple_tx_hash(struct net_device *dev, struct sk_buff *skb)\n \tu32 hash, ihl;\n \tu8 ip_proto;\n \n-\tif (unlikely(!simple_tx_hashrnd_initialized)) {\n-\t\tget_random_bytes(&simple_tx_hashrnd, 4);\n-\t\tsimple_tx_hashrnd_initialized = 1;\n+\tif (unlikely(!simple_hashrnd_initialized)) {\n+\t\tget_random_bytes(&simple_hashrnd, 4);\n+\t\tsimple_hashrnd_initialized = 1;\n \t}\n \n \tswitch (skb->protocol) {\n@@ -1708,7 +1708,7 @@ static u16 simple_tx_hash(struct net_device *dev, struct sk_buff *skb)\n \t\tbreak;\n \t}\n \n-\thash = jhash_3words(addr1, addr2, ports, simple_tx_hashrnd);\n+\thash = jhash_3words(addr1, addr2, ports, simple_hashrnd);\n \n \treturn (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);\n }\n@@ -1878,75 +1878,6 @@ int weight_p __read_mostly = 64; /* old backlog weight */\n DEFINE_PER_CPU(struct netif_rx_stats, netdev_rx_stat) = { 0, };\n \n \n-/**\n- *\tnetif_rx\t-\tpost buffer to the network code\n- *\t@skb: buffer to post\n- *\n- *\tThis function receives a packet from a device driver and queues it for\n- *\tthe upper (protocol) levels to process. It always succeeds. The buffer\n- *\tmay be dropped during processing for congestion control or by the\n- *\tprotocol layers.\n- *\n- *\treturn values:\n- *\tNET_RX_SUCCESS\t(no congestion)\n- *\tNET_RX_DROP (packet was dropped)\n- *\n- */\n-\n-int netif_rx(struct sk_buff *skb)\n-{\n-\tstruct softnet_data *queue;\n-\tunsigned long flags;\n-\n-\t/* if netpoll wants it, pretend we never saw it */\n-\tif (netpoll_rx(skb))\n-\t\treturn NET_RX_DROP;\n-\n-\tif (!skb->tstamp.tv64)\n-\t\tnet_timestamp(skb);\n-\n-\t/*\n-\t * The code is rearranged so that the path is the most\n-\t * short when CPU is congested, but is still operating.\n-\t */\n-\tlocal_irq_save(flags);\n-\tqueue = &__get_cpu_var(softnet_data);\n-\n-\t__get_cpu_var(netdev_rx_stat).total++;\n-\tif (queue->input_pkt_queue.qlen <= netdev_max_backlog) {\n-\t\tif (queue->input_pkt_queue.qlen) {\n-enqueue:\n-\t\t\t__skb_queue_tail(&queue->input_pkt_queue, skb);\n-\t\t\tlocal_irq_restore(flags);\n-\t\t\treturn NET_RX_SUCCESS;\n-\t\t}\n-\n-\t\tnapi_schedule(&queue->backlog);\n-\t\tgoto enqueue;\n-\t}\n-\n-\t__get_cpu_var(netdev_rx_stat).dropped++;\n-\tlocal_irq_restore(flags);\n-\n-\tkfree_skb(skb);\n-\treturn NET_RX_DROP;\n-}\n-\n-int netif_rx_ni(struct sk_buff *skb)\n-{\n-\tint err;\n-\n-\tpreempt_disable();\n-\terr = netif_rx(skb);\n-\tif (local_softirq_pending())\n-\t\tdo_softirq();\n-\tpreempt_enable();\n-\n-\treturn err;\n-}\n-\n-EXPORT_SYMBOL(netif_rx_ni);\n-\n static void net_tx_action(struct softirq_action *h)\n {\n \tstruct softnet_data *sd = &__get_cpu_var(softnet_data);\n@@ -2177,7 +2108,7 @@ void netif_nit_deliver(struct sk_buff *skb)\n *\tNET_RX_SUCCESS: no congestion\n *\tNET_RX_DROP: packet was dropped\n */\n-int netif_receive_skb(struct sk_buff *skb)\n+static int __netif_receive_skb(struct sk_buff *skb)\n {\n \tstruct packet_type *ptype, *pt_prev;\n \tstruct net_device *orig_dev;\n@@ -2185,10 +2116,6 @@ int netif_receive_skb(struct sk_buff *skb)\n \tint ret = NET_RX_DROP;\n \t__be16 type;\n \n-\t/* if we've gotten here through NAPI, check netpoll */\n-\tif (netpoll_receive_skb(skb))\n-\t\treturn NET_RX_DROP;\n-\n \tif (!skb->tstamp.tv64)\n \t\tnet_timestamp(skb);\n \n@@ -2275,45 +2202,152 @@ out:\n \treturn ret;\n }\n \n-/* Network device is going away, flush any packets still pending */\n-static void flush_backlog(void *arg)\n+static void net_receive_action(struct softirq_action *h)\n {\n-\tstruct net_device *dev = arg;\n-\tstruct softnet_data *queue = &__get_cpu_var(softnet_data);\n-\tstruct sk_buff *skb, *tmp;\n+\tstruct list_head *cpu_list, local_list;\n \n-\tskb_queue_walk_safe(&queue->input_pkt_queue, skb, tmp)\n-\t\tif (skb->dev == dev) {\n-\t\t\t__skb_unlink(skb, &queue->input_pkt_queue);\n-\t\t\tkfree_skb(skb);\n-\t\t}\n+\tlocal_irq_disable();\n+\tcpu_list = &__get_cpu_var(softirq_work_list[NET_RECEIVE_SOFTIRQ]);\n+\tlist_replace_init(cpu_list, &local_list);\n+\tlocal_irq_enable();\n+\n+\twhile (!list_empty(&local_list)) {\n+\t\tstruct sk_buff *skb;\n+\n+\t\tskb = list_entry(local_list.next, struct sk_buff, csd.list);\n+\t\tlist_del_init(&skb->csd.list);\n+\t\t__netif_receive_skb(skb);\n+\t}\n }\n \n-static int process_backlog(struct napi_struct *napi, int quota)\n+static u16 *rxflow_cpu_map;\n+static int rxflow_num_cpus;\n+\n+/* skb->data points at the network header, but that is the only thing\n+ * we can rely upon.\n+ */\n+static u16 simple_rx_hash(struct sk_buff *skb)\n {\n-\tint work = 0;\n-\tstruct softnet_data *queue = &__get_cpu_var(softnet_data);\n-\tunsigned long start_time = jiffies;\n+\tu32 addr1, addr2, ports;\n+\tstruct ipv6hdr *ip6;\n+\tstruct iphdr *ip;\n+\tu32 hash, ihl;\n+\tu8 ip_proto;\n \n-\tnapi->weight = weight_p;\n-\tdo {\n-\t\tstruct sk_buff *skb;\n+\tif (unlikely(!simple_hashrnd_initialized)) {\n+\t\tget_random_bytes(&simple_hashrnd, 4);\n+\t\tsimple_hashrnd_initialized = 1;\n+\t}\n \n-\t\tlocal_irq_disable();\n-\t\tskb = __skb_dequeue(&queue->input_pkt_queue);\n-\t\tif (!skb) {\n-\t\t\t__napi_complete(napi);\n-\t\t\tlocal_irq_enable();\n-\t\t\tbreak;\n-\t\t}\n-\t\tlocal_irq_enable();\n+\tswitch (skb->protocol) {\n+\tcase __constant_htons(ETH_P_IP):\n+\t\tif (!pskb_may_pull(skb, sizeof(*ip)))\n+\t\t\treturn 0;\n \n-\t\tnetif_receive_skb(skb);\n-\t} while (++work < quota && jiffies == start_time);\n+\t\tip = (struct iphdr *) skb->data;\n+\t\tip_proto = ip->protocol;\n+\t\taddr1 = ip->saddr;\n+\t\taddr2 = ip->daddr;\n+\t\tihl = ip->ihl;\n+\t\tbreak;\n+\tcase __constant_htons(ETH_P_IPV6):\n+\t\tif (!pskb_may_pull(skb, sizeof(*ip6)))\n+\t\t\treturn 0;\n+\n+\t\tip6 = (struct ipv6hdr *) skb->data;\n+\t\tip_proto = ip6->nexthdr;\n+\t\taddr1 = ip6->saddr.s6_addr32[3];\n+\t\taddr2 = ip6->daddr.s6_addr32[3];\n+\t\tihl = (40 >> 2);\n+\t\tbreak;\n+\tdefault:\n+\t\treturn 0;\n+\t}\n+\n+\tports = 0;\n+\tswitch (ip_proto) {\n+\tcase IPPROTO_TCP:\n+\tcase IPPROTO_UDP:\n+\tcase IPPROTO_DCCP:\n+\tcase IPPROTO_ESP:\n+\tcase IPPROTO_AH:\n+\tcase IPPROTO_SCTP:\n+\tcase IPPROTO_UDPLITE:\n+\t\tif (pskb_may_pull(skb, (ihl * 4) + 4))\n+\t\t\tports = *((u32 *) (skb->data + (ihl * 4)));\n+\t\tbreak;\n \n-\treturn work;\n+\tdefault:\n+\t\tbreak;\n+\t}\n+\n+\thash = jhash_3words(addr1, addr2, ports, simple_hashrnd);\n+\n+\treturn (u16) (((u64) hash * rxflow_num_cpus) >> 32);\n }\n \n+/* Since we are already in softirq context via NAPI, it makes no\n+ * sense to reschedule a softirq locally, so we optimize that case.\n+ */\n+int netif_receive_skb(struct sk_buff *skb)\n+{\n+\tint target_cpu, this_cpu, do_direct;\n+\tunsigned long flags;\n+\n+\t/* If we've gotten here through NAPI, check netpoll. This part\n+\t * has to be synchronous and not get pushed to remote softirq\n+\t * receive packet processing.\n+\t */\n+\tif (netpoll_receive_skb(skb))\n+\t\treturn NET_RX_DROP;\n+\n+\ttarget_cpu = rxflow_cpu_map[simple_rx_hash(skb)];\n+\n+\tlocal_irq_save(flags);\n+\tthis_cpu = smp_processor_id();\n+\tdo_direct = 0;\n+\tif (target_cpu != this_cpu)\n+\t\t__send_remote_softirq(&skb->csd, target_cpu, this_cpu, NET_RECEIVE_SOFTIRQ);\n+\telse\n+\t\tdo_direct = 1;\n+\n+\tlocal_irq_restore(flags);\n+\n+\tif (do_direct)\n+\t\treturn __netif_receive_skb(skb);\n+\n+\treturn NET_RX_SUCCESS;\n+}\n+\n+int netif_rx(struct sk_buff *skb)\n+{\n+\tint target_cpu;\n+\n+\t/* if netpoll wants it, pretend we never saw it */\n+\tif (netpoll_rx(skb))\n+\t\treturn NET_RX_DROP;\n+\n+\ttarget_cpu = rxflow_cpu_map[simple_rx_hash(skb)];\n+\tsend_remote_softirq(&skb->csd, target_cpu, NET_RECEIVE_SOFTIRQ);\n+\n+\treturn NET_RX_SUCCESS;\n+}\n+\n+int netif_rx_ni(struct sk_buff *skb)\n+{\n+\tint err;\n+\n+\tpreempt_disable();\n+\terr = netif_rx(skb);\n+\tif (local_softirq_pending())\n+\t\tdo_softirq();\n+\tpreempt_enable();\n+\n+\treturn err;\n+}\n+\n+EXPORT_SYMBOL(netif_rx_ni);\n+\n /**\n * __napi_schedule - schedule for receive\n * @n: entry to schedule\n@@ -4182,8 +4216,6 @@ void netdev_run_todo(void)\n \n \t\tdev->reg_state = NETREG_UNREGISTERED;\n \n-\t\ton_each_cpu(flush_backlog, dev, 1);\n-\n \t\tnetdev_wait_allrefs(dev);\n \n \t\t/* paranoia */\n@@ -4489,7 +4521,6 @@ static int dev_cpu_callback(struct notifier_block *nfb,\n {\n \tstruct sk_buff **list_skb;\n \tstruct Qdisc **list_net;\n-\tstruct sk_buff *skb;\n \tunsigned int cpu, oldcpu = (unsigned long)ocpu;\n \tstruct softnet_data *sd, *oldsd;\n \n@@ -4520,10 +4551,6 @@ static int dev_cpu_callback(struct notifier_block *nfb,\n \traise_softirq_irqoff(NET_TX_SOFTIRQ);\n \tlocal_irq_enable();\n \n-\t/* Process offline CPU's input_pkt_queue */\n-\twhile ((skb = __skb_dequeue(&oldsd->input_pkt_queue)))\n-\t\tnetif_rx(skb);\n-\n \treturn NOTIFY_OK;\n }\n \n@@ -4793,7 +4820,7 @@ static struct pernet_operations __net_initdata default_device_ops = {\n */\n static int __init net_dev_init(void)\n {\n-\tint i, rc = -ENOMEM;\n+\tint i, index, rc = -ENOMEM;\n \n \tBUG_ON(!dev_boot_phase);\n \n@@ -4813,6 +4840,15 @@ static int __init net_dev_init(void)\n \tif (register_pernet_device(&default_device_ops))\n \t\tgoto out;\n \n+\trxflow_cpu_map = kzalloc(sizeof(u16) * num_possible_cpus(), GFP_KERNEL);\n+\tif (!rxflow_cpu_map)\n+\t\tgoto out;\n+\trxflow_num_cpus = num_online_cpus();\n+\n+\tindex = 0;\n+\tfor_each_online_cpu(i)\n+\t\trxflow_cpu_map[index++] = i;\n+\n \t/*\n \t *\tInitialise the packet receive queues.\n \t */\n@@ -4821,12 +4857,8 @@ static int __init net_dev_init(void)\n \t\tstruct softnet_data *queue;\n \n \t\tqueue = &per_cpu(softnet_data, i);\n-\t\tskb_queue_head_init(&queue->input_pkt_queue);\n \t\tqueue->completion_queue = NULL;\n \t\tINIT_LIST_HEAD(&queue->poll_list);\n-\n-\t\tqueue->backlog.poll = process_backlog;\n-\t\tqueue->backlog.weight = weight_p;\n \t}\n \n \tnetdev_dma_register();\n@@ -4835,6 +4867,7 @@ static int __init net_dev_init(void)\n \n \topen_softirq(NET_TX_SOFTIRQ, net_tx_action);\n \topen_softirq(NET_RX_SOFTIRQ, net_rx_action);\n+\topen_softirq(NET_RECEIVE_SOFTIRQ, net_receive_action);\n \n \thotcpu_notifier(dev_cpu_callback, 0);\n \tdst_init();\n", "prefixes": [ "0/2" ] }