Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/711995/?format=api
{ "id": 711995, "url": "http://patchwork.ozlabs.org/api/patches/711995/?format=api", "web_url": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20170106161125.2030.2948.stgit@localhost.localdomain/", "project": { "id": 46, "url": "http://patchwork.ozlabs.org/api/projects/46/?format=api", "name": "Intel Wired Ethernet development", "link_name": "intel-wired-lan", "list_id": "intel-wired-lan.osuosl.org", "list_email": "intel-wired-lan@osuosl.org", "web_url": "", "scm_url": "", "webscm_url": "", "list_archive_url": "", "list_archive_url_format": "", "commit_url_format": "" }, "msgid": "<20170106161125.2030.2948.stgit@localhost.localdomain>", "list_archive_url": null, "date": "2017-01-06T16:11:25", "name": "[next,8/9] igb: Break out Rx buffer page management", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "63bae797eb7e3b5fa77fd08453e51ce6cb8d749d", "submitter": { "id": 252, "url": "http://patchwork.ozlabs.org/api/people/252/?format=api", "name": "Alexander Duyck", "email": "alexander.duyck@gmail.com" }, "delegate": { "id": 68, "url": "http://patchwork.ozlabs.org/api/users/68/?format=api", "username": "jtkirshe", "first_name": "Jeff", "last_name": "Kirsher", "email": "jeffrey.t.kirsher@intel.com" }, "mbox": "http://patchwork.ozlabs.org/project/intel-wired-lan/patch/20170106161125.2030.2948.stgit@localhost.localdomain/mbox/", "series": [], "comments": "http://patchwork.ozlabs.org/api/patches/711995/comments/", "check": "pending", "checks": "http://patchwork.ozlabs.org/api/patches/711995/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<intel-wired-lan-bounces@lists.osuosl.org>", "X-Original-To": [ "incoming@patchwork.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Delivered-To": [ "patchwork-incoming@bilbo.ozlabs.org", "intel-wired-lan@lists.osuosl.org" ], "Received": [ "from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138])\n\t(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby ozlabs.org (Postfix) with ESMTPS id 3tw8gT45CZz9sXx\n\tfor <incoming@patchwork.ozlabs.org>;\n\tSat, 7 Jan 2017 03:11:33 +1100 (AEDT)", "from localhost (localhost [127.0.0.1])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id 07CA9820CE;\n\tFri, 6 Jan 2017 16:11:32 +0000 (UTC)", "from whitealder.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id Zmeqp7dfikVc; Fri, 6 Jan 2017 16:11:29 +0000 (UTC)", "from ash.osuosl.org (ash.osuosl.org [140.211.166.34])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id 08A8D823C9;\n\tFri, 6 Jan 2017 16:11:29 +0000 (UTC)", "from whitealder.osuosl.org (smtp1.osuosl.org [140.211.166.138])\n\tby ash.osuosl.org (Postfix) with ESMTP id 0482E1BFA57\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tFri, 6 Jan 2017 16:11:28 +0000 (UTC)", "from localhost (localhost [127.0.0.1])\n\tby whitealder.osuosl.org (Postfix) with ESMTP id EF575820CE\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tFri, 6 Jan 2017 16:11:27 +0000 (UTC)", "from whitealder.osuosl.org ([127.0.0.1])\n\tby localhost (.osuosl.org [127.0.0.1]) (amavisd-new, port 10024)\n\twith ESMTP id m7vu7Im1fYGk for <intel-wired-lan@lists.osuosl.org>;\n\tFri, 6 Jan 2017 16:11:27 +0000 (UTC)", "from mail-pf0-f193.google.com (mail-pf0-f193.google.com\n\t[209.85.192.193])\n\tby whitealder.osuosl.org (Postfix) with ESMTPS id 2C1AA8707F\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tFri, 6 Jan 2017 16:11:27 +0000 (UTC)", "by mail-pf0-f193.google.com with SMTP id b22so3775506pfd.3\n\tfor <intel-wired-lan@lists.osuosl.org>;\n\tFri, 06 Jan 2017 08:11:27 -0800 (PST)", "from localhost.localdomain ([2001:470:b:9c3:9e5c:8eff:fe4f:f2d0])\n\tby smtp.gmail.com with ESMTPSA id\n\tq23sm113121584pfg.47.2017.01.06.08.11.25\n\t(version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);\n\tFri, 06 Jan 2017 08:11:26 -0800 (PST)" ], "Authentication-Results": "ozlabs.org;\n\tdkim=fail reason=\"signature verification failed\" (2048-bit key;\n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"JyiaVBh3\"; dkim-atps=neutral", "X-Virus-Scanned": [ "amavisd-new at osuosl.org", "amavisd-new at osuosl.org" ], "X-Greylist": "domain auto-whitelisted by SQLgrey-1.7.6", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025;\n\th=subject:from:to:date:message-id:in-reply-to:references:user-agent\n\t:mime-version:content-transfer-encoding;\n\tbh=sez+Tv4jChjSxIzye1yyEphvm+zHbgVkYj9x3vi6yyI=;\n\tb=JyiaVBh3itPp0gJ5v/1fZvUi1ugFvOJpA3hwlUmf0ZP9iaFQ6vtZV1FczE589O2laF\n\ttfdIb8zovsblZPOJNI1ldZebI8QhDCgRyOwRmI1zFNwp1OEDofQw3Y0oXosZSShDSkj2\n\tfuC96261RwIOVXcAILtzXfkG6eOWLTsOcCMRqBC7J2zlCiU7gIBLxx1pnGxFVcPbCbf7\n\tOG4PI/SAe2r82Z7GldW9z1BISV0GBq3i2DHbmIkdUL+qDyHligBxkjYyRmQ1mvRRTGda\n\tILTgVMAM7hv7t/VXbnVhufyFcCwIaf9K9fy9KgrNjEwk3gA/AswFFQ/xg8hipkKh7+Vs\n\tv9tA==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:subject:from:to:date:message-id:in-reply-to\n\t:references:user-agent:mime-version:content-transfer-encoding;\n\tbh=sez+Tv4jChjSxIzye1yyEphvm+zHbgVkYj9x3vi6yyI=;\n\tb=sxeqvPnXheukRAsIDvBaZ3DBNZiPmp0e5d2nwBf7lFzuzMZl76absuPkytz6PQxtrn\n\teYwDlSAMKVpiVkiA/kkkh3rAV/D+Y90BBriLn7SpV0DmP7P5sPDqJkJLYkhxSJPmA1QR\n\tNDMUzz8OVKQcZY5uW3UmCldPBniuf6du0YCY6dILNXba+pUJPg8rmKqx0ItY0SzOjrQB\n\tr4AXxlNzK37nXxp7X8+MZjnebd5eKUtJ/Eia1l7tkSBLuaIcR1iscitN38avNEOnO8Mn\n\taqvbE8eyjQF5OhTuvuSaVE1hO3f32c9WWBRnSzm5sXCYMO2jFS12gGs0bx20GbuJ5jcO\n\t1rtA==", "X-Gm-Message-State": "AIkVDXK7BuKQqRKqtWJQoHcnxOZxaXjT5KHHGX8ZfkB/lSrmZfUFstzuhf6kKedyUcfP6Q==", "X-Received": "by 10.99.48.68 with SMTP id w65mr143074110pgw.68.1483719086574; \n\tFri, 06 Jan 2017 08:11:26 -0800 (PST)", "From": "Alexander Duyck <alexander.duyck@gmail.com>", "To": "intel-wired-lan@lists.osuosl.org, jeffrey.t.kirsher@intel.com", "Date": "Fri, 06 Jan 2017 08:11:25 -0800", "Message-ID": "<20170106161125.2030.2948.stgit@localhost.localdomain>", "In-Reply-To": "<20170106160850.2030.39073.stgit@localhost.localdomain>", "References": "<20170106160850.2030.39073.stgit@localhost.localdomain>", "User-Agent": "StGit/0.17.1-dirty", "MIME-Version": "1.0", "Subject": "[Intel-wired-lan] [next PATCH 8/9] igb: Break out Rx buffer page\n\tmanagement", "X-BeenThere": "intel-wired-lan@lists.osuosl.org", "X-Mailman-Version": "2.1.18-1", "Precedence": "list", "List-Id": "Intel Wired Ethernet Linux Kernel Driver Development\n\t<intel-wired-lan.lists.osuosl.org>", "List-Unsubscribe": "<http://lists.osuosl.org/mailman/options/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@lists.osuosl.org?subject=unsubscribe>", "List-Archive": "<http://lists.osuosl.org/pipermail/intel-wired-lan/>", "List-Post": "<mailto:intel-wired-lan@lists.osuosl.org>", "List-Help": "<mailto:intel-wired-lan-request@lists.osuosl.org?subject=help>", "List-Subscribe": "<http://lists.osuosl.org/mailman/listinfo/intel-wired-lan>, \n\t<mailto:intel-wired-lan-request@lists.osuosl.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "intel-wired-lan-bounces@lists.osuosl.org", "Sender": "\"Intel-wired-lan\" <intel-wired-lan-bounces@lists.osuosl.org>" }, "content": "From: Alexander Duyck <alexander.h.duyck@intel.com>\n\nAt this point we have 2 to 3 paths that can be taken depending on what Rx\nmodes are enabled. In order to better support that and improve the\nmaintainability I am breaking out the common bits from those paths and\nmaking them into their own functions.\n\nSigned-off-by: Alexander Duyck <alexander.h.duyck@intel.com>\n---\n drivers/net/ethernet/intel/igb/igb_main.c | 237 +++++++++++++++--------------\n 1 file changed, 123 insertions(+), 114 deletions(-)", "diff": "diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c\nindex 246675a372bf..dfb04d7ba502 100644\n--- a/drivers/net/ethernet/intel/igb/igb_main.c\n+++ b/drivers/net/ethernet/intel/igb/igb_main.c\n@@ -6882,8 +6882,14 @@ static void igb_reuse_rx_page(struct igb_ring *rx_ring,\n \tnta++;\n \trx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;\n \n-\t/* transfer page from old buffer to new buffer */\n-\t*new_buff = *old_buff;\n+\t/* Transfer page from old buffer to new buffer.\n+\t * Move each member individually to avoid possible store\n+\t * forwarding stalls.\n+\t */\n+\tnew_buff->dma\t\t= old_buff->dma;\n+\tnew_buff->page\t\t= old_buff->page;\n+\tnew_buff->page_offset\t= old_buff->page_offset;\n+\tnew_buff->pagecnt_bias\t= old_buff->pagecnt_bias;\n }\n \n static inline bool igb_page_is_reserved(struct page *page)\n@@ -6891,11 +6897,10 @@ static inline bool igb_page_is_reserved(struct page *page)\n \treturn (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);\n }\n \n-static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,\n-\t\t\t\t struct page *page,\n-\t\t\t\t const unsigned int truesize)\n+static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer)\n {\n-\tunsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;\n+\tunsigned int pagecnt_bias = rx_buffer->pagecnt_bias;\n+\tstruct page *page = rx_buffer->page;\n \n \t/* avoid re-using remote pages */\n \tif (unlikely(igb_page_is_reserved(page)))\n@@ -6903,14 +6908,9 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,\n \n #if (PAGE_SIZE < 8192)\n \t/* if we are only owner of page we can reuse it */\n-\tif (unlikely(page_ref_count(page) != pagecnt_bias))\n+\tif (unlikely((page_ref_count(page) - pagecnt_bias) > 1))\n \t\treturn false;\n-\n-\t/* flip page offset to other buffer */\n-\trx_buffer->page_offset ^= truesize;\n #else\n-\t/* move offset up to the next cache line */\n-\trx_buffer->page_offset += truesize;\n #define IGB_LAST_OFFSET \\\n \t(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)\n \n@@ -6922,7 +6922,7 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,\n \t * the pagecnt_bias and page count so that we fully restock the\n \t * number of references the driver holds.\n \t */\n-\tif (unlikely(pagecnt_bias == 1)) {\n+\tif (unlikely(!pagecnt_bias)) {\n \t\tpage_ref_add(page, USHRT_MAX);\n \t\trx_buffer->pagecnt_bias = USHRT_MAX;\n \t}\n@@ -6934,26 +6934,16 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,\n * igb_add_rx_frag - Add contents of Rx buffer to sk_buff\n * @rx_ring: rx descriptor ring to transact packets on\n * @rx_buffer: buffer containing page to add\n- * @rx_desc: descriptor containing length of buffer written by hardware\n * @skb: sk_buff to place the data into\n+ * @size: size of buffer to be added\n *\n * This function will add the data contained in rx_buffer->page to the skb.\n- * This is done either through a direct copy if the data in the buffer is\n- * less than the skb header size, otherwise it will just attach the page as\n- * a frag to the skb.\n- *\n- * The function will then update the page offset if necessary and return\n- * true if the buffer can be reused by the adapter.\n **/\n-static bool igb_add_rx_frag(struct igb_ring *rx_ring,\n+static void igb_add_rx_frag(struct igb_ring *rx_ring,\n \t\t\t struct igb_rx_buffer *rx_buffer,\n-\t\t\t unsigned int size,\n-\t\t\t union e1000_adv_rx_desc *rx_desc,\n-\t\t\t struct sk_buff *skb)\n+\t\t\t struct sk_buff *skb,\n+\t\t\t unsigned int size)\n {\n-\tstruct page *page = rx_buffer->page;\n-\tunsigned char *va = page_address(page) + rx_buffer->page_offset;\n-\tunsigned int pull_len;\n #if (PAGE_SIZE < 8192)\n \tunsigned int truesize = igb_rx_pg_size(rx_ring) / 2;\n #else\n@@ -6961,9 +6951,39 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,\n \t\t\t\tSKB_DATA_ALIGN(IGB_SKB_PAD + size) :\n \t\t\t\tSKB_DATA_ALIGN(size);\n #endif\n+\tskb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,\n+\t\t\trx_buffer->page_offset, size, truesize);\n+#if (PAGE_SIZE < 8192)\n+\trx_buffer->page_offset ^= truesize;\n+#else\n+\trx_buffer->page_offset += truesize;\n+#endif\n+}\n+\n+static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,\n+\t\t\t\t\t struct igb_rx_buffer *rx_buffer,\n+\t\t\t\t\t union e1000_adv_rx_desc *rx_desc,\n+\t\t\t\t\t unsigned int size)\n+{\n+\tvoid *va = page_address(rx_buffer->page) + rx_buffer->page_offset;\n+#if (PAGE_SIZE < 8192)\n+\tunsigned int truesize = igb_rx_pg_size(rx_ring) / 2;\n+#else\n+\tunsigned int truesize = SKB_DATA_ALIGN(size);\n+#endif\n+\tunsigned int headlen;\n+\tstruct sk_buff *skb;\n \n-\tif (unlikely(skb_is_nonlinear(skb)))\n-\t\tgoto add_tail_frag;\n+\t/* prefetch first cache line of first page */\n+\tprefetch(va);\n+#if L1_CACHE_BYTES < 128\n+\tprefetch(va + L1_CACHE_BYTES);\n+#endif\n+\n+\t/* allocate a skb to store the frags */\n+\tskb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);\n+\tif (unlikely(!skb))\n+\t\treturn NULL;\n \n \tif (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) {\n \t\tigb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);\n@@ -6971,95 +6991,31 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,\n \t\tsize -= IGB_TS_HDR_LEN;\n \t}\n \n-\tif (likely(size <= IGB_RX_HDR_LEN)) {\n-\t\tmemcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));\n-\n-\t\t/* page is not reserved, we can reuse buffer as-is */\n-\t\tif (likely(!igb_page_is_reserved(page)))\n-\t\t\treturn true;\n-\n-\t\t/* this page cannot be reused so discard it */\n-\t\treturn false;\n-\t}\n-\n-\t/* we need the header to contain the greater of either ETH_HLEN or\n-\t * 60 bytes if the skb->len is less than 60 for skb_pad.\n-\t */\n-\tpull_len = eth_get_headlen(va, IGB_RX_HDR_LEN);\n+\t/* Determine available headroom for copy */\n+\theadlen = size;\n+\tif (headlen > IGB_RX_HDR_LEN)\n+\t\theadlen = eth_get_headlen(va, IGB_RX_HDR_LEN);\n+\telse\n+\t\theadlen = size;\n \n \t/* align pull length to size of long to optimize memcpy performance */\n-\tmemcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));\n+\tmemcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));\n \n \t/* update all of the pointers */\n-\tva += pull_len;\n-\tsize -= pull_len;\n-\n-add_tail_frag:\n-\tskb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,\n-\t\t\tva - page_address(page), size, truesize);\n-\n-\treturn igb_can_reuse_rx_page(rx_buffer, page, truesize);\n-}\n-\n-static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,\n-\t\t\t\t\t union e1000_adv_rx_desc *rx_desc,\n-\t\t\t\t\t struct sk_buff *skb)\n-{\n-\tunsigned int size = le16_to_cpu(rx_desc->wb.upper.length);\n-\tstruct igb_rx_buffer *rx_buffer;\n-\tstruct page *page;\n-\n-\trx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];\n-\tpage = rx_buffer->page;\n-\tprefetchw(page);\n-\n-\t/* we are reusing so sync this buffer for CPU use */\n-\tdma_sync_single_range_for_cpu(rx_ring->dev,\n-\t\t\t\t rx_buffer->dma,\n-\t\t\t\t rx_buffer->page_offset,\n-\t\t\t\t size,\n-\t\t\t\t DMA_FROM_DEVICE);\n-\n-\tif (likely(!skb)) {\n-\t\tvoid *va = page_address(page) + rx_buffer->page_offset;\n-\n-\t\t/* prefetch first cache line of first page */\n-\t\tprefetch(va);\n-#if L1_CACHE_BYTES < 128\n-\t\tprefetch(va + L1_CACHE_BYTES);\n+\tsize -= headlen;\n+\tif (size) {\n+\t\tskb_add_rx_frag(skb, 0, rx_buffer->page,\n+\t\t\t\t(va + headlen) - page_address(rx_buffer->page),\n+\t\t\t\tsize, truesize);\n+#if (PAGE_SIZE < 8192)\n+\t\trx_buffer->page_offset ^= truesize;\n+#else\n+\t\trx_buffer->page_offset += truesize;\n #endif\n-\n-\t\t/* allocate a skb to store the frags */\n-\t\tskb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);\n-\t\tif (unlikely(!skb)) {\n-\t\t\trx_ring->rx_stats.alloc_failed++;\n-\t\t\treturn NULL;\n-\t\t}\n-\n-\t\t/* we will be copying header into skb->data in\n-\t\t * pskb_may_pull so it is in our interest to prefetch\n-\t\t * it now to avoid a possible cache miss\n-\t\t */\n-\t\tprefetchw(skb->data);\n-\t}\n-\n-\t/* pull page into skb */\n-\tif (igb_add_rx_frag(rx_ring, rx_buffer, size, rx_desc, skb)) {\n-\t\t/* hand second half of page back to the ring */\n-\t\tigb_reuse_rx_page(rx_ring, rx_buffer);\n \t} else {\n-\t\t/* We are not reusing the buffer so unmap it and free\n-\t\t * any references we are holding to it\n-\t\t */\n-\t\tdma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,\n-\t\t\t\t igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,\n-\t\t\t\t IGB_RX_DMA_ATTR);\n-\t\t__page_frag_cache_drain(page, rx_buffer->pagecnt_bias);\n+\t\trx_buffer->pagecnt_bias++;\n \t}\n \n-\t/* clear contents of rx_buffer */\n-\trx_buffer->page = NULL;\n-\n \treturn skb;\n }\n \n@@ -7217,6 +7173,47 @@ static void igb_process_skb_fields(struct igb_ring *rx_ring,\n \tskb->protocol = eth_type_trans(skb, rx_ring->netdev);\n }\n \n+static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,\n+\t\t\t\t\t const unsigned int size)\n+{\n+\tstruct igb_rx_buffer *rx_buffer;\n+\n+\trx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];\n+\tprefetchw(rx_buffer->page);\n+\n+\t/* we are reusing so sync this buffer for CPU use */\n+\tdma_sync_single_range_for_cpu(rx_ring->dev,\n+\t\t\t\t rx_buffer->dma,\n+\t\t\t\t rx_buffer->page_offset,\n+\t\t\t\t size,\n+\t\t\t\t DMA_FROM_DEVICE);\n+\n+\trx_buffer->pagecnt_bias--;\n+\n+\treturn rx_buffer;\n+}\n+\n+static void igb_put_rx_buffer(struct igb_ring *rx_ring,\n+\t\t\t struct igb_rx_buffer *rx_buffer)\n+{\n+\tif (igb_can_reuse_rx_page(rx_buffer)) {\n+\t\t/* hand second half of page back to the ring */\n+\t\tigb_reuse_rx_page(rx_ring, rx_buffer);\n+\t} else {\n+\t\t/* We are not reusing the buffer so unmap it and free\n+\t\t * any references we are holding to it\n+\t\t */\n+\t\tdma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,\n+\t\t\t\t igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,\n+\t\t\t\t IGB_RX_DMA_ATTR);\n+\t\t__page_frag_cache_drain(rx_buffer->page,\n+\t\t\t\t\trx_buffer->pagecnt_bias);\n+\t}\n+\n+\t/* clear contents of rx_buffer */\n+\trx_buffer->page = NULL;\n+}\n+\n static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)\n {\n \tstruct igb_ring *rx_ring = q_vector->rx.ring;\n@@ -7226,6 +7223,8 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)\n \n \twhile (likely(total_packets < budget)) {\n \t\tunion e1000_adv_rx_desc *rx_desc;\n+\t\tstruct igb_rx_buffer *rx_buffer;\n+\t\tunsigned int size;\n \n \t\t/* return some buffers to hardware, one at a time is too slow */\n \t\tif (cleaned_count >= IGB_RX_BUFFER_WRITE) {\n@@ -7234,8 +7233,8 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)\n \t\t}\n \n \t\trx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);\n-\n-\t\tif (!rx_desc->wb.upper.length)\n+\t\tsize = le16_to_cpu(rx_desc->wb.upper.length);\n+\t\tif (!size)\n \t\t\tbreak;\n \n \t\t/* This memory barrier is needed to keep us from reading\n@@ -7244,13 +7243,23 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)\n \t\t */\n \t\tdma_rmb();\n \n+\t\trx_buffer = igb_get_rx_buffer(rx_ring, size);\n+\n \t\t/* retrieve a buffer from the ring */\n-\t\tskb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);\n+\t\tif (skb)\n+\t\t\tigb_add_rx_frag(rx_ring, rx_buffer, skb, size);\n+\t\telse\n+\t\t\tskb = igb_construct_skb(rx_ring, rx_buffer,\n+\t\t\t\t\t\trx_desc, size);\n \n \t\t/* exit if we failed to retrieve a buffer */\n-\t\tif (!skb)\n+\t\tif (!skb) {\n+\t\t\trx_ring->rx_stats.alloc_failed++;\n+\t\t\trx_buffer->pagecnt_bias++;\n \t\t\tbreak;\n+\t\t}\n \n+\t\tigb_put_rx_buffer(rx_ring, rx_buffer);\n \t\tcleaned_count++;\n \n \t\t/* fetch next buffer in frame if non-eop */\n", "prefixes": [ "next", "8/9" ] }